1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx600 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX6 %s
3; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
4; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
5; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
6; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
7; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-NOTTGSPLIT %s
8; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-TGSPLIT %s
9
10define amdgpu_kernel void @local_singlethread_unordered_load(
11; GFX6-LABEL: local_singlethread_unordered_load:
12; GFX6:       ; %bb.0: ; %entry
13; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
14; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
15; GFX6-NEXT:    s_mov_b32 m0, -1
16; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
17; GFX6-NEXT:    v_mov_b32_e32 v0, s0
18; GFX6-NEXT:    ds_read_b32 v0, v0
19; GFX6-NEXT:    v_mov_b32_e32 v1, s1
20; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
21; GFX6-NEXT:    ds_write_b32 v1, v0
22; GFX6-NEXT:    s_endpgm
23;
24; GFX7-LABEL: local_singlethread_unordered_load:
25; GFX7:       ; %bb.0: ; %entry
26; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
27; GFX7-NEXT:    s_mov_b32 m0, -1
28; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
29; GFX7-NEXT:    v_mov_b32_e32 v0, s0
30; GFX7-NEXT:    ds_read_b32 v0, v0
31; GFX7-NEXT:    v_mov_b32_e32 v1, s1
32; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
33; GFX7-NEXT:    ds_write_b32 v1, v0
34; GFX7-NEXT:    s_endpgm
35;
36; GFX10-WGP-LABEL: local_singlethread_unordered_load:
37; GFX10-WGP:       ; %bb.0: ; %entry
38; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
39; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
40; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
41; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
42; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
43; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
44; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
45; GFX10-WGP-NEXT:    s_endpgm
46;
47; GFX10-CU-LABEL: local_singlethread_unordered_load:
48; GFX10-CU:       ; %bb.0: ; %entry
49; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
50; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
51; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
52; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
53; GFX10-CU-NEXT:    ds_read_b32 v0, v0
54; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
55; GFX10-CU-NEXT:    ds_write_b32 v1, v0
56; GFX10-CU-NEXT:    s_endpgm
57;
58; SKIP-CACHE-INV-LABEL: local_singlethread_unordered_load:
59; SKIP-CACHE-INV:       ; %bb.0: ; %entry
60; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
61; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
62; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
63; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
64; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
65; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
66; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
67; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
68; SKIP-CACHE-INV-NEXT:    s_endpgm
69;
70; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_unordered_load:
71; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
72; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
73; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
74; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
75; GFX90A-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
76; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
77; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
78; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
79; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
80;
81; GFX90A-TGSPLIT-LABEL: local_singlethread_unordered_load:
82; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
83; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
84; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
85; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
86; GFX90A-TGSPLIT-NEXT:    ds_read_b32 v0, v0
87; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
88; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
89; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v1, v0
90; GFX90A-TGSPLIT-NEXT:    s_endpgm
91    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
92entry:
93  %val = load atomic i32, i32 addrspace(3)* %in syncscope("singlethread") unordered, align 4
94  store i32 %val, i32 addrspace(3)* %out
95  ret void
96}
97
98define amdgpu_kernel void @local_singlethread_monotonic_load(
99; GFX6-LABEL: local_singlethread_monotonic_load:
100; GFX6:       ; %bb.0: ; %entry
101; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
102; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
103; GFX6-NEXT:    s_mov_b32 m0, -1
104; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
105; GFX6-NEXT:    v_mov_b32_e32 v0, s0
106; GFX6-NEXT:    ds_read_b32 v0, v0
107; GFX6-NEXT:    v_mov_b32_e32 v1, s1
108; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
109; GFX6-NEXT:    ds_write_b32 v1, v0
110; GFX6-NEXT:    s_endpgm
111;
112; GFX7-LABEL: local_singlethread_monotonic_load:
113; GFX7:       ; %bb.0: ; %entry
114; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
115; GFX7-NEXT:    s_mov_b32 m0, -1
116; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
117; GFX7-NEXT:    v_mov_b32_e32 v0, s0
118; GFX7-NEXT:    ds_read_b32 v0, v0
119; GFX7-NEXT:    v_mov_b32_e32 v1, s1
120; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
121; GFX7-NEXT:    ds_write_b32 v1, v0
122; GFX7-NEXT:    s_endpgm
123;
124; GFX10-WGP-LABEL: local_singlethread_monotonic_load:
125; GFX10-WGP:       ; %bb.0: ; %entry
126; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
127; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
128; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
129; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
130; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
131; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
132; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
133; GFX10-WGP-NEXT:    s_endpgm
134;
135; GFX10-CU-LABEL: local_singlethread_monotonic_load:
136; GFX10-CU:       ; %bb.0: ; %entry
137; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
138; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
139; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
140; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
141; GFX10-CU-NEXT:    ds_read_b32 v0, v0
142; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
143; GFX10-CU-NEXT:    ds_write_b32 v1, v0
144; GFX10-CU-NEXT:    s_endpgm
145;
146; SKIP-CACHE-INV-LABEL: local_singlethread_monotonic_load:
147; SKIP-CACHE-INV:       ; %bb.0: ; %entry
148; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
149; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
150; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
151; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
152; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
153; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
154; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
155; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
156; SKIP-CACHE-INV-NEXT:    s_endpgm
157;
158; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_monotonic_load:
159; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
160; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
161; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
162; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
163; GFX90A-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
164; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
165; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
166; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
167; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
168;
169; GFX90A-TGSPLIT-LABEL: local_singlethread_monotonic_load:
170; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
171; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
172; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
173; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
174; GFX90A-TGSPLIT-NEXT:    ds_read_b32 v0, v0
175; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
176; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
177; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v1, v0
178; GFX90A-TGSPLIT-NEXT:    s_endpgm
179    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
180entry:
181  %val = load atomic i32, i32 addrspace(3)* %in syncscope("singlethread") monotonic, align 4
182  store i32 %val, i32 addrspace(3)* %out
183  ret void
184}
185
186define amdgpu_kernel void @local_singlethread_acquire_load(
187; GFX6-LABEL: local_singlethread_acquire_load:
188; GFX6:       ; %bb.0: ; %entry
189; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
190; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
191; GFX6-NEXT:    s_mov_b32 m0, -1
192; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
193; GFX6-NEXT:    v_mov_b32_e32 v0, s0
194; GFX6-NEXT:    ds_read_b32 v0, v0
195; GFX6-NEXT:    v_mov_b32_e32 v1, s1
196; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
197; GFX6-NEXT:    ds_write_b32 v1, v0
198; GFX6-NEXT:    s_endpgm
199;
200; GFX7-LABEL: local_singlethread_acquire_load:
201; GFX7:       ; %bb.0: ; %entry
202; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
203; GFX7-NEXT:    s_mov_b32 m0, -1
204; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
205; GFX7-NEXT:    v_mov_b32_e32 v0, s0
206; GFX7-NEXT:    ds_read_b32 v0, v0
207; GFX7-NEXT:    v_mov_b32_e32 v1, s1
208; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
209; GFX7-NEXT:    ds_write_b32 v1, v0
210; GFX7-NEXT:    s_endpgm
211;
212; GFX10-WGP-LABEL: local_singlethread_acquire_load:
213; GFX10-WGP:       ; %bb.0: ; %entry
214; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
215; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
216; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
217; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
218; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
219; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
220; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
221; GFX10-WGP-NEXT:    s_endpgm
222;
223; GFX10-CU-LABEL: local_singlethread_acquire_load:
224; GFX10-CU:       ; %bb.0: ; %entry
225; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
226; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
227; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
228; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
229; GFX10-CU-NEXT:    ds_read_b32 v0, v0
230; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
231; GFX10-CU-NEXT:    ds_write_b32 v1, v0
232; GFX10-CU-NEXT:    s_endpgm
233;
234; SKIP-CACHE-INV-LABEL: local_singlethread_acquire_load:
235; SKIP-CACHE-INV:       ; %bb.0: ; %entry
236; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
237; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
238; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
239; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
240; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
241; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
242; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
243; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
244; SKIP-CACHE-INV-NEXT:    s_endpgm
245;
246; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acquire_load:
247; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
248; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
249; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
250; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
251; GFX90A-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
252; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
253; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
254; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
255; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
256;
257; GFX90A-TGSPLIT-LABEL: local_singlethread_acquire_load:
258; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
259; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
260; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
261; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
262; GFX90A-TGSPLIT-NEXT:    ds_read_b32 v0, v0
263; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
264; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
265; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v1, v0
266; GFX90A-TGSPLIT-NEXT:    s_endpgm
267    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
268entry:
269  %val = load atomic i32, i32 addrspace(3)* %in syncscope("singlethread") acquire, align 4
270  store i32 %val, i32 addrspace(3)* %out
271  ret void
272}
273
274define amdgpu_kernel void @local_singlethread_seq_cst_load(
275; GFX6-LABEL: local_singlethread_seq_cst_load:
276; GFX6:       ; %bb.0: ; %entry
277; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
278; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
279; GFX6-NEXT:    s_mov_b32 m0, -1
280; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
281; GFX6-NEXT:    v_mov_b32_e32 v0, s0
282; GFX6-NEXT:    ds_read_b32 v0, v0
283; GFX6-NEXT:    v_mov_b32_e32 v1, s1
284; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
285; GFX6-NEXT:    ds_write_b32 v1, v0
286; GFX6-NEXT:    s_endpgm
287;
288; GFX7-LABEL: local_singlethread_seq_cst_load:
289; GFX7:       ; %bb.0: ; %entry
290; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
291; GFX7-NEXT:    s_mov_b32 m0, -1
292; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
293; GFX7-NEXT:    v_mov_b32_e32 v0, s0
294; GFX7-NEXT:    ds_read_b32 v0, v0
295; GFX7-NEXT:    v_mov_b32_e32 v1, s1
296; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
297; GFX7-NEXT:    ds_write_b32 v1, v0
298; GFX7-NEXT:    s_endpgm
299;
300; GFX10-WGP-LABEL: local_singlethread_seq_cst_load:
301; GFX10-WGP:       ; %bb.0: ; %entry
302; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
303; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
304; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
305; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
306; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
307; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
308; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
309; GFX10-WGP-NEXT:    s_endpgm
310;
311; GFX10-CU-LABEL: local_singlethread_seq_cst_load:
312; GFX10-CU:       ; %bb.0: ; %entry
313; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
314; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
315; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
316; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
317; GFX10-CU-NEXT:    ds_read_b32 v0, v0
318; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
319; GFX10-CU-NEXT:    ds_write_b32 v1, v0
320; GFX10-CU-NEXT:    s_endpgm
321;
322; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_load:
323; SKIP-CACHE-INV:       ; %bb.0: ; %entry
324; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
325; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
326; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
327; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
328; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
329; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
330; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
331; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
332; SKIP-CACHE-INV-NEXT:    s_endpgm
333;
334; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_load:
335; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
336; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
337; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
338; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
339; GFX90A-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
340; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
341; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
342; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
343; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
344;
345; GFX90A-TGSPLIT-LABEL: local_singlethread_seq_cst_load:
346; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
347; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
348; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
349; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
350; GFX90A-TGSPLIT-NEXT:    ds_read_b32 v0, v0
351; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
352; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
353; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v1, v0
354; GFX90A-TGSPLIT-NEXT:    s_endpgm
355    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
356entry:
357  %val = load atomic i32, i32 addrspace(3)* %in syncscope("singlethread") seq_cst, align 4
358  store i32 %val, i32 addrspace(3)* %out
359  ret void
360}
361
362define amdgpu_kernel void @local_singlethread_unordered_store(
363; GFX6-LABEL: local_singlethread_unordered_store:
364; GFX6:       ; %bb.0: ; %entry
365; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x1
366; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x0
367; GFX6-NEXT:    s_mov_b32 m0, -1
368; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
369; GFX6-NEXT:    v_mov_b32_e32 v0, s0
370; GFX6-NEXT:    v_mov_b32_e32 v1, s1
371; GFX6-NEXT:    ds_write_b32 v0, v1
372; GFX6-NEXT:    s_endpgm
373;
374; GFX7-LABEL: local_singlethread_unordered_store:
375; GFX7:       ; %bb.0: ; %entry
376; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
377; GFX7-NEXT:    s_mov_b32 m0, -1
378; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
379; GFX7-NEXT:    v_mov_b32_e32 v0, s1
380; GFX7-NEXT:    v_mov_b32_e32 v1, s0
381; GFX7-NEXT:    ds_write_b32 v0, v1
382; GFX7-NEXT:    s_endpgm
383;
384; GFX10-WGP-LABEL: local_singlethread_unordered_store:
385; GFX10-WGP:       ; %bb.0: ; %entry
386; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
387; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
388; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
389; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
390; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
391; GFX10-WGP-NEXT:    s_endpgm
392;
393; GFX10-CU-LABEL: local_singlethread_unordered_store:
394; GFX10-CU:       ; %bb.0: ; %entry
395; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
396; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
397; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
398; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
399; GFX10-CU-NEXT:    ds_write_b32 v0, v1
400; GFX10-CU-NEXT:    s_endpgm
401;
402; SKIP-CACHE-INV-LABEL: local_singlethread_unordered_store:
403; SKIP-CACHE-INV:       ; %bb.0: ; %entry
404; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
405; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
406; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
407; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
408; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
409; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
410; SKIP-CACHE-INV-NEXT:    s_endpgm
411;
412; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_unordered_store:
413; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
414; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
415; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
416; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
417; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
418; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
419; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
420;
421; GFX90A-TGSPLIT-LABEL: local_singlethread_unordered_store:
422; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
423; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
424; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
425; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
426; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
427; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
428; GFX90A-TGSPLIT-NEXT:    s_endpgm
429    i32 %in, i32 addrspace(3)* %out) {
430entry:
431  store atomic i32 %in, i32 addrspace(3)* %out syncscope("singlethread") unordered, align 4
432  ret void
433}
434
435define amdgpu_kernel void @local_singlethread_monotonic_store(
436; GFX6-LABEL: local_singlethread_monotonic_store:
437; GFX6:       ; %bb.0: ; %entry
438; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x1
439; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x0
440; GFX6-NEXT:    s_mov_b32 m0, -1
441; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
442; GFX6-NEXT:    v_mov_b32_e32 v0, s0
443; GFX6-NEXT:    v_mov_b32_e32 v1, s1
444; GFX6-NEXT:    ds_write_b32 v0, v1
445; GFX6-NEXT:    s_endpgm
446;
447; GFX7-LABEL: local_singlethread_monotonic_store:
448; GFX7:       ; %bb.0: ; %entry
449; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
450; GFX7-NEXT:    s_mov_b32 m0, -1
451; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
452; GFX7-NEXT:    v_mov_b32_e32 v0, s1
453; GFX7-NEXT:    v_mov_b32_e32 v1, s0
454; GFX7-NEXT:    ds_write_b32 v0, v1
455; GFX7-NEXT:    s_endpgm
456;
457; GFX10-WGP-LABEL: local_singlethread_monotonic_store:
458; GFX10-WGP:       ; %bb.0: ; %entry
459; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
460; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
461; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
462; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
463; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
464; GFX10-WGP-NEXT:    s_endpgm
465;
466; GFX10-CU-LABEL: local_singlethread_monotonic_store:
467; GFX10-CU:       ; %bb.0: ; %entry
468; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
469; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
470; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
471; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
472; GFX10-CU-NEXT:    ds_write_b32 v0, v1
473; GFX10-CU-NEXT:    s_endpgm
474;
475; SKIP-CACHE-INV-LABEL: local_singlethread_monotonic_store:
476; SKIP-CACHE-INV:       ; %bb.0: ; %entry
477; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
478; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
479; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
480; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
481; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
482; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
483; SKIP-CACHE-INV-NEXT:    s_endpgm
484;
485; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_monotonic_store:
486; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
487; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
488; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
489; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
490; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
491; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
492; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
493;
494; GFX90A-TGSPLIT-LABEL: local_singlethread_monotonic_store:
495; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
496; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
497; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
498; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
499; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
500; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
501; GFX90A-TGSPLIT-NEXT:    s_endpgm
502    i32 %in, i32 addrspace(3)* %out) {
503entry:
504  store atomic i32 %in, i32 addrspace(3)* %out syncscope("singlethread") monotonic, align 4
505  ret void
506}
507
508define amdgpu_kernel void @local_singlethread_release_store(
509; GFX6-LABEL: local_singlethread_release_store:
510; GFX6:       ; %bb.0: ; %entry
511; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x1
512; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x0
513; GFX6-NEXT:    s_mov_b32 m0, -1
514; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
515; GFX6-NEXT:    v_mov_b32_e32 v0, s0
516; GFX6-NEXT:    v_mov_b32_e32 v1, s1
517; GFX6-NEXT:    ds_write_b32 v0, v1
518; GFX6-NEXT:    s_endpgm
519;
520; GFX7-LABEL: local_singlethread_release_store:
521; GFX7:       ; %bb.0: ; %entry
522; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
523; GFX7-NEXT:    s_mov_b32 m0, -1
524; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
525; GFX7-NEXT:    v_mov_b32_e32 v0, s1
526; GFX7-NEXT:    v_mov_b32_e32 v1, s0
527; GFX7-NEXT:    ds_write_b32 v0, v1
528; GFX7-NEXT:    s_endpgm
529;
530; GFX10-WGP-LABEL: local_singlethread_release_store:
531; GFX10-WGP:       ; %bb.0: ; %entry
532; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
533; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
534; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
535; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
536; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
537; GFX10-WGP-NEXT:    s_endpgm
538;
539; GFX10-CU-LABEL: local_singlethread_release_store:
540; GFX10-CU:       ; %bb.0: ; %entry
541; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
542; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
543; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
544; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
545; GFX10-CU-NEXT:    ds_write_b32 v0, v1
546; GFX10-CU-NEXT:    s_endpgm
547;
548; SKIP-CACHE-INV-LABEL: local_singlethread_release_store:
549; SKIP-CACHE-INV:       ; %bb.0: ; %entry
550; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
551; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
552; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
553; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
554; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
555; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
556; SKIP-CACHE-INV-NEXT:    s_endpgm
557;
558; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_release_store:
559; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
560; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
561; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
562; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
563; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
564; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
565; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
566;
567; GFX90A-TGSPLIT-LABEL: local_singlethread_release_store:
568; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
569; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
570; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
571; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
572; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
573; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
574; GFX90A-TGSPLIT-NEXT:    s_endpgm
575    i32 %in, i32 addrspace(3)* %out) {
576entry:
577  store atomic i32 %in, i32 addrspace(3)* %out syncscope("singlethread") release, align 4
578  ret void
579}
580
581define amdgpu_kernel void @local_singlethread_seq_cst_store(
582; GFX6-LABEL: local_singlethread_seq_cst_store:
583; GFX6:       ; %bb.0: ; %entry
584; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x1
585; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x0
586; GFX6-NEXT:    s_mov_b32 m0, -1
587; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
588; GFX6-NEXT:    v_mov_b32_e32 v0, s0
589; GFX6-NEXT:    v_mov_b32_e32 v1, s1
590; GFX6-NEXT:    ds_write_b32 v0, v1
591; GFX6-NEXT:    s_endpgm
592;
593; GFX7-LABEL: local_singlethread_seq_cst_store:
594; GFX7:       ; %bb.0: ; %entry
595; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
596; GFX7-NEXT:    s_mov_b32 m0, -1
597; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
598; GFX7-NEXT:    v_mov_b32_e32 v0, s1
599; GFX7-NEXT:    v_mov_b32_e32 v1, s0
600; GFX7-NEXT:    ds_write_b32 v0, v1
601; GFX7-NEXT:    s_endpgm
602;
603; GFX10-WGP-LABEL: local_singlethread_seq_cst_store:
604; GFX10-WGP:       ; %bb.0: ; %entry
605; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
606; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
607; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
608; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
609; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
610; GFX10-WGP-NEXT:    s_endpgm
611;
612; GFX10-CU-LABEL: local_singlethread_seq_cst_store:
613; GFX10-CU:       ; %bb.0: ; %entry
614; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
615; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
616; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
617; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
618; GFX10-CU-NEXT:    ds_write_b32 v0, v1
619; GFX10-CU-NEXT:    s_endpgm
620;
621; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_store:
622; SKIP-CACHE-INV:       ; %bb.0: ; %entry
623; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
624; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
625; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
626; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
627; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
628; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
629; SKIP-CACHE-INV-NEXT:    s_endpgm
630;
631; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_store:
632; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
633; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
634; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
635; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
636; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
637; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
638; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
639;
640; GFX90A-TGSPLIT-LABEL: local_singlethread_seq_cst_store:
641; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
642; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
643; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
644; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
645; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
646; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
647; GFX90A-TGSPLIT-NEXT:    s_endpgm
648    i32 %in, i32 addrspace(3)* %out) {
649entry:
650  store atomic i32 %in, i32 addrspace(3)* %out syncscope("singlethread") seq_cst, align 4
651  ret void
652}
653
654define amdgpu_kernel void @local_singlethread_monotonic_atomicrmw(
655; GFX6-LABEL: local_singlethread_monotonic_atomicrmw:
656; GFX6:       ; %bb.0: ; %entry
657; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
658; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
659; GFX6-NEXT:    s_mov_b32 m0, -1
660; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
661; GFX6-NEXT:    v_mov_b32_e32 v0, s0
662; GFX6-NEXT:    v_mov_b32_e32 v1, s1
663; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
664; GFX6-NEXT:    s_endpgm
665;
666; GFX7-LABEL: local_singlethread_monotonic_atomicrmw:
667; GFX7:       ; %bb.0: ; %entry
668; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
669; GFX7-NEXT:    s_mov_b32 m0, -1
670; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
671; GFX7-NEXT:    v_mov_b32_e32 v0, s0
672; GFX7-NEXT:    v_mov_b32_e32 v1, s1
673; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
674; GFX7-NEXT:    s_endpgm
675;
676; GFX10-WGP-LABEL: local_singlethread_monotonic_atomicrmw:
677; GFX10-WGP:       ; %bb.0: ; %entry
678; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
679; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
680; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
681; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
682; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
683; GFX10-WGP-NEXT:    s_endpgm
684;
685; GFX10-CU-LABEL: local_singlethread_monotonic_atomicrmw:
686; GFX10-CU:       ; %bb.0: ; %entry
687; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
688; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
689; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
690; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
691; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
692; GFX10-CU-NEXT:    s_endpgm
693;
694; SKIP-CACHE-INV-LABEL: local_singlethread_monotonic_atomicrmw:
695; SKIP-CACHE-INV:       ; %bb.0: ; %entry
696; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
697; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
698; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
699; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
700; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
701; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
702; SKIP-CACHE-INV-NEXT:    s_endpgm
703;
704; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_monotonic_atomicrmw:
705; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
706; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
707; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
708; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
709; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
710; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
711; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
712;
713; GFX90A-TGSPLIT-LABEL: local_singlethread_monotonic_atomicrmw:
714; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
715; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
716; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
717; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
718; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
719; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
720; GFX90A-TGSPLIT-NEXT:    s_endpgm
721    i32 addrspace(3)* %out, i32 %in) {
722entry:
723  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread") monotonic
724  ret void
725}
726
727define amdgpu_kernel void @local_singlethread_acquire_atomicrmw(
728; GFX6-LABEL: local_singlethread_acquire_atomicrmw:
729; GFX6:       ; %bb.0: ; %entry
730; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
731; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
732; GFX6-NEXT:    s_mov_b32 m0, -1
733; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
734; GFX6-NEXT:    v_mov_b32_e32 v0, s0
735; GFX6-NEXT:    v_mov_b32_e32 v1, s1
736; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
737; GFX6-NEXT:    s_endpgm
738;
739; GFX7-LABEL: local_singlethread_acquire_atomicrmw:
740; GFX7:       ; %bb.0: ; %entry
741; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
742; GFX7-NEXT:    s_mov_b32 m0, -1
743; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
744; GFX7-NEXT:    v_mov_b32_e32 v0, s0
745; GFX7-NEXT:    v_mov_b32_e32 v1, s1
746; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
747; GFX7-NEXT:    s_endpgm
748;
749; GFX10-WGP-LABEL: local_singlethread_acquire_atomicrmw:
750; GFX10-WGP:       ; %bb.0: ; %entry
751; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
752; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
753; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
754; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
755; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
756; GFX10-WGP-NEXT:    s_endpgm
757;
758; GFX10-CU-LABEL: local_singlethread_acquire_atomicrmw:
759; GFX10-CU:       ; %bb.0: ; %entry
760; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
761; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
762; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
763; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
764; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
765; GFX10-CU-NEXT:    s_endpgm
766;
767; SKIP-CACHE-INV-LABEL: local_singlethread_acquire_atomicrmw:
768; SKIP-CACHE-INV:       ; %bb.0: ; %entry
769; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
770; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
771; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
772; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
773; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
774; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
775; SKIP-CACHE-INV-NEXT:    s_endpgm
776;
777; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acquire_atomicrmw:
778; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
779; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
780; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
781; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
782; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
783; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
784; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
785;
786; GFX90A-TGSPLIT-LABEL: local_singlethread_acquire_atomicrmw:
787; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
788; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
789; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
790; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
791; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
792; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
793; GFX90A-TGSPLIT-NEXT:    s_endpgm
794    i32 addrspace(3)* %out, i32 %in) {
795entry:
796  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread") acquire
797  ret void
798}
799
800define amdgpu_kernel void @local_singlethread_release_atomicrmw(
801; GFX6-LABEL: local_singlethread_release_atomicrmw:
802; GFX6:       ; %bb.0: ; %entry
803; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
804; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
805; GFX6-NEXT:    s_mov_b32 m0, -1
806; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
807; GFX6-NEXT:    v_mov_b32_e32 v0, s0
808; GFX6-NEXT:    v_mov_b32_e32 v1, s1
809; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
810; GFX6-NEXT:    s_endpgm
811;
812; GFX7-LABEL: local_singlethread_release_atomicrmw:
813; GFX7:       ; %bb.0: ; %entry
814; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
815; GFX7-NEXT:    s_mov_b32 m0, -1
816; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
817; GFX7-NEXT:    v_mov_b32_e32 v0, s0
818; GFX7-NEXT:    v_mov_b32_e32 v1, s1
819; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
820; GFX7-NEXT:    s_endpgm
821;
822; GFX10-WGP-LABEL: local_singlethread_release_atomicrmw:
823; GFX10-WGP:       ; %bb.0: ; %entry
824; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
825; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
826; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
827; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
828; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
829; GFX10-WGP-NEXT:    s_endpgm
830;
831; GFX10-CU-LABEL: local_singlethread_release_atomicrmw:
832; GFX10-CU:       ; %bb.0: ; %entry
833; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
834; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
835; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
836; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
837; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
838; GFX10-CU-NEXT:    s_endpgm
839;
840; SKIP-CACHE-INV-LABEL: local_singlethread_release_atomicrmw:
841; SKIP-CACHE-INV:       ; %bb.0: ; %entry
842; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
843; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
844; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
845; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
846; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
847; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
848; SKIP-CACHE-INV-NEXT:    s_endpgm
849;
850; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_release_atomicrmw:
851; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
852; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
853; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
854; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
855; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
856; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
857; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
858;
859; GFX90A-TGSPLIT-LABEL: local_singlethread_release_atomicrmw:
860; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
861; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
862; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
863; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
864; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
865; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
866; GFX90A-TGSPLIT-NEXT:    s_endpgm
867    i32 addrspace(3)* %out, i32 %in) {
868entry:
869  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread") release
870  ret void
871}
872
873define amdgpu_kernel void @local_singlethread_acq_rel_atomicrmw(
874; GFX6-LABEL: local_singlethread_acq_rel_atomicrmw:
875; GFX6:       ; %bb.0: ; %entry
876; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
877; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
878; GFX6-NEXT:    s_mov_b32 m0, -1
879; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
880; GFX6-NEXT:    v_mov_b32_e32 v0, s0
881; GFX6-NEXT:    v_mov_b32_e32 v1, s1
882; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
883; GFX6-NEXT:    s_endpgm
884;
885; GFX7-LABEL: local_singlethread_acq_rel_atomicrmw:
886; GFX7:       ; %bb.0: ; %entry
887; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
888; GFX7-NEXT:    s_mov_b32 m0, -1
889; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
890; GFX7-NEXT:    v_mov_b32_e32 v0, s0
891; GFX7-NEXT:    v_mov_b32_e32 v1, s1
892; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
893; GFX7-NEXT:    s_endpgm
894;
895; GFX10-WGP-LABEL: local_singlethread_acq_rel_atomicrmw:
896; GFX10-WGP:       ; %bb.0: ; %entry
897; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
898; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
899; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
900; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
901; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
902; GFX10-WGP-NEXT:    s_endpgm
903;
904; GFX10-CU-LABEL: local_singlethread_acq_rel_atomicrmw:
905; GFX10-CU:       ; %bb.0: ; %entry
906; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
907; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
908; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
909; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
910; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
911; GFX10-CU-NEXT:    s_endpgm
912;
913; SKIP-CACHE-INV-LABEL: local_singlethread_acq_rel_atomicrmw:
914; SKIP-CACHE-INV:       ; %bb.0: ; %entry
915; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
916; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
917; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
918; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
919; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
920; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
921; SKIP-CACHE-INV-NEXT:    s_endpgm
922;
923; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_atomicrmw:
924; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
925; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
926; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
927; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
928; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
929; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
930; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
931;
932; GFX90A-TGSPLIT-LABEL: local_singlethread_acq_rel_atomicrmw:
933; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
934; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
935; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
936; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
937; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
938; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
939; GFX90A-TGSPLIT-NEXT:    s_endpgm
940    i32 addrspace(3)* %out, i32 %in) {
941entry:
942  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread") acq_rel
943  ret void
944}
945
946define amdgpu_kernel void @local_singlethread_seq_cst_atomicrmw(
947; GFX6-LABEL: local_singlethread_seq_cst_atomicrmw:
948; GFX6:       ; %bb.0: ; %entry
949; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
950; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
951; GFX6-NEXT:    s_mov_b32 m0, -1
952; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
953; GFX6-NEXT:    v_mov_b32_e32 v0, s0
954; GFX6-NEXT:    v_mov_b32_e32 v1, s1
955; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
956; GFX6-NEXT:    s_endpgm
957;
958; GFX7-LABEL: local_singlethread_seq_cst_atomicrmw:
959; GFX7:       ; %bb.0: ; %entry
960; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
961; GFX7-NEXT:    s_mov_b32 m0, -1
962; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
963; GFX7-NEXT:    v_mov_b32_e32 v0, s0
964; GFX7-NEXT:    v_mov_b32_e32 v1, s1
965; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
966; GFX7-NEXT:    s_endpgm
967;
968; GFX10-WGP-LABEL: local_singlethread_seq_cst_atomicrmw:
969; GFX10-WGP:       ; %bb.0: ; %entry
970; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
971; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
972; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
973; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
974; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
975; GFX10-WGP-NEXT:    s_endpgm
976;
977; GFX10-CU-LABEL: local_singlethread_seq_cst_atomicrmw:
978; GFX10-CU:       ; %bb.0: ; %entry
979; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
980; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
981; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
982; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
983; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
984; GFX10-CU-NEXT:    s_endpgm
985;
986; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_atomicrmw:
987; SKIP-CACHE-INV:       ; %bb.0: ; %entry
988; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
989; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
990; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
991; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
992; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
993; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
994; SKIP-CACHE-INV-NEXT:    s_endpgm
995;
996; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_atomicrmw:
997; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
998; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
999; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1000; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1001; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1002; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1003; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1004;
1005; GFX90A-TGSPLIT-LABEL: local_singlethread_seq_cst_atomicrmw:
1006; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1007; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1008; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1009; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1010; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1011; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
1012; GFX90A-TGSPLIT-NEXT:    s_endpgm
1013    i32 addrspace(3)* %out, i32 %in) {
1014entry:
1015  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread") seq_cst
1016  ret void
1017}
1018
1019define amdgpu_kernel void @local_singlethread_acquire_ret_atomicrmw(
1020; GFX6-LABEL: local_singlethread_acquire_ret_atomicrmw:
1021; GFX6:       ; %bb.0: ; %entry
1022; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
1023; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
1024; GFX6-NEXT:    s_mov_b32 m0, -1
1025; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1026; GFX6-NEXT:    v_mov_b32_e32 v0, s0
1027; GFX6-NEXT:    v_mov_b32_e32 v1, s1
1028; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1029; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1030; GFX6-NEXT:    ds_write_b32 v0, v1
1031; GFX6-NEXT:    s_endpgm
1032;
1033; GFX7-LABEL: local_singlethread_acquire_ret_atomicrmw:
1034; GFX7:       ; %bb.0: ; %entry
1035; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1036; GFX7-NEXT:    s_mov_b32 m0, -1
1037; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1038; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1039; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1040; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1041; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1042; GFX7-NEXT:    ds_write_b32 v0, v1
1043; GFX7-NEXT:    s_endpgm
1044;
1045; GFX10-WGP-LABEL: local_singlethread_acquire_ret_atomicrmw:
1046; GFX10-WGP:       ; %bb.0: ; %entry
1047; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1048; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1049; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1050; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1051; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1052; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1053; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
1054; GFX10-WGP-NEXT:    s_endpgm
1055;
1056; GFX10-CU-LABEL: local_singlethread_acquire_ret_atomicrmw:
1057; GFX10-CU:       ; %bb.0: ; %entry
1058; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1059; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1060; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1061; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1062; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1063; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1064; GFX10-CU-NEXT:    ds_write_b32 v0, v1
1065; GFX10-CU-NEXT:    s_endpgm
1066;
1067; SKIP-CACHE-INV-LABEL: local_singlethread_acquire_ret_atomicrmw:
1068; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1069; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1070; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
1071; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1072; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1073; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
1074; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1075; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1076; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
1077; SKIP-CACHE-INV-NEXT:    s_endpgm
1078;
1079; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acquire_ret_atomicrmw:
1080; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1081; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1082; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1083; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1084; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1085; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1086; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1087; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
1088; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1089;
1090; GFX90A-TGSPLIT-LABEL: local_singlethread_acquire_ret_atomicrmw:
1091; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1092; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1093; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1094; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1095; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1096; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1097; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1098; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
1099; GFX90A-TGSPLIT-NEXT:    s_endpgm
1100    i32 addrspace(3)* %out, i32 %in) {
1101entry:
1102  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread") acquire
1103  store i32 %val, i32 addrspace(3)* %out, align 4
1104  ret void
1105}
1106
1107define amdgpu_kernel void @local_singlethread_acq_rel_ret_atomicrmw(
1108; GFX6-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
1109; GFX6:       ; %bb.0: ; %entry
1110; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
1111; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
1112; GFX6-NEXT:    s_mov_b32 m0, -1
1113; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1114; GFX6-NEXT:    v_mov_b32_e32 v0, s0
1115; GFX6-NEXT:    v_mov_b32_e32 v1, s1
1116; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1117; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1118; GFX6-NEXT:    ds_write_b32 v0, v1
1119; GFX6-NEXT:    s_endpgm
1120;
1121; GFX7-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
1122; GFX7:       ; %bb.0: ; %entry
1123; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1124; GFX7-NEXT:    s_mov_b32 m0, -1
1125; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1126; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1127; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1128; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1129; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1130; GFX7-NEXT:    ds_write_b32 v0, v1
1131; GFX7-NEXT:    s_endpgm
1132;
1133; GFX10-WGP-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
1134; GFX10-WGP:       ; %bb.0: ; %entry
1135; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1136; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1137; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1138; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1139; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1140; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1141; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
1142; GFX10-WGP-NEXT:    s_endpgm
1143;
1144; GFX10-CU-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
1145; GFX10-CU:       ; %bb.0: ; %entry
1146; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1147; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1148; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1149; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1150; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1151; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1152; GFX10-CU-NEXT:    ds_write_b32 v0, v1
1153; GFX10-CU-NEXT:    s_endpgm
1154;
1155; SKIP-CACHE-INV-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
1156; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1157; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1158; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
1159; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1160; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1161; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
1162; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1163; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1164; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
1165; SKIP-CACHE-INV-NEXT:    s_endpgm
1166;
1167; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
1168; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1169; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1170; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1171; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1172; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1173; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1174; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1175; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
1176; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1177;
1178; GFX90A-TGSPLIT-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
1179; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1180; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1181; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1182; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1183; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1184; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1185; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1186; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
1187; GFX90A-TGSPLIT-NEXT:    s_endpgm
1188    i32 addrspace(3)* %out, i32 %in) {
1189entry:
1190  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread") acq_rel
1191  store i32 %val, i32 addrspace(3)* %out, align 4
1192  ret void
1193}
1194
1195define amdgpu_kernel void @local_singlethread_seq_cst_ret_atomicrmw(
1196; GFX6-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
1197; GFX6:       ; %bb.0: ; %entry
1198; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
1199; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
1200; GFX6-NEXT:    s_mov_b32 m0, -1
1201; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1202; GFX6-NEXT:    v_mov_b32_e32 v0, s0
1203; GFX6-NEXT:    v_mov_b32_e32 v1, s1
1204; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1205; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1206; GFX6-NEXT:    ds_write_b32 v0, v1
1207; GFX6-NEXT:    s_endpgm
1208;
1209; GFX7-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
1210; GFX7:       ; %bb.0: ; %entry
1211; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1212; GFX7-NEXT:    s_mov_b32 m0, -1
1213; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1214; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1215; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1216; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1217; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1218; GFX7-NEXT:    ds_write_b32 v0, v1
1219; GFX7-NEXT:    s_endpgm
1220;
1221; GFX10-WGP-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
1222; GFX10-WGP:       ; %bb.0: ; %entry
1223; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1224; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1225; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1226; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1227; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1228; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1229; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
1230; GFX10-WGP-NEXT:    s_endpgm
1231;
1232; GFX10-CU-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
1233; GFX10-CU:       ; %bb.0: ; %entry
1234; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1235; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1236; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1237; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1238; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1239; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1240; GFX10-CU-NEXT:    ds_write_b32 v0, v1
1241; GFX10-CU-NEXT:    s_endpgm
1242;
1243; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
1244; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1245; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1246; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
1247; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1248; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1249; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
1250; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1251; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1252; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
1253; SKIP-CACHE-INV-NEXT:    s_endpgm
1254;
1255; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
1256; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1257; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1258; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1259; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1260; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1261; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1262; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1263; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
1264; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1265;
1266; GFX90A-TGSPLIT-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
1267; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1268; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1269; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1270; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1271; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
1272; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
1273; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1274; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
1275; GFX90A-TGSPLIT-NEXT:    s_endpgm
1276    i32 addrspace(3)* %out, i32 %in) {
1277entry:
1278  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread") seq_cst
1279  store i32 %val, i32 addrspace(3)* %out, align 4
1280  ret void
1281}
1282
1283define amdgpu_kernel void @local_singlethread_monotonic_monotonic_cmpxchg(
1284; GFX6-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
1285; GFX6:       ; %bb.0: ; %entry
1286; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
1287; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
1288; GFX6-NEXT:    s_mov_b32 m0, -1
1289; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1290; GFX6-NEXT:    v_mov_b32_e32 v0, s2
1291; GFX6-NEXT:    v_mov_b32_e32 v1, s1
1292; GFX6-NEXT:    v_mov_b32_e32 v2, s0
1293; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1294; GFX6-NEXT:    s_endpgm
1295;
1296; GFX7-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
1297; GFX7:       ; %bb.0: ; %entry
1298; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1299; GFX7-NEXT:    s_mov_b32 m0, -1
1300; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1301; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1302; GFX7-NEXT:    v_mov_b32_e32 v1, s2
1303; GFX7-NEXT:    v_mov_b32_e32 v2, s1
1304; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1305; GFX7-NEXT:    s_endpgm
1306;
1307; GFX10-WGP-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
1308; GFX10-WGP:       ; %bb.0: ; %entry
1309; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1310; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1311; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1312; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1313; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
1314; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1315; GFX10-WGP-NEXT:    s_endpgm
1316;
1317; GFX10-CU-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
1318; GFX10-CU:       ; %bb.0: ; %entry
1319; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1320; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1321; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1322; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1323; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
1324; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1325; GFX10-CU-NEXT:    s_endpgm
1326;
1327; SKIP-CACHE-INV-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
1328; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1329; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
1330; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
1331; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1332; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1333; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
1334; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
1335; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1336; SKIP-CACHE-INV-NEXT:    s_endpgm
1337;
1338; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
1339; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1340; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1341; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1342; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1343; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1344; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
1345; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1346; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1347;
1348; GFX90A-TGSPLIT-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
1349; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1350; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1351; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1352; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1353; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1354; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
1355; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1356; GFX90A-TGSPLIT-NEXT:    s_endpgm
1357    i32 addrspace(3)* %out, i32 %in, i32 %old) {
1358entry:
1359  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
1360  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic monotonic
1361  ret void
1362}
1363
1364define amdgpu_kernel void @local_singlethread_acquire_monotonic_cmpxchg(
1365; GFX6-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
1366; GFX6:       ; %bb.0: ; %entry
1367; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
1368; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
1369; GFX6-NEXT:    s_mov_b32 m0, -1
1370; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1371; GFX6-NEXT:    v_mov_b32_e32 v0, s2
1372; GFX6-NEXT:    v_mov_b32_e32 v1, s1
1373; GFX6-NEXT:    v_mov_b32_e32 v2, s0
1374; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1375; GFX6-NEXT:    s_endpgm
1376;
1377; GFX7-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
1378; GFX7:       ; %bb.0: ; %entry
1379; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1380; GFX7-NEXT:    s_mov_b32 m0, -1
1381; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1382; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1383; GFX7-NEXT:    v_mov_b32_e32 v1, s2
1384; GFX7-NEXT:    v_mov_b32_e32 v2, s1
1385; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1386; GFX7-NEXT:    s_endpgm
1387;
1388; GFX10-WGP-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
1389; GFX10-WGP:       ; %bb.0: ; %entry
1390; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1391; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1392; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1393; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1394; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
1395; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1396; GFX10-WGP-NEXT:    s_endpgm
1397;
1398; GFX10-CU-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
1399; GFX10-CU:       ; %bb.0: ; %entry
1400; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1401; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1402; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1403; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1404; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
1405; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1406; GFX10-CU-NEXT:    s_endpgm
1407;
1408; SKIP-CACHE-INV-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
1409; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1410; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
1411; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
1412; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1413; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1414; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
1415; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
1416; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1417; SKIP-CACHE-INV-NEXT:    s_endpgm
1418;
1419; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
1420; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1421; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1422; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1423; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1424; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1425; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
1426; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1427; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1428;
1429; GFX90A-TGSPLIT-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
1430; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1431; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1432; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1433; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1434; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1435; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
1436; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1437; GFX90A-TGSPLIT-NEXT:    s_endpgm
1438    i32 addrspace(3)* %out, i32 %in, i32 %old) {
1439entry:
1440  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
1441  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire monotonic
1442  ret void
1443}
1444
1445define amdgpu_kernel void @local_singlethread_release_monotonic_cmpxchg(
1446; GFX6-LABEL: local_singlethread_release_monotonic_cmpxchg:
1447; GFX6:       ; %bb.0: ; %entry
1448; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
1449; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
1450; GFX6-NEXT:    s_mov_b32 m0, -1
1451; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1452; GFX6-NEXT:    v_mov_b32_e32 v0, s2
1453; GFX6-NEXT:    v_mov_b32_e32 v1, s1
1454; GFX6-NEXT:    v_mov_b32_e32 v2, s0
1455; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1456; GFX6-NEXT:    s_endpgm
1457;
1458; GFX7-LABEL: local_singlethread_release_monotonic_cmpxchg:
1459; GFX7:       ; %bb.0: ; %entry
1460; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1461; GFX7-NEXT:    s_mov_b32 m0, -1
1462; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1463; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1464; GFX7-NEXT:    v_mov_b32_e32 v1, s2
1465; GFX7-NEXT:    v_mov_b32_e32 v2, s1
1466; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1467; GFX7-NEXT:    s_endpgm
1468;
1469; GFX10-WGP-LABEL: local_singlethread_release_monotonic_cmpxchg:
1470; GFX10-WGP:       ; %bb.0: ; %entry
1471; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1472; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1473; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1474; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1475; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
1476; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1477; GFX10-WGP-NEXT:    s_endpgm
1478;
1479; GFX10-CU-LABEL: local_singlethread_release_monotonic_cmpxchg:
1480; GFX10-CU:       ; %bb.0: ; %entry
1481; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1482; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1483; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1484; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1485; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
1486; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1487; GFX10-CU-NEXT:    s_endpgm
1488;
1489; SKIP-CACHE-INV-LABEL: local_singlethread_release_monotonic_cmpxchg:
1490; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1491; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
1492; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
1493; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1494; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1495; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
1496; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
1497; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1498; SKIP-CACHE-INV-NEXT:    s_endpgm
1499;
1500; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_release_monotonic_cmpxchg:
1501; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1502; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1503; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1504; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1505; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1506; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
1507; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1508; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1509;
1510; GFX90A-TGSPLIT-LABEL: local_singlethread_release_monotonic_cmpxchg:
1511; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1512; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1513; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1514; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1515; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1516; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
1517; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1518; GFX90A-TGSPLIT-NEXT:    s_endpgm
1519    i32 addrspace(3)* %out, i32 %in, i32 %old) {
1520entry:
1521  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
1522  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") release monotonic
1523  ret void
1524}
1525
1526define amdgpu_kernel void @local_singlethread_acq_rel_monotonic_cmpxchg(
1527; GFX6-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
1528; GFX6:       ; %bb.0: ; %entry
1529; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
1530; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
1531; GFX6-NEXT:    s_mov_b32 m0, -1
1532; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1533; GFX6-NEXT:    v_mov_b32_e32 v0, s2
1534; GFX6-NEXT:    v_mov_b32_e32 v1, s1
1535; GFX6-NEXT:    v_mov_b32_e32 v2, s0
1536; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1537; GFX6-NEXT:    s_endpgm
1538;
1539; GFX7-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
1540; GFX7:       ; %bb.0: ; %entry
1541; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1542; GFX7-NEXT:    s_mov_b32 m0, -1
1543; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1544; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1545; GFX7-NEXT:    v_mov_b32_e32 v1, s2
1546; GFX7-NEXT:    v_mov_b32_e32 v2, s1
1547; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1548; GFX7-NEXT:    s_endpgm
1549;
1550; GFX10-WGP-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
1551; GFX10-WGP:       ; %bb.0: ; %entry
1552; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1553; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1554; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1555; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1556; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
1557; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1558; GFX10-WGP-NEXT:    s_endpgm
1559;
1560; GFX10-CU-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
1561; GFX10-CU:       ; %bb.0: ; %entry
1562; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1563; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1564; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1565; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1566; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
1567; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1568; GFX10-CU-NEXT:    s_endpgm
1569;
1570; SKIP-CACHE-INV-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
1571; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1572; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
1573; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
1574; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1575; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1576; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
1577; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
1578; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1579; SKIP-CACHE-INV-NEXT:    s_endpgm
1580;
1581; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
1582; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1583; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1584; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1585; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1586; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1587; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
1588; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1589; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1590;
1591; GFX90A-TGSPLIT-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
1592; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1593; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1594; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1595; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1596; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1597; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
1598; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1599; GFX90A-TGSPLIT-NEXT:    s_endpgm
1600    i32 addrspace(3)* %out, i32 %in, i32 %old) {
1601entry:
1602  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
1603  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel monotonic
1604  ret void
1605}
1606
1607define amdgpu_kernel void @local_singlethread_seq_cst_monotonic_cmpxchg(
1608; GFX6-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
1609; GFX6:       ; %bb.0: ; %entry
1610; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
1611; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
1612; GFX6-NEXT:    s_mov_b32 m0, -1
1613; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1614; GFX6-NEXT:    v_mov_b32_e32 v0, s2
1615; GFX6-NEXT:    v_mov_b32_e32 v1, s1
1616; GFX6-NEXT:    v_mov_b32_e32 v2, s0
1617; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1618; GFX6-NEXT:    s_endpgm
1619;
1620; GFX7-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
1621; GFX7:       ; %bb.0: ; %entry
1622; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1623; GFX7-NEXT:    s_mov_b32 m0, -1
1624; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1625; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1626; GFX7-NEXT:    v_mov_b32_e32 v1, s2
1627; GFX7-NEXT:    v_mov_b32_e32 v2, s1
1628; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1629; GFX7-NEXT:    s_endpgm
1630;
1631; GFX10-WGP-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
1632; GFX10-WGP:       ; %bb.0: ; %entry
1633; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1634; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1635; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1636; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1637; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
1638; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1639; GFX10-WGP-NEXT:    s_endpgm
1640;
1641; GFX10-CU-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
1642; GFX10-CU:       ; %bb.0: ; %entry
1643; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1644; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1645; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1646; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1647; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
1648; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1649; GFX10-CU-NEXT:    s_endpgm
1650;
1651; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
1652; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1653; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
1654; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
1655; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1656; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1657; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
1658; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
1659; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1660; SKIP-CACHE-INV-NEXT:    s_endpgm
1661;
1662; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
1663; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1664; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1665; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1666; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1667; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1668; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
1669; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1670; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1671;
1672; GFX90A-TGSPLIT-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
1673; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1674; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1675; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1676; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1677; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1678; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
1679; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1680; GFX90A-TGSPLIT-NEXT:    s_endpgm
1681    i32 addrspace(3)* %out, i32 %in, i32 %old) {
1682entry:
1683  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
1684  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst monotonic
1685  ret void
1686}
1687
1688define amdgpu_kernel void @local_singlethread_monotonic_acquire_cmpxchg(
1689; GFX6-LABEL: local_singlethread_monotonic_acquire_cmpxchg:
1690; GFX6:       ; %bb.0: ; %entry
1691; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
1692; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
1693; GFX6-NEXT:    s_mov_b32 m0, -1
1694; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1695; GFX6-NEXT:    v_mov_b32_e32 v0, s2
1696; GFX6-NEXT:    v_mov_b32_e32 v1, s1
1697; GFX6-NEXT:    v_mov_b32_e32 v2, s0
1698; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1699; GFX6-NEXT:    s_endpgm
1700;
1701; GFX7-LABEL: local_singlethread_monotonic_acquire_cmpxchg:
1702; GFX7:       ; %bb.0: ; %entry
1703; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1704; GFX7-NEXT:    s_mov_b32 m0, -1
1705; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1706; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1707; GFX7-NEXT:    v_mov_b32_e32 v1, s2
1708; GFX7-NEXT:    v_mov_b32_e32 v2, s1
1709; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1710; GFX7-NEXT:    s_endpgm
1711;
1712; GFX10-WGP-LABEL: local_singlethread_monotonic_acquire_cmpxchg:
1713; GFX10-WGP:       ; %bb.0: ; %entry
1714; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1715; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1716; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1717; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1718; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
1719; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1720; GFX10-WGP-NEXT:    s_endpgm
1721;
1722; GFX10-CU-LABEL: local_singlethread_monotonic_acquire_cmpxchg:
1723; GFX10-CU:       ; %bb.0: ; %entry
1724; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1725; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1726; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1727; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1728; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
1729; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1730; GFX10-CU-NEXT:    s_endpgm
1731;
1732; SKIP-CACHE-INV-LABEL: local_singlethread_monotonic_acquire_cmpxchg:
1733; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1734; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
1735; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
1736; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1737; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1738; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
1739; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
1740; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1741; SKIP-CACHE-INV-NEXT:    s_endpgm
1742;
1743; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_monotonic_acquire_cmpxchg:
1744; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1745; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1746; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1747; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1748; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1749; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
1750; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1751; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1752;
1753; GFX90A-TGSPLIT-LABEL: local_singlethread_monotonic_acquire_cmpxchg:
1754; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1755; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1756; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1757; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1758; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1759; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
1760; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1761; GFX90A-TGSPLIT-NEXT:    s_endpgm
1762    i32 addrspace(3)* %out, i32 %in, i32 %old) {
1763entry:
1764  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
1765  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic acquire
1766  ret void
1767}
1768
1769define amdgpu_kernel void @local_singlethread_acquire_acquire_cmpxchg(
1770; GFX6-LABEL: local_singlethread_acquire_acquire_cmpxchg:
1771; GFX6:       ; %bb.0: ; %entry
1772; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
1773; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
1774; GFX6-NEXT:    s_mov_b32 m0, -1
1775; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1776; GFX6-NEXT:    v_mov_b32_e32 v0, s2
1777; GFX6-NEXT:    v_mov_b32_e32 v1, s1
1778; GFX6-NEXT:    v_mov_b32_e32 v2, s0
1779; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1780; GFX6-NEXT:    s_endpgm
1781;
1782; GFX7-LABEL: local_singlethread_acquire_acquire_cmpxchg:
1783; GFX7:       ; %bb.0: ; %entry
1784; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1785; GFX7-NEXT:    s_mov_b32 m0, -1
1786; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1787; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1788; GFX7-NEXT:    v_mov_b32_e32 v1, s2
1789; GFX7-NEXT:    v_mov_b32_e32 v2, s1
1790; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1791; GFX7-NEXT:    s_endpgm
1792;
1793; GFX10-WGP-LABEL: local_singlethread_acquire_acquire_cmpxchg:
1794; GFX10-WGP:       ; %bb.0: ; %entry
1795; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1796; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1797; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1798; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1799; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
1800; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1801; GFX10-WGP-NEXT:    s_endpgm
1802;
1803; GFX10-CU-LABEL: local_singlethread_acquire_acquire_cmpxchg:
1804; GFX10-CU:       ; %bb.0: ; %entry
1805; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1806; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1807; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1808; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1809; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
1810; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1811; GFX10-CU-NEXT:    s_endpgm
1812;
1813; SKIP-CACHE-INV-LABEL: local_singlethread_acquire_acquire_cmpxchg:
1814; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1815; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
1816; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
1817; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1818; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1819; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
1820; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
1821; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1822; SKIP-CACHE-INV-NEXT:    s_endpgm
1823;
1824; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acquire_acquire_cmpxchg:
1825; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1826; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1827; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1828; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1829; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1830; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
1831; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1832; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1833;
1834; GFX90A-TGSPLIT-LABEL: local_singlethread_acquire_acquire_cmpxchg:
1835; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1836; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1837; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1838; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1839; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1840; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
1841; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1842; GFX90A-TGSPLIT-NEXT:    s_endpgm
1843    i32 addrspace(3)* %out, i32 %in, i32 %old) {
1844entry:
1845  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
1846  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire acquire
1847  ret void
1848}
1849
1850define amdgpu_kernel void @local_singlethread_release_acquire_cmpxchg(
1851; GFX6-LABEL: local_singlethread_release_acquire_cmpxchg:
1852; GFX6:       ; %bb.0: ; %entry
1853; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
1854; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
1855; GFX6-NEXT:    s_mov_b32 m0, -1
1856; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1857; GFX6-NEXT:    v_mov_b32_e32 v0, s2
1858; GFX6-NEXT:    v_mov_b32_e32 v1, s1
1859; GFX6-NEXT:    v_mov_b32_e32 v2, s0
1860; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1861; GFX6-NEXT:    s_endpgm
1862;
1863; GFX7-LABEL: local_singlethread_release_acquire_cmpxchg:
1864; GFX7:       ; %bb.0: ; %entry
1865; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1866; GFX7-NEXT:    s_mov_b32 m0, -1
1867; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1868; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1869; GFX7-NEXT:    v_mov_b32_e32 v1, s2
1870; GFX7-NEXT:    v_mov_b32_e32 v2, s1
1871; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1872; GFX7-NEXT:    s_endpgm
1873;
1874; GFX10-WGP-LABEL: local_singlethread_release_acquire_cmpxchg:
1875; GFX10-WGP:       ; %bb.0: ; %entry
1876; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1877; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1878; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1879; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1880; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
1881; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1882; GFX10-WGP-NEXT:    s_endpgm
1883;
1884; GFX10-CU-LABEL: local_singlethread_release_acquire_cmpxchg:
1885; GFX10-CU:       ; %bb.0: ; %entry
1886; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1887; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1888; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1889; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1890; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
1891; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1892; GFX10-CU-NEXT:    s_endpgm
1893;
1894; SKIP-CACHE-INV-LABEL: local_singlethread_release_acquire_cmpxchg:
1895; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1896; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
1897; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
1898; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1899; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1900; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
1901; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
1902; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1903; SKIP-CACHE-INV-NEXT:    s_endpgm
1904;
1905; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_release_acquire_cmpxchg:
1906; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1907; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1908; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1909; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1910; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1911; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
1912; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1913; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1914;
1915; GFX90A-TGSPLIT-LABEL: local_singlethread_release_acquire_cmpxchg:
1916; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1917; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1918; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1919; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1920; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1921; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
1922; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1923; GFX90A-TGSPLIT-NEXT:    s_endpgm
1924    i32 addrspace(3)* %out, i32 %in, i32 %old) {
1925entry:
1926  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
1927  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") release acquire
1928  ret void
1929}
1930
1931define amdgpu_kernel void @local_singlethread_acq_rel_acquire_cmpxchg(
1932; GFX6-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
1933; GFX6:       ; %bb.0: ; %entry
1934; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
1935; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
1936; GFX6-NEXT:    s_mov_b32 m0, -1
1937; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1938; GFX6-NEXT:    v_mov_b32_e32 v0, s2
1939; GFX6-NEXT:    v_mov_b32_e32 v1, s1
1940; GFX6-NEXT:    v_mov_b32_e32 v2, s0
1941; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1942; GFX6-NEXT:    s_endpgm
1943;
1944; GFX7-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
1945; GFX7:       ; %bb.0: ; %entry
1946; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1947; GFX7-NEXT:    s_mov_b32 m0, -1
1948; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1949; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1950; GFX7-NEXT:    v_mov_b32_e32 v1, s2
1951; GFX7-NEXT:    v_mov_b32_e32 v2, s1
1952; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1953; GFX7-NEXT:    s_endpgm
1954;
1955; GFX10-WGP-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
1956; GFX10-WGP:       ; %bb.0: ; %entry
1957; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1958; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1959; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1960; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1961; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
1962; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1963; GFX10-WGP-NEXT:    s_endpgm
1964;
1965; GFX10-CU-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
1966; GFX10-CU:       ; %bb.0: ; %entry
1967; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1968; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1969; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1970; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1971; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
1972; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1973; GFX10-CU-NEXT:    s_endpgm
1974;
1975; SKIP-CACHE-INV-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
1976; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1977; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
1978; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
1979; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1980; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1981; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
1982; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
1983; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1984; SKIP-CACHE-INV-NEXT:    s_endpgm
1985;
1986; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
1987; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1988; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1989; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1990; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
1991; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1992; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
1993; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
1994; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1995;
1996; GFX90A-TGSPLIT-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
1997; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1998; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
1999; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2000; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2001; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2002; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2003; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2004; GFX90A-TGSPLIT-NEXT:    s_endpgm
2005    i32 addrspace(3)* %out, i32 %in, i32 %old) {
2006entry:
2007  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
2008  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel acquire
2009  ret void
2010}
2011
2012define amdgpu_kernel void @local_singlethread_seq_cst_acquire_cmpxchg(
2013; GFX6-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
2014; GFX6:       ; %bb.0: ; %entry
2015; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
2016; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
2017; GFX6-NEXT:    s_mov_b32 m0, -1
2018; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2019; GFX6-NEXT:    v_mov_b32_e32 v0, s2
2020; GFX6-NEXT:    v_mov_b32_e32 v1, s1
2021; GFX6-NEXT:    v_mov_b32_e32 v2, s0
2022; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2023; GFX6-NEXT:    s_endpgm
2024;
2025; GFX7-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
2026; GFX7:       ; %bb.0: ; %entry
2027; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2028; GFX7-NEXT:    s_mov_b32 m0, -1
2029; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2030; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2031; GFX7-NEXT:    v_mov_b32_e32 v1, s2
2032; GFX7-NEXT:    v_mov_b32_e32 v2, s1
2033; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2034; GFX7-NEXT:    s_endpgm
2035;
2036; GFX10-WGP-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
2037; GFX10-WGP:       ; %bb.0: ; %entry
2038; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2039; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2040; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2041; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
2042; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
2043; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2044; GFX10-WGP-NEXT:    s_endpgm
2045;
2046; GFX10-CU-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
2047; GFX10-CU:       ; %bb.0: ; %entry
2048; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2049; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2050; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2051; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
2052; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
2053; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2054; GFX10-CU-NEXT:    s_endpgm
2055;
2056; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
2057; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2058; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2059; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
2060; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2061; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2062; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
2063; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
2064; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2065; SKIP-CACHE-INV-NEXT:    s_endpgm
2066;
2067; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
2068; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2069; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2070; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2071; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2072; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2073; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2074; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2075; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2076;
2077; GFX90A-TGSPLIT-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
2078; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2079; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2080; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2081; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2082; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2083; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2084; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2085; GFX90A-TGSPLIT-NEXT:    s_endpgm
2086    i32 addrspace(3)* %out, i32 %in, i32 %old) {
2087entry:
2088  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
2089  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst acquire
2090  ret void
2091}
2092
2093define amdgpu_kernel void @local_singlethread_monotonic_seq_cst_cmpxchg(
2094; GFX6-LABEL: local_singlethread_monotonic_seq_cst_cmpxchg:
2095; GFX6:       ; %bb.0: ; %entry
2096; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
2097; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
2098; GFX6-NEXT:    s_mov_b32 m0, -1
2099; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2100; GFX6-NEXT:    v_mov_b32_e32 v0, s2
2101; GFX6-NEXT:    v_mov_b32_e32 v1, s1
2102; GFX6-NEXT:    v_mov_b32_e32 v2, s0
2103; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2104; GFX6-NEXT:    s_endpgm
2105;
2106; GFX7-LABEL: local_singlethread_monotonic_seq_cst_cmpxchg:
2107; GFX7:       ; %bb.0: ; %entry
2108; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2109; GFX7-NEXT:    s_mov_b32 m0, -1
2110; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2111; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2112; GFX7-NEXT:    v_mov_b32_e32 v1, s2
2113; GFX7-NEXT:    v_mov_b32_e32 v2, s1
2114; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2115; GFX7-NEXT:    s_endpgm
2116;
2117; GFX10-WGP-LABEL: local_singlethread_monotonic_seq_cst_cmpxchg:
2118; GFX10-WGP:       ; %bb.0: ; %entry
2119; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2120; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2121; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2122; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
2123; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
2124; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2125; GFX10-WGP-NEXT:    s_endpgm
2126;
2127; GFX10-CU-LABEL: local_singlethread_monotonic_seq_cst_cmpxchg:
2128; GFX10-CU:       ; %bb.0: ; %entry
2129; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2130; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2131; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2132; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
2133; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
2134; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2135; GFX10-CU-NEXT:    s_endpgm
2136;
2137; SKIP-CACHE-INV-LABEL: local_singlethread_monotonic_seq_cst_cmpxchg:
2138; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2139; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2140; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
2141; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2142; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2143; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
2144; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
2145; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2146; SKIP-CACHE-INV-NEXT:    s_endpgm
2147;
2148; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_monotonic_seq_cst_cmpxchg:
2149; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2150; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2151; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2152; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2153; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2154; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2155; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2156; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2157;
2158; GFX90A-TGSPLIT-LABEL: local_singlethread_monotonic_seq_cst_cmpxchg:
2159; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2160; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2161; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2162; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2163; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2164; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2165; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2166; GFX90A-TGSPLIT-NEXT:    s_endpgm
2167    i32 addrspace(3)* %out, i32 %in, i32 %old) {
2168entry:
2169  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
2170  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic seq_cst
2171  ret void
2172}
2173
2174define amdgpu_kernel void @local_singlethread_acquire_seq_cst_cmpxchg(
2175; GFX6-LABEL: local_singlethread_acquire_seq_cst_cmpxchg:
2176; GFX6:       ; %bb.0: ; %entry
2177; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
2178; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
2179; GFX6-NEXT:    s_mov_b32 m0, -1
2180; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2181; GFX6-NEXT:    v_mov_b32_e32 v0, s2
2182; GFX6-NEXT:    v_mov_b32_e32 v1, s1
2183; GFX6-NEXT:    v_mov_b32_e32 v2, s0
2184; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2185; GFX6-NEXT:    s_endpgm
2186;
2187; GFX7-LABEL: local_singlethread_acquire_seq_cst_cmpxchg:
2188; GFX7:       ; %bb.0: ; %entry
2189; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2190; GFX7-NEXT:    s_mov_b32 m0, -1
2191; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2192; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2193; GFX7-NEXT:    v_mov_b32_e32 v1, s2
2194; GFX7-NEXT:    v_mov_b32_e32 v2, s1
2195; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2196; GFX7-NEXT:    s_endpgm
2197;
2198; GFX10-WGP-LABEL: local_singlethread_acquire_seq_cst_cmpxchg:
2199; GFX10-WGP:       ; %bb.0: ; %entry
2200; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2201; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2202; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2203; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
2204; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
2205; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2206; GFX10-WGP-NEXT:    s_endpgm
2207;
2208; GFX10-CU-LABEL: local_singlethread_acquire_seq_cst_cmpxchg:
2209; GFX10-CU:       ; %bb.0: ; %entry
2210; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2211; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2212; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2213; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
2214; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
2215; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2216; GFX10-CU-NEXT:    s_endpgm
2217;
2218; SKIP-CACHE-INV-LABEL: local_singlethread_acquire_seq_cst_cmpxchg:
2219; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2220; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2221; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
2222; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2223; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2224; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
2225; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
2226; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2227; SKIP-CACHE-INV-NEXT:    s_endpgm
2228;
2229; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acquire_seq_cst_cmpxchg:
2230; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2231; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2232; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2233; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2234; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2235; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2236; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2237; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2238;
2239; GFX90A-TGSPLIT-LABEL: local_singlethread_acquire_seq_cst_cmpxchg:
2240; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2241; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2242; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2243; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2244; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2245; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2246; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2247; GFX90A-TGSPLIT-NEXT:    s_endpgm
2248    i32 addrspace(3)* %out, i32 %in, i32 %old) {
2249entry:
2250  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
2251  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire seq_cst
2252  ret void
2253}
2254
2255define amdgpu_kernel void @local_singlethread_release_seq_cst_cmpxchg(
2256; GFX6-LABEL: local_singlethread_release_seq_cst_cmpxchg:
2257; GFX6:       ; %bb.0: ; %entry
2258; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
2259; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
2260; GFX6-NEXT:    s_mov_b32 m0, -1
2261; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2262; GFX6-NEXT:    v_mov_b32_e32 v0, s2
2263; GFX6-NEXT:    v_mov_b32_e32 v1, s1
2264; GFX6-NEXT:    v_mov_b32_e32 v2, s0
2265; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2266; GFX6-NEXT:    s_endpgm
2267;
2268; GFX7-LABEL: local_singlethread_release_seq_cst_cmpxchg:
2269; GFX7:       ; %bb.0: ; %entry
2270; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2271; GFX7-NEXT:    s_mov_b32 m0, -1
2272; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2273; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2274; GFX7-NEXT:    v_mov_b32_e32 v1, s2
2275; GFX7-NEXT:    v_mov_b32_e32 v2, s1
2276; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2277; GFX7-NEXT:    s_endpgm
2278;
2279; GFX10-WGP-LABEL: local_singlethread_release_seq_cst_cmpxchg:
2280; GFX10-WGP:       ; %bb.0: ; %entry
2281; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2282; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2283; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2284; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
2285; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
2286; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2287; GFX10-WGP-NEXT:    s_endpgm
2288;
2289; GFX10-CU-LABEL: local_singlethread_release_seq_cst_cmpxchg:
2290; GFX10-CU:       ; %bb.0: ; %entry
2291; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2292; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2293; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2294; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
2295; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
2296; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2297; GFX10-CU-NEXT:    s_endpgm
2298;
2299; SKIP-CACHE-INV-LABEL: local_singlethread_release_seq_cst_cmpxchg:
2300; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2301; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2302; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
2303; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2304; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2305; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
2306; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
2307; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2308; SKIP-CACHE-INV-NEXT:    s_endpgm
2309;
2310; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_release_seq_cst_cmpxchg:
2311; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2312; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2313; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2314; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2315; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2316; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2317; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2318; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2319;
2320; GFX90A-TGSPLIT-LABEL: local_singlethread_release_seq_cst_cmpxchg:
2321; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2322; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2323; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2324; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2325; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2326; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2327; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2328; GFX90A-TGSPLIT-NEXT:    s_endpgm
2329    i32 addrspace(3)* %out, i32 %in, i32 %old) {
2330entry:
2331  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
2332  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") release seq_cst
2333  ret void
2334}
2335
2336define amdgpu_kernel void @local_singlethread_acq_rel_seq_cst_cmpxchg(
2337; GFX6-LABEL: local_singlethread_acq_rel_seq_cst_cmpxchg:
2338; GFX6:       ; %bb.0: ; %entry
2339; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
2340; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
2341; GFX6-NEXT:    s_mov_b32 m0, -1
2342; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2343; GFX6-NEXT:    v_mov_b32_e32 v0, s2
2344; GFX6-NEXT:    v_mov_b32_e32 v1, s1
2345; GFX6-NEXT:    v_mov_b32_e32 v2, s0
2346; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2347; GFX6-NEXT:    s_endpgm
2348;
2349; GFX7-LABEL: local_singlethread_acq_rel_seq_cst_cmpxchg:
2350; GFX7:       ; %bb.0: ; %entry
2351; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2352; GFX7-NEXT:    s_mov_b32 m0, -1
2353; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2354; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2355; GFX7-NEXT:    v_mov_b32_e32 v1, s2
2356; GFX7-NEXT:    v_mov_b32_e32 v2, s1
2357; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2358; GFX7-NEXT:    s_endpgm
2359;
2360; GFX10-WGP-LABEL: local_singlethread_acq_rel_seq_cst_cmpxchg:
2361; GFX10-WGP:       ; %bb.0: ; %entry
2362; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2363; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2364; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2365; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
2366; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
2367; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2368; GFX10-WGP-NEXT:    s_endpgm
2369;
2370; GFX10-CU-LABEL: local_singlethread_acq_rel_seq_cst_cmpxchg:
2371; GFX10-CU:       ; %bb.0: ; %entry
2372; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2373; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2374; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2375; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
2376; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
2377; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2378; GFX10-CU-NEXT:    s_endpgm
2379;
2380; SKIP-CACHE-INV-LABEL: local_singlethread_acq_rel_seq_cst_cmpxchg:
2381; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2382; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2383; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
2384; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2385; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2386; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
2387; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
2388; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2389; SKIP-CACHE-INV-NEXT:    s_endpgm
2390;
2391; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_seq_cst_cmpxchg:
2392; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2393; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2394; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2395; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2396; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2397; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2398; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2399; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2400;
2401; GFX90A-TGSPLIT-LABEL: local_singlethread_acq_rel_seq_cst_cmpxchg:
2402; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2403; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2404; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2405; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2406; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2407; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2408; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2409; GFX90A-TGSPLIT-NEXT:    s_endpgm
2410    i32 addrspace(3)* %out, i32 %in, i32 %old) {
2411entry:
2412  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
2413  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel seq_cst
2414  ret void
2415}
2416
2417define amdgpu_kernel void @local_singlethread_seq_cst_seq_cst_cmpxchg(
2418; GFX6-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
2419; GFX6:       ; %bb.0: ; %entry
2420; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
2421; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
2422; GFX6-NEXT:    s_mov_b32 m0, -1
2423; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2424; GFX6-NEXT:    v_mov_b32_e32 v0, s2
2425; GFX6-NEXT:    v_mov_b32_e32 v1, s1
2426; GFX6-NEXT:    v_mov_b32_e32 v2, s0
2427; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2428; GFX6-NEXT:    s_endpgm
2429;
2430; GFX7-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
2431; GFX7:       ; %bb.0: ; %entry
2432; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2433; GFX7-NEXT:    s_mov_b32 m0, -1
2434; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2435; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2436; GFX7-NEXT:    v_mov_b32_e32 v1, s2
2437; GFX7-NEXT:    v_mov_b32_e32 v2, s1
2438; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2439; GFX7-NEXT:    s_endpgm
2440;
2441; GFX10-WGP-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
2442; GFX10-WGP:       ; %bb.0: ; %entry
2443; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2444; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2445; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2446; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
2447; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
2448; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2449; GFX10-WGP-NEXT:    s_endpgm
2450;
2451; GFX10-CU-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
2452; GFX10-CU:       ; %bb.0: ; %entry
2453; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2454; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2455; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2456; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
2457; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
2458; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2459; GFX10-CU-NEXT:    s_endpgm
2460;
2461; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
2462; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2463; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2464; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
2465; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2466; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2467; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
2468; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
2469; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2470; SKIP-CACHE-INV-NEXT:    s_endpgm
2471;
2472; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
2473; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2474; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2475; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2476; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2477; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2478; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2479; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2480; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2481;
2482; GFX90A-TGSPLIT-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
2483; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2484; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2485; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2486; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2487; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2488; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2489; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
2490; GFX90A-TGSPLIT-NEXT:    s_endpgm
2491    i32 addrspace(3)* %out, i32 %in, i32 %old) {
2492entry:
2493  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
2494  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst seq_cst
2495  ret void
2496}
2497
2498define amdgpu_kernel void @local_singlethread_monotonic_monotonic_ret_cmpxchg(
2499; GFX6-LABEL: local_singlethread_monotonic_monotonic_ret_cmpxchg:
2500; GFX6:       ; %bb.0: ; %entry
2501; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
2502; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
2503; GFX6-NEXT:    s_mov_b32 m0, -1
2504; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2505; GFX6-NEXT:    v_mov_b32_e32 v0, s2
2506; GFX6-NEXT:    v_mov_b32_e32 v1, s1
2507; GFX6-NEXT:    v_mov_b32_e32 v2, s0
2508; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
2509; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2510; GFX6-NEXT:    ds_write_b32 v0, v1
2511; GFX6-NEXT:    s_endpgm
2512;
2513; GFX7-LABEL: local_singlethread_monotonic_monotonic_ret_cmpxchg:
2514; GFX7:       ; %bb.0: ; %entry
2515; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2516; GFX7-NEXT:    s_mov_b32 m0, -1
2517; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2518; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2519; GFX7-NEXT:    v_mov_b32_e32 v1, s2
2520; GFX7-NEXT:    v_mov_b32_e32 v2, s1
2521; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
2522; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2523; GFX7-NEXT:    ds_write_b32 v0, v1
2524; GFX7-NEXT:    s_endpgm
2525;
2526; GFX10-WGP-LABEL: local_singlethread_monotonic_monotonic_ret_cmpxchg:
2527; GFX10-WGP:       ; %bb.0: ; %entry
2528; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2529; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2530; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2531; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
2532; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
2533; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
2534; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2535; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
2536; GFX10-WGP-NEXT:    s_endpgm
2537;
2538; GFX10-CU-LABEL: local_singlethread_monotonic_monotonic_ret_cmpxchg:
2539; GFX10-CU:       ; %bb.0: ; %entry
2540; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2541; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2542; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2543; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
2544; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
2545; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
2546; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2547; GFX10-CU-NEXT:    ds_write_b32 v0, v1
2548; GFX10-CU-NEXT:    s_endpgm
2549;
2550; SKIP-CACHE-INV-LABEL: local_singlethread_monotonic_monotonic_ret_cmpxchg:
2551; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2552; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2553; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
2554; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2555; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2556; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
2557; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
2558; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
2559; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2560; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
2561; SKIP-CACHE-INV-NEXT:    s_endpgm
2562;
2563; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_monotonic_monotonic_ret_cmpxchg:
2564; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2565; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2566; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2567; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2568; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2569; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2570; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
2571; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2572; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
2573; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2574;
2575; GFX90A-TGSPLIT-LABEL: local_singlethread_monotonic_monotonic_ret_cmpxchg:
2576; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2577; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2578; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2579; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2580; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2581; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2582; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
2583; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2584; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
2585; GFX90A-TGSPLIT-NEXT:    s_endpgm
2586    i32 addrspace(3)* %out, i32 %in, i32 %old) {
2587entry:
2588  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
2589  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic monotonic
2590  %val0 = extractvalue { i32, i1 } %val, 0
2591  store i32 %val0, i32 addrspace(3)* %out, align 4
2592  ret void
2593}
2594
2595define amdgpu_kernel void @local_singlethread_acquire_monotonic_ret_cmpxchg(
2596; GFX6-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
2597; GFX6:       ; %bb.0: ; %entry
2598; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
2599; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
2600; GFX6-NEXT:    s_mov_b32 m0, -1
2601; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2602; GFX6-NEXT:    v_mov_b32_e32 v0, s2
2603; GFX6-NEXT:    v_mov_b32_e32 v1, s1
2604; GFX6-NEXT:    v_mov_b32_e32 v2, s0
2605; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
2606; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2607; GFX6-NEXT:    ds_write_b32 v0, v1
2608; GFX6-NEXT:    s_endpgm
2609;
2610; GFX7-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
2611; GFX7:       ; %bb.0: ; %entry
2612; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2613; GFX7-NEXT:    s_mov_b32 m0, -1
2614; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2615; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2616; GFX7-NEXT:    v_mov_b32_e32 v1, s2
2617; GFX7-NEXT:    v_mov_b32_e32 v2, s1
2618; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
2619; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2620; GFX7-NEXT:    ds_write_b32 v0, v1
2621; GFX7-NEXT:    s_endpgm
2622;
2623; GFX10-WGP-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
2624; GFX10-WGP:       ; %bb.0: ; %entry
2625; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2626; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2627; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2628; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
2629; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
2630; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
2631; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2632; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
2633; GFX10-WGP-NEXT:    s_endpgm
2634;
2635; GFX10-CU-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
2636; GFX10-CU:       ; %bb.0: ; %entry
2637; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2638; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2639; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2640; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
2641; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
2642; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
2643; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2644; GFX10-CU-NEXT:    ds_write_b32 v0, v1
2645; GFX10-CU-NEXT:    s_endpgm
2646;
2647; SKIP-CACHE-INV-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
2648; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2649; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2650; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
2651; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2652; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2653; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
2654; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
2655; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
2656; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2657; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
2658; SKIP-CACHE-INV-NEXT:    s_endpgm
2659;
2660; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
2661; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2662; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2663; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2664; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2665; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2666; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2667; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
2668; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2669; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
2670; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2671;
2672; GFX90A-TGSPLIT-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
2673; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2674; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2675; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2676; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2677; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2678; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2679; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
2680; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2681; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
2682; GFX90A-TGSPLIT-NEXT:    s_endpgm
2683    i32 addrspace(3)* %out, i32 %in, i32 %old) {
2684entry:
2685  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
2686  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire monotonic
2687  %val0 = extractvalue { i32, i1 } %val, 0
2688  store i32 %val0, i32 addrspace(3)* %out, align 4
2689  ret void
2690}
2691
2692define amdgpu_kernel void @local_singlethread_release_monotonic_ret_cmpxchg(
2693; GFX6-LABEL: local_singlethread_release_monotonic_ret_cmpxchg:
2694; GFX6:       ; %bb.0: ; %entry
2695; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
2696; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
2697; GFX6-NEXT:    s_mov_b32 m0, -1
2698; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2699; GFX6-NEXT:    v_mov_b32_e32 v0, s2
2700; GFX6-NEXT:    v_mov_b32_e32 v1, s1
2701; GFX6-NEXT:    v_mov_b32_e32 v2, s0
2702; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
2703; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2704; GFX6-NEXT:    ds_write_b32 v0, v1
2705; GFX6-NEXT:    s_endpgm
2706;
2707; GFX7-LABEL: local_singlethread_release_monotonic_ret_cmpxchg:
2708; GFX7:       ; %bb.0: ; %entry
2709; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2710; GFX7-NEXT:    s_mov_b32 m0, -1
2711; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2712; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2713; GFX7-NEXT:    v_mov_b32_e32 v1, s2
2714; GFX7-NEXT:    v_mov_b32_e32 v2, s1
2715; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
2716; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2717; GFX7-NEXT:    ds_write_b32 v0, v1
2718; GFX7-NEXT:    s_endpgm
2719;
2720; GFX10-WGP-LABEL: local_singlethread_release_monotonic_ret_cmpxchg:
2721; GFX10-WGP:       ; %bb.0: ; %entry
2722; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2723; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2724; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2725; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
2726; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
2727; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
2728; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2729; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
2730; GFX10-WGP-NEXT:    s_endpgm
2731;
2732; GFX10-CU-LABEL: local_singlethread_release_monotonic_ret_cmpxchg:
2733; GFX10-CU:       ; %bb.0: ; %entry
2734; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2735; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2736; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2737; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
2738; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
2739; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
2740; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2741; GFX10-CU-NEXT:    ds_write_b32 v0, v1
2742; GFX10-CU-NEXT:    s_endpgm
2743;
2744; SKIP-CACHE-INV-LABEL: local_singlethread_release_monotonic_ret_cmpxchg:
2745; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2746; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2747; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
2748; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2749; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2750; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
2751; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
2752; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
2753; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2754; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
2755; SKIP-CACHE-INV-NEXT:    s_endpgm
2756;
2757; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_release_monotonic_ret_cmpxchg:
2758; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2759; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2760; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2761; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2762; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2763; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2764; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
2765; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2766; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
2767; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2768;
2769; GFX90A-TGSPLIT-LABEL: local_singlethread_release_monotonic_ret_cmpxchg:
2770; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2771; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2772; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2773; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2774; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2775; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2776; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
2777; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2778; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
2779; GFX90A-TGSPLIT-NEXT:    s_endpgm
2780    i32 addrspace(3)* %out, i32 %in, i32 %old) {
2781entry:
2782  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
2783  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") release monotonic
2784  %val0 = extractvalue { i32, i1 } %val, 0
2785  store i32 %val0, i32 addrspace(3)* %out, align 4
2786  ret void
2787}
2788
2789define amdgpu_kernel void @local_singlethread_acq_rel_monotonic_ret_cmpxchg(
2790; GFX6-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
2791; GFX6:       ; %bb.0: ; %entry
2792; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
2793; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
2794; GFX6-NEXT:    s_mov_b32 m0, -1
2795; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2796; GFX6-NEXT:    v_mov_b32_e32 v0, s2
2797; GFX6-NEXT:    v_mov_b32_e32 v1, s1
2798; GFX6-NEXT:    v_mov_b32_e32 v2, s0
2799; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
2800; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2801; GFX6-NEXT:    ds_write_b32 v0, v1
2802; GFX6-NEXT:    s_endpgm
2803;
2804; GFX7-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
2805; GFX7:       ; %bb.0: ; %entry
2806; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2807; GFX7-NEXT:    s_mov_b32 m0, -1
2808; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2809; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2810; GFX7-NEXT:    v_mov_b32_e32 v1, s2
2811; GFX7-NEXT:    v_mov_b32_e32 v2, s1
2812; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
2813; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2814; GFX7-NEXT:    ds_write_b32 v0, v1
2815; GFX7-NEXT:    s_endpgm
2816;
2817; GFX10-WGP-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
2818; GFX10-WGP:       ; %bb.0: ; %entry
2819; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2820; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2821; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2822; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
2823; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
2824; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
2825; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2826; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
2827; GFX10-WGP-NEXT:    s_endpgm
2828;
2829; GFX10-CU-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
2830; GFX10-CU:       ; %bb.0: ; %entry
2831; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2832; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2833; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2834; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
2835; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
2836; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
2837; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2838; GFX10-CU-NEXT:    ds_write_b32 v0, v1
2839; GFX10-CU-NEXT:    s_endpgm
2840;
2841; SKIP-CACHE-INV-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
2842; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2843; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2844; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
2845; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2846; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2847; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
2848; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
2849; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
2850; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2851; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
2852; SKIP-CACHE-INV-NEXT:    s_endpgm
2853;
2854; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
2855; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2856; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2857; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2858; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2859; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2860; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2861; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
2862; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2863; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
2864; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2865;
2866; GFX90A-TGSPLIT-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
2867; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2868; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2869; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2870; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2871; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2872; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2873; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
2874; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2875; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
2876; GFX90A-TGSPLIT-NEXT:    s_endpgm
2877    i32 addrspace(3)* %out, i32 %in, i32 %old) {
2878entry:
2879  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
2880  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel monotonic
2881  %val0 = extractvalue { i32, i1 } %val, 0
2882  store i32 %val0, i32 addrspace(3)* %out, align 4
2883  ret void
2884}
2885
2886define amdgpu_kernel void @local_singlethread_seq_cst_monotonic_ret_cmpxchg(
2887; GFX6-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
2888; GFX6:       ; %bb.0: ; %entry
2889; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
2890; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
2891; GFX6-NEXT:    s_mov_b32 m0, -1
2892; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2893; GFX6-NEXT:    v_mov_b32_e32 v0, s2
2894; GFX6-NEXT:    v_mov_b32_e32 v1, s1
2895; GFX6-NEXT:    v_mov_b32_e32 v2, s0
2896; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
2897; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2898; GFX6-NEXT:    ds_write_b32 v0, v1
2899; GFX6-NEXT:    s_endpgm
2900;
2901; GFX7-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
2902; GFX7:       ; %bb.0: ; %entry
2903; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2904; GFX7-NEXT:    s_mov_b32 m0, -1
2905; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2906; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2907; GFX7-NEXT:    v_mov_b32_e32 v1, s2
2908; GFX7-NEXT:    v_mov_b32_e32 v2, s1
2909; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
2910; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2911; GFX7-NEXT:    ds_write_b32 v0, v1
2912; GFX7-NEXT:    s_endpgm
2913;
2914; GFX10-WGP-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
2915; GFX10-WGP:       ; %bb.0: ; %entry
2916; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2917; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2918; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2919; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
2920; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
2921; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
2922; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2923; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
2924; GFX10-WGP-NEXT:    s_endpgm
2925;
2926; GFX10-CU-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
2927; GFX10-CU:       ; %bb.0: ; %entry
2928; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2929; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2930; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2931; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
2932; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
2933; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
2934; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2935; GFX10-CU-NEXT:    ds_write_b32 v0, v1
2936; GFX10-CU-NEXT:    s_endpgm
2937;
2938; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
2939; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2940; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
2941; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
2942; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2943; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2944; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
2945; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
2946; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
2947; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2948; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
2949; SKIP-CACHE-INV-NEXT:    s_endpgm
2950;
2951; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
2952; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2953; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2954; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2955; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2956; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2957; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2958; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
2959; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2960; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
2961; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2962;
2963; GFX90A-TGSPLIT-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
2964; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2965; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
2966; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2967; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
2968; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2969; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
2970; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
2971; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2972; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
2973; GFX90A-TGSPLIT-NEXT:    s_endpgm
2974    i32 addrspace(3)* %out, i32 %in, i32 %old) {
2975entry:
2976  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
2977  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst monotonic
2978  %val0 = extractvalue { i32, i1 } %val, 0
2979  store i32 %val0, i32 addrspace(3)* %out, align 4
2980  ret void
2981}
2982
2983define amdgpu_kernel void @local_singlethread_monotonic_acquire_ret_cmpxchg(
2984; GFX6-LABEL: local_singlethread_monotonic_acquire_ret_cmpxchg:
2985; GFX6:       ; %bb.0: ; %entry
2986; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
2987; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
2988; GFX6-NEXT:    s_mov_b32 m0, -1
2989; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2990; GFX6-NEXT:    v_mov_b32_e32 v0, s2
2991; GFX6-NEXT:    v_mov_b32_e32 v1, s1
2992; GFX6-NEXT:    v_mov_b32_e32 v2, s0
2993; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
2994; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2995; GFX6-NEXT:    ds_write_b32 v0, v1
2996; GFX6-NEXT:    s_endpgm
2997;
2998; GFX7-LABEL: local_singlethread_monotonic_acquire_ret_cmpxchg:
2999; GFX7:       ; %bb.0: ; %entry
3000; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3001; GFX7-NEXT:    s_mov_b32 m0, -1
3002; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3003; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3004; GFX7-NEXT:    v_mov_b32_e32 v1, s2
3005; GFX7-NEXT:    v_mov_b32_e32 v2, s1
3006; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3007; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3008; GFX7-NEXT:    ds_write_b32 v0, v1
3009; GFX7-NEXT:    s_endpgm
3010;
3011; GFX10-WGP-LABEL: local_singlethread_monotonic_acquire_ret_cmpxchg:
3012; GFX10-WGP:       ; %bb.0: ; %entry
3013; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3014; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3015; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3016; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
3017; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
3018; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3019; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3020; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
3021; GFX10-WGP-NEXT:    s_endpgm
3022;
3023; GFX10-CU-LABEL: local_singlethread_monotonic_acquire_ret_cmpxchg:
3024; GFX10-CU:       ; %bb.0: ; %entry
3025; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3026; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3027; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3028; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
3029; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
3030; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3031; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3032; GFX10-CU-NEXT:    ds_write_b32 v0, v1
3033; GFX10-CU-NEXT:    s_endpgm
3034;
3035; SKIP-CACHE-INV-LABEL: local_singlethread_monotonic_acquire_ret_cmpxchg:
3036; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3037; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3038; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
3039; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3040; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3041; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
3042; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
3043; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3044; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3045; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
3046; SKIP-CACHE-INV-NEXT:    s_endpgm
3047;
3048; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_monotonic_acquire_ret_cmpxchg:
3049; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3050; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3051; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3052; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3053; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3054; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3055; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3056; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3057; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
3058; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3059;
3060; GFX90A-TGSPLIT-LABEL: local_singlethread_monotonic_acquire_ret_cmpxchg:
3061; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3062; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3063; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3064; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3065; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3066; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3067; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3068; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3069; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
3070; GFX90A-TGSPLIT-NEXT:    s_endpgm
3071    i32 addrspace(3)* %out, i32 %in, i32 %old) {
3072entry:
3073  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
3074  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic acquire
3075  %val0 = extractvalue { i32, i1 } %val, 0
3076  store i32 %val0, i32 addrspace(3)* %out, align 4
3077  ret void
3078}
3079
3080define amdgpu_kernel void @local_singlethread_acquire_acquire_ret_cmpxchg(
3081; GFX6-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
3082; GFX6:       ; %bb.0: ; %entry
3083; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
3084; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
3085; GFX6-NEXT:    s_mov_b32 m0, -1
3086; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3087; GFX6-NEXT:    v_mov_b32_e32 v0, s2
3088; GFX6-NEXT:    v_mov_b32_e32 v1, s1
3089; GFX6-NEXT:    v_mov_b32_e32 v2, s0
3090; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3091; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3092; GFX6-NEXT:    ds_write_b32 v0, v1
3093; GFX6-NEXT:    s_endpgm
3094;
3095; GFX7-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
3096; GFX7:       ; %bb.0: ; %entry
3097; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3098; GFX7-NEXT:    s_mov_b32 m0, -1
3099; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3100; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3101; GFX7-NEXT:    v_mov_b32_e32 v1, s2
3102; GFX7-NEXT:    v_mov_b32_e32 v2, s1
3103; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3104; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3105; GFX7-NEXT:    ds_write_b32 v0, v1
3106; GFX7-NEXT:    s_endpgm
3107;
3108; GFX10-WGP-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
3109; GFX10-WGP:       ; %bb.0: ; %entry
3110; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3111; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3112; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3113; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
3114; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
3115; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3116; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3117; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
3118; GFX10-WGP-NEXT:    s_endpgm
3119;
3120; GFX10-CU-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
3121; GFX10-CU:       ; %bb.0: ; %entry
3122; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3123; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3124; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3125; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
3126; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
3127; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3128; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3129; GFX10-CU-NEXT:    ds_write_b32 v0, v1
3130; GFX10-CU-NEXT:    s_endpgm
3131;
3132; SKIP-CACHE-INV-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
3133; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3134; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3135; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
3136; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3137; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3138; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
3139; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
3140; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3141; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3142; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
3143; SKIP-CACHE-INV-NEXT:    s_endpgm
3144;
3145; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
3146; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3147; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3148; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3149; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3150; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3151; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3152; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3153; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3154; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
3155; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3156;
3157; GFX90A-TGSPLIT-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
3158; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3159; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3160; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3161; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3162; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3163; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3164; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3165; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3166; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
3167; GFX90A-TGSPLIT-NEXT:    s_endpgm
3168    i32 addrspace(3)* %out, i32 %in, i32 %old) {
3169entry:
3170  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
3171  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire acquire
3172  %val0 = extractvalue { i32, i1 } %val, 0
3173  store i32 %val0, i32 addrspace(3)* %out, align 4
3174  ret void
3175}
3176
3177define amdgpu_kernel void @local_singlethread_release_acquire_ret_cmpxchg(
3178; GFX6-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
3179; GFX6:       ; %bb.0: ; %entry
3180; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
3181; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
3182; GFX6-NEXT:    s_mov_b32 m0, -1
3183; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3184; GFX6-NEXT:    v_mov_b32_e32 v0, s2
3185; GFX6-NEXT:    v_mov_b32_e32 v1, s1
3186; GFX6-NEXT:    v_mov_b32_e32 v2, s0
3187; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3188; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3189; GFX6-NEXT:    ds_write_b32 v0, v1
3190; GFX6-NEXT:    s_endpgm
3191;
3192; GFX7-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
3193; GFX7:       ; %bb.0: ; %entry
3194; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3195; GFX7-NEXT:    s_mov_b32 m0, -1
3196; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3197; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3198; GFX7-NEXT:    v_mov_b32_e32 v1, s2
3199; GFX7-NEXT:    v_mov_b32_e32 v2, s1
3200; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3201; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3202; GFX7-NEXT:    ds_write_b32 v0, v1
3203; GFX7-NEXT:    s_endpgm
3204;
3205; GFX10-WGP-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
3206; GFX10-WGP:       ; %bb.0: ; %entry
3207; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3208; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3209; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3210; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
3211; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
3212; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3213; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3214; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
3215; GFX10-WGP-NEXT:    s_endpgm
3216;
3217; GFX10-CU-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
3218; GFX10-CU:       ; %bb.0: ; %entry
3219; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3220; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3221; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3222; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
3223; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
3224; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3225; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3226; GFX10-CU-NEXT:    ds_write_b32 v0, v1
3227; GFX10-CU-NEXT:    s_endpgm
3228;
3229; SKIP-CACHE-INV-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
3230; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3231; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3232; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
3233; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3234; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3235; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
3236; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
3237; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3238; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3239; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
3240; SKIP-CACHE-INV-NEXT:    s_endpgm
3241;
3242; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
3243; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3244; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3245; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3246; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3247; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3248; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3249; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3250; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3251; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
3252; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3253;
3254; GFX90A-TGSPLIT-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
3255; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3256; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3257; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3258; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3259; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3260; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3261; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3262; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3263; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
3264; GFX90A-TGSPLIT-NEXT:    s_endpgm
3265    i32 addrspace(3)* %out, i32 %in, i32 %old) {
3266entry:
3267  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
3268  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") release acquire
3269  %val0 = extractvalue { i32, i1 } %val, 0
3270  store i32 %val0, i32 addrspace(3)* %out, align 4
3271  ret void
3272}
3273
3274define amdgpu_kernel void @local_singlethread_acq_rel_acquire_ret_cmpxchg(
3275; GFX6-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
3276; GFX6:       ; %bb.0: ; %entry
3277; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
3278; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
3279; GFX6-NEXT:    s_mov_b32 m0, -1
3280; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3281; GFX6-NEXT:    v_mov_b32_e32 v0, s2
3282; GFX6-NEXT:    v_mov_b32_e32 v1, s1
3283; GFX6-NEXT:    v_mov_b32_e32 v2, s0
3284; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3285; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3286; GFX6-NEXT:    ds_write_b32 v0, v1
3287; GFX6-NEXT:    s_endpgm
3288;
3289; GFX7-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
3290; GFX7:       ; %bb.0: ; %entry
3291; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3292; GFX7-NEXT:    s_mov_b32 m0, -1
3293; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3294; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3295; GFX7-NEXT:    v_mov_b32_e32 v1, s2
3296; GFX7-NEXT:    v_mov_b32_e32 v2, s1
3297; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3298; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3299; GFX7-NEXT:    ds_write_b32 v0, v1
3300; GFX7-NEXT:    s_endpgm
3301;
3302; GFX10-WGP-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
3303; GFX10-WGP:       ; %bb.0: ; %entry
3304; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3305; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3306; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3307; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
3308; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
3309; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3310; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3311; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
3312; GFX10-WGP-NEXT:    s_endpgm
3313;
3314; GFX10-CU-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
3315; GFX10-CU:       ; %bb.0: ; %entry
3316; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3317; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3318; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3319; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
3320; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
3321; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3322; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3323; GFX10-CU-NEXT:    ds_write_b32 v0, v1
3324; GFX10-CU-NEXT:    s_endpgm
3325;
3326; SKIP-CACHE-INV-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
3327; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3328; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3329; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
3330; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3331; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3332; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
3333; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
3334; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3335; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3336; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
3337; SKIP-CACHE-INV-NEXT:    s_endpgm
3338;
3339; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
3340; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3341; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3342; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3343; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3344; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3345; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3346; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3347; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3348; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
3349; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3350;
3351; GFX90A-TGSPLIT-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
3352; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3353; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3354; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3355; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3356; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3357; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3358; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3359; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3360; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
3361; GFX90A-TGSPLIT-NEXT:    s_endpgm
3362    i32 addrspace(3)* %out, i32 %in, i32 %old) {
3363entry:
3364  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
3365  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel acquire
3366  %val0 = extractvalue { i32, i1 } %val, 0
3367  store i32 %val0, i32 addrspace(3)* %out, align 4
3368  ret void
3369}
3370
3371define amdgpu_kernel void @local_singlethread_seq_cst_acquire_ret_cmpxchg(
3372; GFX6-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
3373; GFX6:       ; %bb.0: ; %entry
3374; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
3375; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
3376; GFX6-NEXT:    s_mov_b32 m0, -1
3377; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3378; GFX6-NEXT:    v_mov_b32_e32 v0, s2
3379; GFX6-NEXT:    v_mov_b32_e32 v1, s1
3380; GFX6-NEXT:    v_mov_b32_e32 v2, s0
3381; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3382; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3383; GFX6-NEXT:    ds_write_b32 v0, v1
3384; GFX6-NEXT:    s_endpgm
3385;
3386; GFX7-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
3387; GFX7:       ; %bb.0: ; %entry
3388; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3389; GFX7-NEXT:    s_mov_b32 m0, -1
3390; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3391; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3392; GFX7-NEXT:    v_mov_b32_e32 v1, s2
3393; GFX7-NEXT:    v_mov_b32_e32 v2, s1
3394; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3395; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3396; GFX7-NEXT:    ds_write_b32 v0, v1
3397; GFX7-NEXT:    s_endpgm
3398;
3399; GFX10-WGP-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
3400; GFX10-WGP:       ; %bb.0: ; %entry
3401; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3402; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3403; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3404; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
3405; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
3406; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3407; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3408; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
3409; GFX10-WGP-NEXT:    s_endpgm
3410;
3411; GFX10-CU-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
3412; GFX10-CU:       ; %bb.0: ; %entry
3413; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3414; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3415; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3416; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
3417; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
3418; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3419; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3420; GFX10-CU-NEXT:    ds_write_b32 v0, v1
3421; GFX10-CU-NEXT:    s_endpgm
3422;
3423; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
3424; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3425; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3426; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
3427; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3428; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3429; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
3430; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
3431; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3432; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3433; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
3434; SKIP-CACHE-INV-NEXT:    s_endpgm
3435;
3436; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
3437; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3438; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3439; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3440; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3441; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3442; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3443; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3444; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3445; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
3446; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3447;
3448; GFX90A-TGSPLIT-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
3449; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3450; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3451; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3452; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3453; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3454; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3455; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3456; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3457; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
3458; GFX90A-TGSPLIT-NEXT:    s_endpgm
3459    i32 addrspace(3)* %out, i32 %in, i32 %old) {
3460entry:
3461  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
3462  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst acquire
3463  %val0 = extractvalue { i32, i1 } %val, 0
3464  store i32 %val0, i32 addrspace(3)* %out, align 4
3465  ret void
3466}
3467
3468define amdgpu_kernel void @local_singlethread_monotonic_seq_cst_ret_cmpxchg(
3469; GFX6-LABEL: local_singlethread_monotonic_seq_cst_ret_cmpxchg:
3470; GFX6:       ; %bb.0: ; %entry
3471; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
3472; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
3473; GFX6-NEXT:    s_mov_b32 m0, -1
3474; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3475; GFX6-NEXT:    v_mov_b32_e32 v0, s2
3476; GFX6-NEXT:    v_mov_b32_e32 v1, s1
3477; GFX6-NEXT:    v_mov_b32_e32 v2, s0
3478; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3479; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3480; GFX6-NEXT:    ds_write_b32 v0, v1
3481; GFX6-NEXT:    s_endpgm
3482;
3483; GFX7-LABEL: local_singlethread_monotonic_seq_cst_ret_cmpxchg:
3484; GFX7:       ; %bb.0: ; %entry
3485; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3486; GFX7-NEXT:    s_mov_b32 m0, -1
3487; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3488; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3489; GFX7-NEXT:    v_mov_b32_e32 v1, s2
3490; GFX7-NEXT:    v_mov_b32_e32 v2, s1
3491; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3492; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3493; GFX7-NEXT:    ds_write_b32 v0, v1
3494; GFX7-NEXT:    s_endpgm
3495;
3496; GFX10-WGP-LABEL: local_singlethread_monotonic_seq_cst_ret_cmpxchg:
3497; GFX10-WGP:       ; %bb.0: ; %entry
3498; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3499; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3500; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3501; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
3502; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
3503; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3504; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3505; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
3506; GFX10-WGP-NEXT:    s_endpgm
3507;
3508; GFX10-CU-LABEL: local_singlethread_monotonic_seq_cst_ret_cmpxchg:
3509; GFX10-CU:       ; %bb.0: ; %entry
3510; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3511; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3512; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3513; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
3514; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
3515; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3516; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3517; GFX10-CU-NEXT:    ds_write_b32 v0, v1
3518; GFX10-CU-NEXT:    s_endpgm
3519;
3520; SKIP-CACHE-INV-LABEL: local_singlethread_monotonic_seq_cst_ret_cmpxchg:
3521; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3522; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3523; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
3524; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3525; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3526; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
3527; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
3528; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3529; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3530; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
3531; SKIP-CACHE-INV-NEXT:    s_endpgm
3532;
3533; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_monotonic_seq_cst_ret_cmpxchg:
3534; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3535; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3536; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3537; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3538; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3539; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3540; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3541; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3542; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
3543; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3544;
3545; GFX90A-TGSPLIT-LABEL: local_singlethread_monotonic_seq_cst_ret_cmpxchg:
3546; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3547; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3548; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3549; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3550; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3551; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3552; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3553; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3554; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
3555; GFX90A-TGSPLIT-NEXT:    s_endpgm
3556    i32 addrspace(3)* %out, i32 %in, i32 %old) {
3557entry:
3558  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
3559  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic seq_cst
3560  %val0 = extractvalue { i32, i1 } %val, 0
3561  store i32 %val0, i32 addrspace(3)* %out, align 4
3562  ret void
3563}
3564
3565define amdgpu_kernel void @local_singlethread_acquire_seq_cst_ret_cmpxchg(
3566; GFX6-LABEL: local_singlethread_acquire_seq_cst_ret_cmpxchg:
3567; GFX6:       ; %bb.0: ; %entry
3568; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
3569; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
3570; GFX6-NEXT:    s_mov_b32 m0, -1
3571; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3572; GFX6-NEXT:    v_mov_b32_e32 v0, s2
3573; GFX6-NEXT:    v_mov_b32_e32 v1, s1
3574; GFX6-NEXT:    v_mov_b32_e32 v2, s0
3575; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3576; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3577; GFX6-NEXT:    ds_write_b32 v0, v1
3578; GFX6-NEXT:    s_endpgm
3579;
3580; GFX7-LABEL: local_singlethread_acquire_seq_cst_ret_cmpxchg:
3581; GFX7:       ; %bb.0: ; %entry
3582; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3583; GFX7-NEXT:    s_mov_b32 m0, -1
3584; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3585; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3586; GFX7-NEXT:    v_mov_b32_e32 v1, s2
3587; GFX7-NEXT:    v_mov_b32_e32 v2, s1
3588; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3589; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3590; GFX7-NEXT:    ds_write_b32 v0, v1
3591; GFX7-NEXT:    s_endpgm
3592;
3593; GFX10-WGP-LABEL: local_singlethread_acquire_seq_cst_ret_cmpxchg:
3594; GFX10-WGP:       ; %bb.0: ; %entry
3595; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3596; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3597; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3598; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
3599; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
3600; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3601; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3602; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
3603; GFX10-WGP-NEXT:    s_endpgm
3604;
3605; GFX10-CU-LABEL: local_singlethread_acquire_seq_cst_ret_cmpxchg:
3606; GFX10-CU:       ; %bb.0: ; %entry
3607; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3608; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3609; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3610; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
3611; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
3612; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3613; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3614; GFX10-CU-NEXT:    ds_write_b32 v0, v1
3615; GFX10-CU-NEXT:    s_endpgm
3616;
3617; SKIP-CACHE-INV-LABEL: local_singlethread_acquire_seq_cst_ret_cmpxchg:
3618; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3619; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3620; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
3621; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3622; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3623; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
3624; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
3625; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3626; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3627; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
3628; SKIP-CACHE-INV-NEXT:    s_endpgm
3629;
3630; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acquire_seq_cst_ret_cmpxchg:
3631; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3632; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3633; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3634; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3635; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3636; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3637; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3638; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3639; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
3640; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3641;
3642; GFX90A-TGSPLIT-LABEL: local_singlethread_acquire_seq_cst_ret_cmpxchg:
3643; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3644; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3645; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3646; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3647; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3648; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3649; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3650; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3651; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
3652; GFX90A-TGSPLIT-NEXT:    s_endpgm
3653    i32 addrspace(3)* %out, i32 %in, i32 %old) {
3654entry:
3655  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
3656  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire seq_cst
3657  %val0 = extractvalue { i32, i1 } %val, 0
3658  store i32 %val0, i32 addrspace(3)* %out, align 4
3659  ret void
3660}
3661
3662define amdgpu_kernel void @local_singlethread_release_seq_cst_ret_cmpxchg(
3663; GFX6-LABEL: local_singlethread_release_seq_cst_ret_cmpxchg:
3664; GFX6:       ; %bb.0: ; %entry
3665; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
3666; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
3667; GFX6-NEXT:    s_mov_b32 m0, -1
3668; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3669; GFX6-NEXT:    v_mov_b32_e32 v0, s2
3670; GFX6-NEXT:    v_mov_b32_e32 v1, s1
3671; GFX6-NEXT:    v_mov_b32_e32 v2, s0
3672; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3673; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3674; GFX6-NEXT:    ds_write_b32 v0, v1
3675; GFX6-NEXT:    s_endpgm
3676;
3677; GFX7-LABEL: local_singlethread_release_seq_cst_ret_cmpxchg:
3678; GFX7:       ; %bb.0: ; %entry
3679; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3680; GFX7-NEXT:    s_mov_b32 m0, -1
3681; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3682; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3683; GFX7-NEXT:    v_mov_b32_e32 v1, s2
3684; GFX7-NEXT:    v_mov_b32_e32 v2, s1
3685; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3686; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3687; GFX7-NEXT:    ds_write_b32 v0, v1
3688; GFX7-NEXT:    s_endpgm
3689;
3690; GFX10-WGP-LABEL: local_singlethread_release_seq_cst_ret_cmpxchg:
3691; GFX10-WGP:       ; %bb.0: ; %entry
3692; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3693; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3694; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3695; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
3696; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
3697; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3698; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3699; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
3700; GFX10-WGP-NEXT:    s_endpgm
3701;
3702; GFX10-CU-LABEL: local_singlethread_release_seq_cst_ret_cmpxchg:
3703; GFX10-CU:       ; %bb.0: ; %entry
3704; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3705; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3706; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3707; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
3708; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
3709; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3710; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3711; GFX10-CU-NEXT:    ds_write_b32 v0, v1
3712; GFX10-CU-NEXT:    s_endpgm
3713;
3714; SKIP-CACHE-INV-LABEL: local_singlethread_release_seq_cst_ret_cmpxchg:
3715; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3716; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3717; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
3718; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3719; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3720; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
3721; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
3722; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3723; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3724; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
3725; SKIP-CACHE-INV-NEXT:    s_endpgm
3726;
3727; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_release_seq_cst_ret_cmpxchg:
3728; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3729; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3730; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3731; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3732; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3733; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3734; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3735; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3736; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
3737; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3738;
3739; GFX90A-TGSPLIT-LABEL: local_singlethread_release_seq_cst_ret_cmpxchg:
3740; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3741; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3742; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3743; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3744; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3745; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3746; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3747; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3748; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
3749; GFX90A-TGSPLIT-NEXT:    s_endpgm
3750    i32 addrspace(3)* %out, i32 %in, i32 %old) {
3751entry:
3752  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
3753  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") release seq_cst
3754  %val0 = extractvalue { i32, i1 } %val, 0
3755  store i32 %val0, i32 addrspace(3)* %out, align 4
3756  ret void
3757}
3758
3759define amdgpu_kernel void @local_singlethread_acq_rel_seq_cst_ret_cmpxchg(
3760; GFX6-LABEL: local_singlethread_acq_rel_seq_cst_ret_cmpxchg:
3761; GFX6:       ; %bb.0: ; %entry
3762; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
3763; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
3764; GFX6-NEXT:    s_mov_b32 m0, -1
3765; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3766; GFX6-NEXT:    v_mov_b32_e32 v0, s2
3767; GFX6-NEXT:    v_mov_b32_e32 v1, s1
3768; GFX6-NEXT:    v_mov_b32_e32 v2, s0
3769; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3770; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3771; GFX6-NEXT:    ds_write_b32 v0, v1
3772; GFX6-NEXT:    s_endpgm
3773;
3774; GFX7-LABEL: local_singlethread_acq_rel_seq_cst_ret_cmpxchg:
3775; GFX7:       ; %bb.0: ; %entry
3776; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3777; GFX7-NEXT:    s_mov_b32 m0, -1
3778; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3779; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3780; GFX7-NEXT:    v_mov_b32_e32 v1, s2
3781; GFX7-NEXT:    v_mov_b32_e32 v2, s1
3782; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3783; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3784; GFX7-NEXT:    ds_write_b32 v0, v1
3785; GFX7-NEXT:    s_endpgm
3786;
3787; GFX10-WGP-LABEL: local_singlethread_acq_rel_seq_cst_ret_cmpxchg:
3788; GFX10-WGP:       ; %bb.0: ; %entry
3789; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3790; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3791; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3792; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
3793; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
3794; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3795; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3796; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
3797; GFX10-WGP-NEXT:    s_endpgm
3798;
3799; GFX10-CU-LABEL: local_singlethread_acq_rel_seq_cst_ret_cmpxchg:
3800; GFX10-CU:       ; %bb.0: ; %entry
3801; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3802; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3803; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3804; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
3805; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
3806; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3807; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3808; GFX10-CU-NEXT:    ds_write_b32 v0, v1
3809; GFX10-CU-NEXT:    s_endpgm
3810;
3811; SKIP-CACHE-INV-LABEL: local_singlethread_acq_rel_seq_cst_ret_cmpxchg:
3812; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3813; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3814; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
3815; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3816; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3817; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
3818; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
3819; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3820; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3821; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
3822; SKIP-CACHE-INV-NEXT:    s_endpgm
3823;
3824; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_acq_rel_seq_cst_ret_cmpxchg:
3825; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3826; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3827; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3828; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3829; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3830; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3831; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3832; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3833; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
3834; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3835;
3836; GFX90A-TGSPLIT-LABEL: local_singlethread_acq_rel_seq_cst_ret_cmpxchg:
3837; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3838; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3839; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3840; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3841; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3842; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3843; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3844; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3845; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
3846; GFX90A-TGSPLIT-NEXT:    s_endpgm
3847    i32 addrspace(3)* %out, i32 %in, i32 %old) {
3848entry:
3849  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
3850  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel seq_cst
3851  %val0 = extractvalue { i32, i1 } %val, 0
3852  store i32 %val0, i32 addrspace(3)* %out, align 4
3853  ret void
3854}
3855
3856define amdgpu_kernel void @local_singlethread_seq_cst_seq_cst_ret_cmpxchg(
3857; GFX6-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
3858; GFX6:       ; %bb.0: ; %entry
3859; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
3860; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
3861; GFX6-NEXT:    s_mov_b32 m0, -1
3862; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3863; GFX6-NEXT:    v_mov_b32_e32 v0, s2
3864; GFX6-NEXT:    v_mov_b32_e32 v1, s1
3865; GFX6-NEXT:    v_mov_b32_e32 v2, s0
3866; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3867; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3868; GFX6-NEXT:    ds_write_b32 v0, v1
3869; GFX6-NEXT:    s_endpgm
3870;
3871; GFX7-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
3872; GFX7:       ; %bb.0: ; %entry
3873; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3874; GFX7-NEXT:    s_mov_b32 m0, -1
3875; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3876; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3877; GFX7-NEXT:    v_mov_b32_e32 v1, s2
3878; GFX7-NEXT:    v_mov_b32_e32 v2, s1
3879; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3880; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3881; GFX7-NEXT:    ds_write_b32 v0, v1
3882; GFX7-NEXT:    s_endpgm
3883;
3884; GFX10-WGP-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
3885; GFX10-WGP:       ; %bb.0: ; %entry
3886; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3887; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3888; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3889; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
3890; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
3891; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3892; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3893; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
3894; GFX10-WGP-NEXT:    s_endpgm
3895;
3896; GFX10-CU-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
3897; GFX10-CU:       ; %bb.0: ; %entry
3898; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3899; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3900; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3901; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
3902; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
3903; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3904; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3905; GFX10-CU-NEXT:    ds_write_b32 v0, v1
3906; GFX10-CU-NEXT:    s_endpgm
3907;
3908; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
3909; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3910; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
3911; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
3912; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3913; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3914; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
3915; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
3916; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3917; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3918; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
3919; SKIP-CACHE-INV-NEXT:    s_endpgm
3920;
3921; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
3922; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3923; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3924; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3925; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3926; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3927; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3928; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3929; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3930; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
3931; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3932;
3933; GFX90A-TGSPLIT-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
3934; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3935; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3936; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3937; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3938; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3939; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
3940; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
3941; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3942; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
3943; GFX90A-TGSPLIT-NEXT:    s_endpgm
3944    i32 addrspace(3)* %out, i32 %in, i32 %old) {
3945entry:
3946  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
3947  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst seq_cst
3948  %val0 = extractvalue { i32, i1 } %val, 0
3949  store i32 %val0, i32 addrspace(3)* %out, align 4
3950  ret void
3951}
3952
3953define amdgpu_kernel void @local_singlethread_one_as_unordered_load(
3954; GFX6-LABEL: local_singlethread_one_as_unordered_load:
3955; GFX6:       ; %bb.0: ; %entry
3956; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
3957; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
3958; GFX6-NEXT:    s_mov_b32 m0, -1
3959; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3960; GFX6-NEXT:    v_mov_b32_e32 v0, s0
3961; GFX6-NEXT:    ds_read_b32 v0, v0
3962; GFX6-NEXT:    v_mov_b32_e32 v1, s1
3963; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3964; GFX6-NEXT:    ds_write_b32 v1, v0
3965; GFX6-NEXT:    s_endpgm
3966;
3967; GFX7-LABEL: local_singlethread_one_as_unordered_load:
3968; GFX7:       ; %bb.0: ; %entry
3969; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3970; GFX7-NEXT:    s_mov_b32 m0, -1
3971; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3972; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3973; GFX7-NEXT:    ds_read_b32 v0, v0
3974; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3975; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3976; GFX7-NEXT:    ds_write_b32 v1, v0
3977; GFX7-NEXT:    s_endpgm
3978;
3979; GFX10-WGP-LABEL: local_singlethread_one_as_unordered_load:
3980; GFX10-WGP:       ; %bb.0: ; %entry
3981; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3982; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3983; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3984; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3985; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
3986; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3987; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
3988; GFX10-WGP-NEXT:    s_endpgm
3989;
3990; GFX10-CU-LABEL: local_singlethread_one_as_unordered_load:
3991; GFX10-CU:       ; %bb.0: ; %entry
3992; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3993; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3994; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3995; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3996; GFX10-CU-NEXT:    ds_read_b32 v0, v0
3997; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3998; GFX10-CU-NEXT:    ds_write_b32 v1, v0
3999; GFX10-CU-NEXT:    s_endpgm
4000;
4001; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_unordered_load:
4002; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4003; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
4004; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
4005; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4006; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4007; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
4008; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
4009; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4010; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
4011; SKIP-CACHE-INV-NEXT:    s_endpgm
4012;
4013; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_unordered_load:
4014; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4015; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4016; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4017; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4018; GFX90A-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
4019; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
4020; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4021; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
4022; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4023;
4024; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_unordered_load:
4025; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4026; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4027; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4028; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4029; GFX90A-TGSPLIT-NEXT:    ds_read_b32 v0, v0
4030; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
4031; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4032; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v1, v0
4033; GFX90A-TGSPLIT-NEXT:    s_endpgm
4034    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
4035entry:
4036  %val = load atomic i32, i32 addrspace(3)* %in syncscope("singlethread-one-as") unordered, align 4
4037  store i32 %val, i32 addrspace(3)* %out
4038  ret void
4039}
4040
4041define amdgpu_kernel void @local_singlethread_one_as_monotonic_load(
4042; GFX6-LABEL: local_singlethread_one_as_monotonic_load:
4043; GFX6:       ; %bb.0: ; %entry
4044; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
4045; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
4046; GFX6-NEXT:    s_mov_b32 m0, -1
4047; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4048; GFX6-NEXT:    v_mov_b32_e32 v0, s0
4049; GFX6-NEXT:    ds_read_b32 v0, v0
4050; GFX6-NEXT:    v_mov_b32_e32 v1, s1
4051; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4052; GFX6-NEXT:    ds_write_b32 v1, v0
4053; GFX6-NEXT:    s_endpgm
4054;
4055; GFX7-LABEL: local_singlethread_one_as_monotonic_load:
4056; GFX7:       ; %bb.0: ; %entry
4057; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4058; GFX7-NEXT:    s_mov_b32 m0, -1
4059; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4060; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4061; GFX7-NEXT:    ds_read_b32 v0, v0
4062; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4063; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4064; GFX7-NEXT:    ds_write_b32 v1, v0
4065; GFX7-NEXT:    s_endpgm
4066;
4067; GFX10-WGP-LABEL: local_singlethread_one_as_monotonic_load:
4068; GFX10-WGP:       ; %bb.0: ; %entry
4069; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4070; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4071; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4072; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4073; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
4074; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4075; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
4076; GFX10-WGP-NEXT:    s_endpgm
4077;
4078; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_load:
4079; GFX10-CU:       ; %bb.0: ; %entry
4080; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4081; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4082; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4083; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4084; GFX10-CU-NEXT:    ds_read_b32 v0, v0
4085; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4086; GFX10-CU-NEXT:    ds_write_b32 v1, v0
4087; GFX10-CU-NEXT:    s_endpgm
4088;
4089; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_monotonic_load:
4090; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4091; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
4092; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
4093; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4094; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4095; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
4096; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
4097; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4098; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
4099; SKIP-CACHE-INV-NEXT:    s_endpgm
4100;
4101; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_load:
4102; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4103; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4104; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4105; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4106; GFX90A-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
4107; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
4108; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4109; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
4110; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4111;
4112; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_load:
4113; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4114; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4115; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4116; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4117; GFX90A-TGSPLIT-NEXT:    ds_read_b32 v0, v0
4118; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
4119; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4120; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v1, v0
4121; GFX90A-TGSPLIT-NEXT:    s_endpgm
4122    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
4123entry:
4124  %val = load atomic i32, i32 addrspace(3)* %in syncscope("singlethread-one-as") monotonic, align 4
4125  store i32 %val, i32 addrspace(3)* %out
4126  ret void
4127}
4128
4129define amdgpu_kernel void @local_singlethread_one_as_acquire_load(
4130; GFX6-LABEL: local_singlethread_one_as_acquire_load:
4131; GFX6:       ; %bb.0: ; %entry
4132; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
4133; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
4134; GFX6-NEXT:    s_mov_b32 m0, -1
4135; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4136; GFX6-NEXT:    v_mov_b32_e32 v0, s0
4137; GFX6-NEXT:    ds_read_b32 v0, v0
4138; GFX6-NEXT:    v_mov_b32_e32 v1, s1
4139; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4140; GFX6-NEXT:    ds_write_b32 v1, v0
4141; GFX6-NEXT:    s_endpgm
4142;
4143; GFX7-LABEL: local_singlethread_one_as_acquire_load:
4144; GFX7:       ; %bb.0: ; %entry
4145; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4146; GFX7-NEXT:    s_mov_b32 m0, -1
4147; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4148; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4149; GFX7-NEXT:    ds_read_b32 v0, v0
4150; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4151; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4152; GFX7-NEXT:    ds_write_b32 v1, v0
4153; GFX7-NEXT:    s_endpgm
4154;
4155; GFX10-WGP-LABEL: local_singlethread_one_as_acquire_load:
4156; GFX10-WGP:       ; %bb.0: ; %entry
4157; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4158; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4159; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4160; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4161; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
4162; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4163; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
4164; GFX10-WGP-NEXT:    s_endpgm
4165;
4166; GFX10-CU-LABEL: local_singlethread_one_as_acquire_load:
4167; GFX10-CU:       ; %bb.0: ; %entry
4168; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4169; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4170; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4171; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4172; GFX10-CU-NEXT:    ds_read_b32 v0, v0
4173; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4174; GFX10-CU-NEXT:    ds_write_b32 v1, v0
4175; GFX10-CU-NEXT:    s_endpgm
4176;
4177; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acquire_load:
4178; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4179; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
4180; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
4181; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4182; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4183; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
4184; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
4185; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4186; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
4187; SKIP-CACHE-INV-NEXT:    s_endpgm
4188;
4189; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_load:
4190; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4191; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4192; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4193; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4194; GFX90A-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
4195; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
4196; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4197; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
4198; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4199;
4200; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acquire_load:
4201; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4202; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4203; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4204; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4205; GFX90A-TGSPLIT-NEXT:    ds_read_b32 v0, v0
4206; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
4207; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4208; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v1, v0
4209; GFX90A-TGSPLIT-NEXT:    s_endpgm
4210    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
4211entry:
4212  %val = load atomic i32, i32 addrspace(3)* %in syncscope("singlethread-one-as") acquire, align 4
4213  store i32 %val, i32 addrspace(3)* %out
4214  ret void
4215}
4216
4217define amdgpu_kernel void @local_singlethread_one_as_seq_cst_load(
4218; GFX6-LABEL: local_singlethread_one_as_seq_cst_load:
4219; GFX6:       ; %bb.0: ; %entry
4220; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
4221; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
4222; GFX6-NEXT:    s_mov_b32 m0, -1
4223; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4224; GFX6-NEXT:    v_mov_b32_e32 v0, s0
4225; GFX6-NEXT:    ds_read_b32 v0, v0
4226; GFX6-NEXT:    v_mov_b32_e32 v1, s1
4227; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4228; GFX6-NEXT:    ds_write_b32 v1, v0
4229; GFX6-NEXT:    s_endpgm
4230;
4231; GFX7-LABEL: local_singlethread_one_as_seq_cst_load:
4232; GFX7:       ; %bb.0: ; %entry
4233; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4234; GFX7-NEXT:    s_mov_b32 m0, -1
4235; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4236; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4237; GFX7-NEXT:    ds_read_b32 v0, v0
4238; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4239; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4240; GFX7-NEXT:    ds_write_b32 v1, v0
4241; GFX7-NEXT:    s_endpgm
4242;
4243; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_load:
4244; GFX10-WGP:       ; %bb.0: ; %entry
4245; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4246; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4247; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4248; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4249; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
4250; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4251; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
4252; GFX10-WGP-NEXT:    s_endpgm
4253;
4254; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_load:
4255; GFX10-CU:       ; %bb.0: ; %entry
4256; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4257; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4258; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4259; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4260; GFX10-CU-NEXT:    ds_read_b32 v0, v0
4261; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4262; GFX10-CU-NEXT:    ds_write_b32 v1, v0
4263; GFX10-CU-NEXT:    s_endpgm
4264;
4265; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_load:
4266; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4267; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
4268; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
4269; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4270; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4271; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
4272; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
4273; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4274; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
4275; SKIP-CACHE-INV-NEXT:    s_endpgm
4276;
4277; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_load:
4278; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4279; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4280; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4281; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4282; GFX90A-NOTTGSPLIT-NEXT:    ds_read_b32 v0, v0
4283; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
4284; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4285; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v1, v0
4286; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4287;
4288; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_load:
4289; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4290; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4291; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4292; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4293; GFX90A-TGSPLIT-NEXT:    ds_read_b32 v0, v0
4294; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
4295; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4296; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v1, v0
4297; GFX90A-TGSPLIT-NEXT:    s_endpgm
4298    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
4299entry:
4300  %val = load atomic i32, i32 addrspace(3)* %in syncscope("singlethread-one-as") seq_cst, align 4
4301  store i32 %val, i32 addrspace(3)* %out
4302  ret void
4303}
4304
4305define amdgpu_kernel void @local_singlethread_one_as_unordered_store(
4306; GFX6-LABEL: local_singlethread_one_as_unordered_store:
4307; GFX6:       ; %bb.0: ; %entry
4308; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x1
4309; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x0
4310; GFX6-NEXT:    s_mov_b32 m0, -1
4311; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4312; GFX6-NEXT:    v_mov_b32_e32 v0, s0
4313; GFX6-NEXT:    v_mov_b32_e32 v1, s1
4314; GFX6-NEXT:    ds_write_b32 v0, v1
4315; GFX6-NEXT:    s_endpgm
4316;
4317; GFX7-LABEL: local_singlethread_one_as_unordered_store:
4318; GFX7:       ; %bb.0: ; %entry
4319; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4320; GFX7-NEXT:    s_mov_b32 m0, -1
4321; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4322; GFX7-NEXT:    v_mov_b32_e32 v0, s1
4323; GFX7-NEXT:    v_mov_b32_e32 v1, s0
4324; GFX7-NEXT:    ds_write_b32 v0, v1
4325; GFX7-NEXT:    s_endpgm
4326;
4327; GFX10-WGP-LABEL: local_singlethread_one_as_unordered_store:
4328; GFX10-WGP:       ; %bb.0: ; %entry
4329; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4330; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4331; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
4332; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
4333; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
4334; GFX10-WGP-NEXT:    s_endpgm
4335;
4336; GFX10-CU-LABEL: local_singlethread_one_as_unordered_store:
4337; GFX10-CU:       ; %bb.0: ; %entry
4338; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4339; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4340; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
4341; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
4342; GFX10-CU-NEXT:    ds_write_b32 v0, v1
4343; GFX10-CU-NEXT:    s_endpgm
4344;
4345; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_unordered_store:
4346; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4347; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
4348; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
4349; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4350; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
4351; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
4352; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
4353; SKIP-CACHE-INV-NEXT:    s_endpgm
4354;
4355; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_unordered_store:
4356; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4357; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4358; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4359; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
4360; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
4361; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
4362; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4363;
4364; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_unordered_store:
4365; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4366; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4367; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4368; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
4369; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
4370; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
4371; GFX90A-TGSPLIT-NEXT:    s_endpgm
4372    i32 %in, i32 addrspace(3)* %out) {
4373entry:
4374  store atomic i32 %in, i32 addrspace(3)* %out syncscope("singlethread-one-as") unordered, align 4
4375  ret void
4376}
4377
4378define amdgpu_kernel void @local_singlethread_one_as_monotonic_store(
4379; GFX6-LABEL: local_singlethread_one_as_monotonic_store:
4380; GFX6:       ; %bb.0: ; %entry
4381; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x1
4382; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x0
4383; GFX6-NEXT:    s_mov_b32 m0, -1
4384; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4385; GFX6-NEXT:    v_mov_b32_e32 v0, s0
4386; GFX6-NEXT:    v_mov_b32_e32 v1, s1
4387; GFX6-NEXT:    ds_write_b32 v0, v1
4388; GFX6-NEXT:    s_endpgm
4389;
4390; GFX7-LABEL: local_singlethread_one_as_monotonic_store:
4391; GFX7:       ; %bb.0: ; %entry
4392; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4393; GFX7-NEXT:    s_mov_b32 m0, -1
4394; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4395; GFX7-NEXT:    v_mov_b32_e32 v0, s1
4396; GFX7-NEXT:    v_mov_b32_e32 v1, s0
4397; GFX7-NEXT:    ds_write_b32 v0, v1
4398; GFX7-NEXT:    s_endpgm
4399;
4400; GFX10-WGP-LABEL: local_singlethread_one_as_monotonic_store:
4401; GFX10-WGP:       ; %bb.0: ; %entry
4402; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4403; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4404; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
4405; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
4406; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
4407; GFX10-WGP-NEXT:    s_endpgm
4408;
4409; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_store:
4410; GFX10-CU:       ; %bb.0: ; %entry
4411; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4412; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4413; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
4414; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
4415; GFX10-CU-NEXT:    ds_write_b32 v0, v1
4416; GFX10-CU-NEXT:    s_endpgm
4417;
4418; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_monotonic_store:
4419; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4420; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
4421; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
4422; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4423; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
4424; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
4425; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
4426; SKIP-CACHE-INV-NEXT:    s_endpgm
4427;
4428; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_store:
4429; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4430; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4431; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4432; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
4433; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
4434; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
4435; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4436;
4437; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_store:
4438; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4439; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4440; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4441; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
4442; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
4443; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
4444; GFX90A-TGSPLIT-NEXT:    s_endpgm
4445    i32 %in, i32 addrspace(3)* %out) {
4446entry:
4447  store atomic i32 %in, i32 addrspace(3)* %out syncscope("singlethread-one-as") monotonic, align 4
4448  ret void
4449}
4450
4451define amdgpu_kernel void @local_singlethread_one_as_release_store(
4452; GFX6-LABEL: local_singlethread_one_as_release_store:
4453; GFX6:       ; %bb.0: ; %entry
4454; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x1
4455; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x0
4456; GFX6-NEXT:    s_mov_b32 m0, -1
4457; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4458; GFX6-NEXT:    v_mov_b32_e32 v0, s0
4459; GFX6-NEXT:    v_mov_b32_e32 v1, s1
4460; GFX6-NEXT:    ds_write_b32 v0, v1
4461; GFX6-NEXT:    s_endpgm
4462;
4463; GFX7-LABEL: local_singlethread_one_as_release_store:
4464; GFX7:       ; %bb.0: ; %entry
4465; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4466; GFX7-NEXT:    s_mov_b32 m0, -1
4467; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4468; GFX7-NEXT:    v_mov_b32_e32 v0, s1
4469; GFX7-NEXT:    v_mov_b32_e32 v1, s0
4470; GFX7-NEXT:    ds_write_b32 v0, v1
4471; GFX7-NEXT:    s_endpgm
4472;
4473; GFX10-WGP-LABEL: local_singlethread_one_as_release_store:
4474; GFX10-WGP:       ; %bb.0: ; %entry
4475; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4476; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4477; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
4478; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
4479; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
4480; GFX10-WGP-NEXT:    s_endpgm
4481;
4482; GFX10-CU-LABEL: local_singlethread_one_as_release_store:
4483; GFX10-CU:       ; %bb.0: ; %entry
4484; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4485; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4486; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
4487; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
4488; GFX10-CU-NEXT:    ds_write_b32 v0, v1
4489; GFX10-CU-NEXT:    s_endpgm
4490;
4491; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_release_store:
4492; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4493; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
4494; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
4495; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4496; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
4497; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
4498; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
4499; SKIP-CACHE-INV-NEXT:    s_endpgm
4500;
4501; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_store:
4502; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4503; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4504; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4505; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
4506; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
4507; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
4508; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4509;
4510; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_release_store:
4511; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4512; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4513; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4514; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
4515; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
4516; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
4517; GFX90A-TGSPLIT-NEXT:    s_endpgm
4518    i32 %in, i32 addrspace(3)* %out) {
4519entry:
4520  store atomic i32 %in, i32 addrspace(3)* %out syncscope("singlethread-one-as") release, align 4
4521  ret void
4522}
4523
4524define amdgpu_kernel void @local_singlethread_one_as_seq_cst_store(
4525; GFX6-LABEL: local_singlethread_one_as_seq_cst_store:
4526; GFX6:       ; %bb.0: ; %entry
4527; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x1
4528; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x0
4529; GFX6-NEXT:    s_mov_b32 m0, -1
4530; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4531; GFX6-NEXT:    v_mov_b32_e32 v0, s0
4532; GFX6-NEXT:    v_mov_b32_e32 v1, s1
4533; GFX6-NEXT:    ds_write_b32 v0, v1
4534; GFX6-NEXT:    s_endpgm
4535;
4536; GFX7-LABEL: local_singlethread_one_as_seq_cst_store:
4537; GFX7:       ; %bb.0: ; %entry
4538; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4539; GFX7-NEXT:    s_mov_b32 m0, -1
4540; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4541; GFX7-NEXT:    v_mov_b32_e32 v0, s1
4542; GFX7-NEXT:    v_mov_b32_e32 v1, s0
4543; GFX7-NEXT:    ds_write_b32 v0, v1
4544; GFX7-NEXT:    s_endpgm
4545;
4546; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_store:
4547; GFX10-WGP:       ; %bb.0: ; %entry
4548; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4549; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4550; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
4551; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
4552; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
4553; GFX10-WGP-NEXT:    s_endpgm
4554;
4555; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_store:
4556; GFX10-CU:       ; %bb.0: ; %entry
4557; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4558; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4559; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
4560; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
4561; GFX10-CU-NEXT:    ds_write_b32 v0, v1
4562; GFX10-CU-NEXT:    s_endpgm
4563;
4564; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_store:
4565; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4566; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
4567; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
4568; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4569; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
4570; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
4571; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
4572; SKIP-CACHE-INV-NEXT:    s_endpgm
4573;
4574; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_store:
4575; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4576; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4577; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4578; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
4579; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
4580; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
4581; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4582;
4583; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_store:
4584; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4585; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4586; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4587; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s1
4588; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s0
4589; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
4590; GFX90A-TGSPLIT-NEXT:    s_endpgm
4591    i32 %in, i32 addrspace(3)* %out) {
4592entry:
4593  store atomic i32 %in, i32 addrspace(3)* %out syncscope("singlethread-one-as") seq_cst, align 4
4594  ret void
4595}
4596
4597define amdgpu_kernel void @local_singlethread_one_as_monotonic_atomicrmw(
4598; GFX6-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
4599; GFX6:       ; %bb.0: ; %entry
4600; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
4601; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
4602; GFX6-NEXT:    s_mov_b32 m0, -1
4603; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4604; GFX6-NEXT:    v_mov_b32_e32 v0, s0
4605; GFX6-NEXT:    v_mov_b32_e32 v1, s1
4606; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
4607; GFX6-NEXT:    s_endpgm
4608;
4609; GFX7-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
4610; GFX7:       ; %bb.0: ; %entry
4611; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4612; GFX7-NEXT:    s_mov_b32 m0, -1
4613; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4614; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4615; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4616; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
4617; GFX7-NEXT:    s_endpgm
4618;
4619; GFX10-WGP-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
4620; GFX10-WGP:       ; %bb.0: ; %entry
4621; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4622; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4623; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4624; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4625; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
4626; GFX10-WGP-NEXT:    s_endpgm
4627;
4628; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
4629; GFX10-CU:       ; %bb.0: ; %entry
4630; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4631; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4632; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4633; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4634; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
4635; GFX10-CU-NEXT:    s_endpgm
4636;
4637; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
4638; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4639; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
4640; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
4641; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4642; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4643; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
4644; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
4645; SKIP-CACHE-INV-NEXT:    s_endpgm
4646;
4647; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
4648; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4649; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4650; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4651; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4652; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
4653; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
4654; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4655;
4656; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
4657; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4658; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4659; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4660; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4661; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
4662; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
4663; GFX90A-TGSPLIT-NEXT:    s_endpgm
4664    i32 addrspace(3)* %out, i32 %in) {
4665entry:
4666  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread-one-as") monotonic
4667  ret void
4668}
4669
4670define amdgpu_kernel void @local_singlethread_one_as_acquire_atomicrmw(
4671; GFX6-LABEL: local_singlethread_one_as_acquire_atomicrmw:
4672; GFX6:       ; %bb.0: ; %entry
4673; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
4674; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
4675; GFX6-NEXT:    s_mov_b32 m0, -1
4676; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4677; GFX6-NEXT:    v_mov_b32_e32 v0, s0
4678; GFX6-NEXT:    v_mov_b32_e32 v1, s1
4679; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
4680; GFX6-NEXT:    s_endpgm
4681;
4682; GFX7-LABEL: local_singlethread_one_as_acquire_atomicrmw:
4683; GFX7:       ; %bb.0: ; %entry
4684; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4685; GFX7-NEXT:    s_mov_b32 m0, -1
4686; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4687; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4688; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4689; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
4690; GFX7-NEXT:    s_endpgm
4691;
4692; GFX10-WGP-LABEL: local_singlethread_one_as_acquire_atomicrmw:
4693; GFX10-WGP:       ; %bb.0: ; %entry
4694; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4695; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4696; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4697; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4698; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
4699; GFX10-WGP-NEXT:    s_endpgm
4700;
4701; GFX10-CU-LABEL: local_singlethread_one_as_acquire_atomicrmw:
4702; GFX10-CU:       ; %bb.0: ; %entry
4703; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4704; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4705; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4706; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4707; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
4708; GFX10-CU-NEXT:    s_endpgm
4709;
4710; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acquire_atomicrmw:
4711; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4712; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
4713; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
4714; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4715; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4716; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
4717; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
4718; SKIP-CACHE-INV-NEXT:    s_endpgm
4719;
4720; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_atomicrmw:
4721; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4722; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4723; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4724; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4725; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
4726; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
4727; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4728;
4729; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acquire_atomicrmw:
4730; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4731; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4732; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4733; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4734; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
4735; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
4736; GFX90A-TGSPLIT-NEXT:    s_endpgm
4737    i32 addrspace(3)* %out, i32 %in) {
4738entry:
4739  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread-one-as") acquire
4740  ret void
4741}
4742
4743define amdgpu_kernel void @local_singlethread_one_as_release_atomicrmw(
4744; GFX6-LABEL: local_singlethread_one_as_release_atomicrmw:
4745; GFX6:       ; %bb.0: ; %entry
4746; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
4747; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
4748; GFX6-NEXT:    s_mov_b32 m0, -1
4749; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4750; GFX6-NEXT:    v_mov_b32_e32 v0, s0
4751; GFX6-NEXT:    v_mov_b32_e32 v1, s1
4752; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
4753; GFX6-NEXT:    s_endpgm
4754;
4755; GFX7-LABEL: local_singlethread_one_as_release_atomicrmw:
4756; GFX7:       ; %bb.0: ; %entry
4757; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4758; GFX7-NEXT:    s_mov_b32 m0, -1
4759; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4760; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4761; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4762; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
4763; GFX7-NEXT:    s_endpgm
4764;
4765; GFX10-WGP-LABEL: local_singlethread_one_as_release_atomicrmw:
4766; GFX10-WGP:       ; %bb.0: ; %entry
4767; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4768; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4769; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4770; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4771; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
4772; GFX10-WGP-NEXT:    s_endpgm
4773;
4774; GFX10-CU-LABEL: local_singlethread_one_as_release_atomicrmw:
4775; GFX10-CU:       ; %bb.0: ; %entry
4776; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4777; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4778; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4779; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4780; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
4781; GFX10-CU-NEXT:    s_endpgm
4782;
4783; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_release_atomicrmw:
4784; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4785; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
4786; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
4787; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4788; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4789; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
4790; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
4791; SKIP-CACHE-INV-NEXT:    s_endpgm
4792;
4793; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_atomicrmw:
4794; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4795; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4796; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4797; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4798; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
4799; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
4800; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4801;
4802; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_release_atomicrmw:
4803; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4804; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4805; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4806; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4807; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
4808; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
4809; GFX90A-TGSPLIT-NEXT:    s_endpgm
4810    i32 addrspace(3)* %out, i32 %in) {
4811entry:
4812  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread-one-as") release
4813  ret void
4814}
4815
4816define amdgpu_kernel void @local_singlethread_one_as_acq_rel_atomicrmw(
4817; GFX6-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
4818; GFX6:       ; %bb.0: ; %entry
4819; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
4820; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
4821; GFX6-NEXT:    s_mov_b32 m0, -1
4822; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4823; GFX6-NEXT:    v_mov_b32_e32 v0, s0
4824; GFX6-NEXT:    v_mov_b32_e32 v1, s1
4825; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
4826; GFX6-NEXT:    s_endpgm
4827;
4828; GFX7-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
4829; GFX7:       ; %bb.0: ; %entry
4830; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4831; GFX7-NEXT:    s_mov_b32 m0, -1
4832; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4833; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4834; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4835; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
4836; GFX7-NEXT:    s_endpgm
4837;
4838; GFX10-WGP-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
4839; GFX10-WGP:       ; %bb.0: ; %entry
4840; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4841; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4842; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4843; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4844; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
4845; GFX10-WGP-NEXT:    s_endpgm
4846;
4847; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
4848; GFX10-CU:       ; %bb.0: ; %entry
4849; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4850; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4851; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4852; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4853; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
4854; GFX10-CU-NEXT:    s_endpgm
4855;
4856; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
4857; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4858; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
4859; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
4860; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4861; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4862; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
4863; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
4864; SKIP-CACHE-INV-NEXT:    s_endpgm
4865;
4866; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
4867; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4868; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4869; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4870; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4871; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
4872; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
4873; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4874;
4875; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
4876; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4877; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4878; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4879; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4880; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
4881; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
4882; GFX90A-TGSPLIT-NEXT:    s_endpgm
4883    i32 addrspace(3)* %out, i32 %in) {
4884entry:
4885  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread-one-as") acq_rel
4886  ret void
4887}
4888
4889define amdgpu_kernel void @local_singlethread_one_as_seq_cst_atomicrmw(
4890; GFX6-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
4891; GFX6:       ; %bb.0: ; %entry
4892; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
4893; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
4894; GFX6-NEXT:    s_mov_b32 m0, -1
4895; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4896; GFX6-NEXT:    v_mov_b32_e32 v0, s0
4897; GFX6-NEXT:    v_mov_b32_e32 v1, s1
4898; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
4899; GFX6-NEXT:    s_endpgm
4900;
4901; GFX7-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
4902; GFX7:       ; %bb.0: ; %entry
4903; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4904; GFX7-NEXT:    s_mov_b32 m0, -1
4905; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4906; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4907; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4908; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
4909; GFX7-NEXT:    s_endpgm
4910;
4911; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
4912; GFX10-WGP:       ; %bb.0: ; %entry
4913; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4914; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4915; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4916; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4917; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
4918; GFX10-WGP-NEXT:    s_endpgm
4919;
4920; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
4921; GFX10-CU:       ; %bb.0: ; %entry
4922; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4923; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4924; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4925; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4926; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
4927; GFX10-CU-NEXT:    s_endpgm
4928;
4929; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
4930; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4931; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
4932; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
4933; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4934; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4935; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
4936; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
4937; SKIP-CACHE-INV-NEXT:    s_endpgm
4938;
4939; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
4940; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4941; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4942; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4943; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4944; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
4945; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
4946; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4947;
4948; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
4949; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4950; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4951; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4952; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4953; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
4954; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
4955; GFX90A-TGSPLIT-NEXT:    s_endpgm
4956    i32 addrspace(3)* %out, i32 %in) {
4957entry:
4958  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread-one-as") seq_cst
4959  ret void
4960}
4961
4962define amdgpu_kernel void @local_singlethread_one_as_acquire_ret_atomicrmw(
4963; GFX6-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
4964; GFX6:       ; %bb.0: ; %entry
4965; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
4966; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
4967; GFX6-NEXT:    s_mov_b32 m0, -1
4968; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4969; GFX6-NEXT:    v_mov_b32_e32 v0, s0
4970; GFX6-NEXT:    v_mov_b32_e32 v1, s1
4971; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
4972; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4973; GFX6-NEXT:    ds_write_b32 v0, v1
4974; GFX6-NEXT:    s_endpgm
4975;
4976; GFX7-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
4977; GFX7:       ; %bb.0: ; %entry
4978; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4979; GFX7-NEXT:    s_mov_b32 m0, -1
4980; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4981; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4982; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4983; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
4984; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4985; GFX7-NEXT:    ds_write_b32 v0, v1
4986; GFX7-NEXT:    s_endpgm
4987;
4988; GFX10-WGP-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
4989; GFX10-WGP:       ; %bb.0: ; %entry
4990; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4991; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4992; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4993; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4994; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
4995; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4996; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
4997; GFX10-WGP-NEXT:    s_endpgm
4998;
4999; GFX10-CU-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
5000; GFX10-CU:       ; %bb.0: ; %entry
5001; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5002; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5003; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5004; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5005; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
5006; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5007; GFX10-CU-NEXT:    ds_write_b32 v0, v1
5008; GFX10-CU-NEXT:    s_endpgm
5009;
5010; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
5011; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5012; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
5013; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
5014; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5015; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5016; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
5017; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
5018; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5019; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
5020; SKIP-CACHE-INV-NEXT:    s_endpgm
5021;
5022; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
5023; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5024; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5025; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5026; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5027; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
5028; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
5029; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5030; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
5031; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5032;
5033; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
5034; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5035; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5036; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5037; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5038; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
5039; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
5040; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5041; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
5042; GFX90A-TGSPLIT-NEXT:    s_endpgm
5043    i32 addrspace(3)* %out, i32 %in) {
5044entry:
5045  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread-one-as") acquire
5046  store i32 %val, i32 addrspace(3)* %out, align 4
5047  ret void
5048}
5049
5050define amdgpu_kernel void @local_singlethread_one_as_acq_rel_ret_atomicrmw(
5051; GFX6-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
5052; GFX6:       ; %bb.0: ; %entry
5053; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
5054; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
5055; GFX6-NEXT:    s_mov_b32 m0, -1
5056; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5057; GFX6-NEXT:    v_mov_b32_e32 v0, s0
5058; GFX6-NEXT:    v_mov_b32_e32 v1, s1
5059; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
5060; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5061; GFX6-NEXT:    ds_write_b32 v0, v1
5062; GFX6-NEXT:    s_endpgm
5063;
5064; GFX7-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
5065; GFX7:       ; %bb.0: ; %entry
5066; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5067; GFX7-NEXT:    s_mov_b32 m0, -1
5068; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5069; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5070; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5071; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
5072; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5073; GFX7-NEXT:    ds_write_b32 v0, v1
5074; GFX7-NEXT:    s_endpgm
5075;
5076; GFX10-WGP-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
5077; GFX10-WGP:       ; %bb.0: ; %entry
5078; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5079; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5080; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5081; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5082; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
5083; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5084; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
5085; GFX10-WGP-NEXT:    s_endpgm
5086;
5087; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
5088; GFX10-CU:       ; %bb.0: ; %entry
5089; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5090; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5091; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5092; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5093; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
5094; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5095; GFX10-CU-NEXT:    ds_write_b32 v0, v1
5096; GFX10-CU-NEXT:    s_endpgm
5097;
5098; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
5099; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5100; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
5101; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
5102; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5103; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5104; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
5105; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
5106; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5107; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
5108; SKIP-CACHE-INV-NEXT:    s_endpgm
5109;
5110; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
5111; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5112; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5113; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5114; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5115; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
5116; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
5117; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5118; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
5119; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5120;
5121; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
5122; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5123; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5124; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5125; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5126; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
5127; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
5128; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5129; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
5130; GFX90A-TGSPLIT-NEXT:    s_endpgm
5131    i32 addrspace(3)* %out, i32 %in) {
5132entry:
5133  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread-one-as") acq_rel
5134  store i32 %val, i32 addrspace(3)* %out, align 4
5135  ret void
5136}
5137
5138define amdgpu_kernel void @local_singlethread_one_as_seq_cst_ret_atomicrmw(
5139; GFX6-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
5140; GFX6:       ; %bb.0: ; %entry
5141; GFX6-NEXT:    s_load_dword s0, s[4:5], 0x0
5142; GFX6-NEXT:    s_load_dword s1, s[4:5], 0x1
5143; GFX6-NEXT:    s_mov_b32 m0, -1
5144; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5145; GFX6-NEXT:    v_mov_b32_e32 v0, s0
5146; GFX6-NEXT:    v_mov_b32_e32 v1, s1
5147; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
5148; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5149; GFX6-NEXT:    ds_write_b32 v0, v1
5150; GFX6-NEXT:    s_endpgm
5151;
5152; GFX7-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
5153; GFX7:       ; %bb.0: ; %entry
5154; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5155; GFX7-NEXT:    s_mov_b32 m0, -1
5156; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5157; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5158; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5159; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
5160; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5161; GFX7-NEXT:    ds_write_b32 v0, v1
5162; GFX7-NEXT:    s_endpgm
5163;
5164; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
5165; GFX10-WGP:       ; %bb.0: ; %entry
5166; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5167; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5168; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5169; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5170; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
5171; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5172; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
5173; GFX10-WGP-NEXT:    s_endpgm
5174;
5175; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
5176; GFX10-CU:       ; %bb.0: ; %entry
5177; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5178; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5179; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5180; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5181; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
5182; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5183; GFX10-CU-NEXT:    ds_write_b32 v0, v1
5184; GFX10-CU-NEXT:    s_endpgm
5185;
5186; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
5187; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5188; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
5189; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
5190; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5191; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5192; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
5193; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
5194; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5195; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
5196; SKIP-CACHE-INV-NEXT:    s_endpgm
5197;
5198; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
5199; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5200; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5201; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5202; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5203; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
5204; GFX90A-NOTTGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
5205; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5206; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
5207; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5208;
5209; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
5210; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5211; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5212; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5213; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5214; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
5215; GFX90A-TGSPLIT-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
5216; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5217; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
5218; GFX90A-TGSPLIT-NEXT:    s_endpgm
5219    i32 addrspace(3)* %out, i32 %in) {
5220entry:
5221  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread-one-as") seq_cst
5222  store i32 %val, i32 addrspace(3)* %out, align 4
5223  ret void
5224}
5225
5226define amdgpu_kernel void @local_singlethread_one_as_monotonic_monotonic_cmpxchg(
5227; GFX6-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
5228; GFX6:       ; %bb.0: ; %entry
5229; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
5230; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
5231; GFX6-NEXT:    s_mov_b32 m0, -1
5232; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5233; GFX6-NEXT:    v_mov_b32_e32 v0, s2
5234; GFX6-NEXT:    v_mov_b32_e32 v1, s1
5235; GFX6-NEXT:    v_mov_b32_e32 v2, s0
5236; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5237; GFX6-NEXT:    s_endpgm
5238;
5239; GFX7-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
5240; GFX7:       ; %bb.0: ; %entry
5241; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5242; GFX7-NEXT:    s_mov_b32 m0, -1
5243; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5244; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5245; GFX7-NEXT:    v_mov_b32_e32 v1, s2
5246; GFX7-NEXT:    v_mov_b32_e32 v2, s1
5247; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5248; GFX7-NEXT:    s_endpgm
5249;
5250; GFX10-WGP-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
5251; GFX10-WGP:       ; %bb.0: ; %entry
5252; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5253; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5254; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5255; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5256; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
5257; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5258; GFX10-WGP-NEXT:    s_endpgm
5259;
5260; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
5261; GFX10-CU:       ; %bb.0: ; %entry
5262; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5263; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5264; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5265; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5266; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
5267; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5268; GFX10-CU-NEXT:    s_endpgm
5269;
5270; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
5271; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5272; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5273; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
5274; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5275; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5276; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
5277; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
5278; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5279; SKIP-CACHE-INV-NEXT:    s_endpgm
5280;
5281; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
5282; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5283; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5284; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5285; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5286; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5287; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5288; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5289; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5290;
5291; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
5292; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5293; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5294; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5295; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5296; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5297; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5298; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5299; GFX90A-TGSPLIT-NEXT:    s_endpgm
5300    i32 addrspace(3)* %out, i32 %in, i32 %old) {
5301entry:
5302  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
5303  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic monotonic
5304  ret void
5305}
5306
5307define amdgpu_kernel void @local_singlethread_one_as_acquire_monotonic_cmpxchg(
5308; GFX6-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
5309; GFX6:       ; %bb.0: ; %entry
5310; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
5311; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
5312; GFX6-NEXT:    s_mov_b32 m0, -1
5313; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5314; GFX6-NEXT:    v_mov_b32_e32 v0, s2
5315; GFX6-NEXT:    v_mov_b32_e32 v1, s1
5316; GFX6-NEXT:    v_mov_b32_e32 v2, s0
5317; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5318; GFX6-NEXT:    s_endpgm
5319;
5320; GFX7-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
5321; GFX7:       ; %bb.0: ; %entry
5322; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5323; GFX7-NEXT:    s_mov_b32 m0, -1
5324; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5325; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5326; GFX7-NEXT:    v_mov_b32_e32 v1, s2
5327; GFX7-NEXT:    v_mov_b32_e32 v2, s1
5328; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5329; GFX7-NEXT:    s_endpgm
5330;
5331; GFX10-WGP-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
5332; GFX10-WGP:       ; %bb.0: ; %entry
5333; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5334; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5335; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5336; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5337; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
5338; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5339; GFX10-WGP-NEXT:    s_endpgm
5340;
5341; GFX10-CU-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
5342; GFX10-CU:       ; %bb.0: ; %entry
5343; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5344; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5345; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5346; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5347; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
5348; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5349; GFX10-CU-NEXT:    s_endpgm
5350;
5351; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
5352; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5353; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5354; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
5355; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5356; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5357; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
5358; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
5359; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5360; SKIP-CACHE-INV-NEXT:    s_endpgm
5361;
5362; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
5363; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5364; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5365; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5366; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5367; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5368; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5369; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5370; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5371;
5372; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
5373; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5374; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5375; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5376; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5377; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5378; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5379; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5380; GFX90A-TGSPLIT-NEXT:    s_endpgm
5381    i32 addrspace(3)* %out, i32 %in, i32 %old) {
5382entry:
5383  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
5384  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire monotonic
5385  ret void
5386}
5387
5388define amdgpu_kernel void @local_singlethread_one_as_release_monotonic_cmpxchg(
5389; GFX6-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
5390; GFX6:       ; %bb.0: ; %entry
5391; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
5392; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
5393; GFX6-NEXT:    s_mov_b32 m0, -1
5394; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5395; GFX6-NEXT:    v_mov_b32_e32 v0, s2
5396; GFX6-NEXT:    v_mov_b32_e32 v1, s1
5397; GFX6-NEXT:    v_mov_b32_e32 v2, s0
5398; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5399; GFX6-NEXT:    s_endpgm
5400;
5401; GFX7-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
5402; GFX7:       ; %bb.0: ; %entry
5403; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5404; GFX7-NEXT:    s_mov_b32 m0, -1
5405; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5406; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5407; GFX7-NEXT:    v_mov_b32_e32 v1, s2
5408; GFX7-NEXT:    v_mov_b32_e32 v2, s1
5409; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5410; GFX7-NEXT:    s_endpgm
5411;
5412; GFX10-WGP-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
5413; GFX10-WGP:       ; %bb.0: ; %entry
5414; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5415; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5416; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5417; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5418; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
5419; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5420; GFX10-WGP-NEXT:    s_endpgm
5421;
5422; GFX10-CU-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
5423; GFX10-CU:       ; %bb.0: ; %entry
5424; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5425; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5426; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5427; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5428; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
5429; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5430; GFX10-CU-NEXT:    s_endpgm
5431;
5432; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
5433; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5434; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5435; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
5436; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5437; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5438; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
5439; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
5440; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5441; SKIP-CACHE-INV-NEXT:    s_endpgm
5442;
5443; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
5444; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5445; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5446; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5447; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5448; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5449; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5450; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5451; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5452;
5453; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
5454; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5455; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5456; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5457; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5458; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5459; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5460; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5461; GFX90A-TGSPLIT-NEXT:    s_endpgm
5462    i32 addrspace(3)* %out, i32 %in, i32 %old) {
5463entry:
5464  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
5465  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release monotonic
5466  ret void
5467}
5468
5469define amdgpu_kernel void @local_singlethread_one_as_acq_rel_monotonic_cmpxchg(
5470; GFX6-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
5471; GFX6:       ; %bb.0: ; %entry
5472; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
5473; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
5474; GFX6-NEXT:    s_mov_b32 m0, -1
5475; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5476; GFX6-NEXT:    v_mov_b32_e32 v0, s2
5477; GFX6-NEXT:    v_mov_b32_e32 v1, s1
5478; GFX6-NEXT:    v_mov_b32_e32 v2, s0
5479; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5480; GFX6-NEXT:    s_endpgm
5481;
5482; GFX7-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
5483; GFX7:       ; %bb.0: ; %entry
5484; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5485; GFX7-NEXT:    s_mov_b32 m0, -1
5486; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5487; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5488; GFX7-NEXT:    v_mov_b32_e32 v1, s2
5489; GFX7-NEXT:    v_mov_b32_e32 v2, s1
5490; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5491; GFX7-NEXT:    s_endpgm
5492;
5493; GFX10-WGP-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
5494; GFX10-WGP:       ; %bb.0: ; %entry
5495; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5496; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5497; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5498; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5499; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
5500; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5501; GFX10-WGP-NEXT:    s_endpgm
5502;
5503; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
5504; GFX10-CU:       ; %bb.0: ; %entry
5505; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5506; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5507; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5508; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5509; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
5510; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5511; GFX10-CU-NEXT:    s_endpgm
5512;
5513; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
5514; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5515; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5516; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
5517; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5518; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5519; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
5520; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
5521; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5522; SKIP-CACHE-INV-NEXT:    s_endpgm
5523;
5524; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
5525; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5526; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5527; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5528; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5529; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5530; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5531; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5532; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5533;
5534; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
5535; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5536; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5537; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5538; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5539; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5540; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5541; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5542; GFX90A-TGSPLIT-NEXT:    s_endpgm
5543    i32 addrspace(3)* %out, i32 %in, i32 %old) {
5544entry:
5545  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
5546  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel monotonic
5547  ret void
5548}
5549
5550define amdgpu_kernel void @local_singlethread_one_as_seq_cst_monotonic_cmpxchg(
5551; GFX6-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
5552; GFX6:       ; %bb.0: ; %entry
5553; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
5554; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
5555; GFX6-NEXT:    s_mov_b32 m0, -1
5556; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5557; GFX6-NEXT:    v_mov_b32_e32 v0, s2
5558; GFX6-NEXT:    v_mov_b32_e32 v1, s1
5559; GFX6-NEXT:    v_mov_b32_e32 v2, s0
5560; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5561; GFX6-NEXT:    s_endpgm
5562;
5563; GFX7-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
5564; GFX7:       ; %bb.0: ; %entry
5565; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5566; GFX7-NEXT:    s_mov_b32 m0, -1
5567; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5568; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5569; GFX7-NEXT:    v_mov_b32_e32 v1, s2
5570; GFX7-NEXT:    v_mov_b32_e32 v2, s1
5571; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5572; GFX7-NEXT:    s_endpgm
5573;
5574; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
5575; GFX10-WGP:       ; %bb.0: ; %entry
5576; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5577; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5578; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5579; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5580; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
5581; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5582; GFX10-WGP-NEXT:    s_endpgm
5583;
5584; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
5585; GFX10-CU:       ; %bb.0: ; %entry
5586; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5587; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5588; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5589; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5590; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
5591; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5592; GFX10-CU-NEXT:    s_endpgm
5593;
5594; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
5595; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5596; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5597; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
5598; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5599; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5600; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
5601; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
5602; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5603; SKIP-CACHE-INV-NEXT:    s_endpgm
5604;
5605; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
5606; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5607; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5608; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5609; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5610; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5611; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5612; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5613; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5614;
5615; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
5616; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5617; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5618; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5619; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5620; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5621; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5622; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5623; GFX90A-TGSPLIT-NEXT:    s_endpgm
5624    i32 addrspace(3)* %out, i32 %in, i32 %old) {
5625entry:
5626  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
5627  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst monotonic
5628  ret void
5629}
5630
5631define amdgpu_kernel void @local_singlethread_one_as_monotonic_acquire_cmpxchg(
5632; GFX6-LABEL: local_singlethread_one_as_monotonic_acquire_cmpxchg:
5633; GFX6:       ; %bb.0: ; %entry
5634; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
5635; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
5636; GFX6-NEXT:    s_mov_b32 m0, -1
5637; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5638; GFX6-NEXT:    v_mov_b32_e32 v0, s2
5639; GFX6-NEXT:    v_mov_b32_e32 v1, s1
5640; GFX6-NEXT:    v_mov_b32_e32 v2, s0
5641; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5642; GFX6-NEXT:    s_endpgm
5643;
5644; GFX7-LABEL: local_singlethread_one_as_monotonic_acquire_cmpxchg:
5645; GFX7:       ; %bb.0: ; %entry
5646; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5647; GFX7-NEXT:    s_mov_b32 m0, -1
5648; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5649; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5650; GFX7-NEXT:    v_mov_b32_e32 v1, s2
5651; GFX7-NEXT:    v_mov_b32_e32 v2, s1
5652; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5653; GFX7-NEXT:    s_endpgm
5654;
5655; GFX10-WGP-LABEL: local_singlethread_one_as_monotonic_acquire_cmpxchg:
5656; GFX10-WGP:       ; %bb.0: ; %entry
5657; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5658; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5659; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5660; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5661; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
5662; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5663; GFX10-WGP-NEXT:    s_endpgm
5664;
5665; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_acquire_cmpxchg:
5666; GFX10-CU:       ; %bb.0: ; %entry
5667; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5668; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5669; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5670; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5671; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
5672; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5673; GFX10-CU-NEXT:    s_endpgm
5674;
5675; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_monotonic_acquire_cmpxchg:
5676; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5677; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5678; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
5679; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5680; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5681; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
5682; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
5683; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5684; SKIP-CACHE-INV-NEXT:    s_endpgm
5685;
5686; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_acquire_cmpxchg:
5687; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5688; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5689; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5690; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5691; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5692; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5693; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5694; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5695;
5696; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_acquire_cmpxchg:
5697; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5698; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5699; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5700; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5701; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5702; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5703; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5704; GFX90A-TGSPLIT-NEXT:    s_endpgm
5705    i32 addrspace(3)* %out, i32 %in, i32 %old) {
5706entry:
5707  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
5708  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic acquire
5709  ret void
5710}
5711
5712define amdgpu_kernel void @local_singlethread_one_as_acquire_acquire_cmpxchg(
5713; GFX6-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
5714; GFX6:       ; %bb.0: ; %entry
5715; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
5716; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
5717; GFX6-NEXT:    s_mov_b32 m0, -1
5718; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5719; GFX6-NEXT:    v_mov_b32_e32 v0, s2
5720; GFX6-NEXT:    v_mov_b32_e32 v1, s1
5721; GFX6-NEXT:    v_mov_b32_e32 v2, s0
5722; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5723; GFX6-NEXT:    s_endpgm
5724;
5725; GFX7-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
5726; GFX7:       ; %bb.0: ; %entry
5727; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5728; GFX7-NEXT:    s_mov_b32 m0, -1
5729; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5730; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5731; GFX7-NEXT:    v_mov_b32_e32 v1, s2
5732; GFX7-NEXT:    v_mov_b32_e32 v2, s1
5733; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5734; GFX7-NEXT:    s_endpgm
5735;
5736; GFX10-WGP-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
5737; GFX10-WGP:       ; %bb.0: ; %entry
5738; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5739; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5740; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5741; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5742; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
5743; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5744; GFX10-WGP-NEXT:    s_endpgm
5745;
5746; GFX10-CU-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
5747; GFX10-CU:       ; %bb.0: ; %entry
5748; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5749; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5750; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5751; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5752; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
5753; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5754; GFX10-CU-NEXT:    s_endpgm
5755;
5756; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
5757; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5758; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5759; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
5760; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5761; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5762; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
5763; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
5764; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5765; SKIP-CACHE-INV-NEXT:    s_endpgm
5766;
5767; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
5768; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5769; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5770; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5771; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5772; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5773; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5774; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5775; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5776;
5777; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
5778; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5779; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5780; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5781; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5782; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5783; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5784; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5785; GFX90A-TGSPLIT-NEXT:    s_endpgm
5786    i32 addrspace(3)* %out, i32 %in, i32 %old) {
5787entry:
5788  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
5789  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire acquire
5790  ret void
5791}
5792
5793define amdgpu_kernel void @local_singlethread_one_as_release_acquire_cmpxchg(
5794; GFX6-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
5795; GFX6:       ; %bb.0: ; %entry
5796; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
5797; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
5798; GFX6-NEXT:    s_mov_b32 m0, -1
5799; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5800; GFX6-NEXT:    v_mov_b32_e32 v0, s2
5801; GFX6-NEXT:    v_mov_b32_e32 v1, s1
5802; GFX6-NEXT:    v_mov_b32_e32 v2, s0
5803; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5804; GFX6-NEXT:    s_endpgm
5805;
5806; GFX7-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
5807; GFX7:       ; %bb.0: ; %entry
5808; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5809; GFX7-NEXT:    s_mov_b32 m0, -1
5810; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5811; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5812; GFX7-NEXT:    v_mov_b32_e32 v1, s2
5813; GFX7-NEXT:    v_mov_b32_e32 v2, s1
5814; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5815; GFX7-NEXT:    s_endpgm
5816;
5817; GFX10-WGP-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
5818; GFX10-WGP:       ; %bb.0: ; %entry
5819; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5820; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5821; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5822; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5823; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
5824; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5825; GFX10-WGP-NEXT:    s_endpgm
5826;
5827; GFX10-CU-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
5828; GFX10-CU:       ; %bb.0: ; %entry
5829; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5830; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5831; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5832; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5833; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
5834; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5835; GFX10-CU-NEXT:    s_endpgm
5836;
5837; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
5838; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5839; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5840; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
5841; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5842; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5843; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
5844; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
5845; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5846; SKIP-CACHE-INV-NEXT:    s_endpgm
5847;
5848; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
5849; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5850; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5851; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5852; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5853; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5854; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5855; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5856; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5857;
5858; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
5859; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5860; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5861; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5862; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5863; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5864; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5865; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5866; GFX90A-TGSPLIT-NEXT:    s_endpgm
5867    i32 addrspace(3)* %out, i32 %in, i32 %old) {
5868entry:
5869  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
5870  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release acquire
5871  ret void
5872}
5873
5874define amdgpu_kernel void @local_singlethread_one_as_acq_rel_acquire_cmpxchg(
5875; GFX6-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
5876; GFX6:       ; %bb.0: ; %entry
5877; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
5878; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
5879; GFX6-NEXT:    s_mov_b32 m0, -1
5880; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5881; GFX6-NEXT:    v_mov_b32_e32 v0, s2
5882; GFX6-NEXT:    v_mov_b32_e32 v1, s1
5883; GFX6-NEXT:    v_mov_b32_e32 v2, s0
5884; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5885; GFX6-NEXT:    s_endpgm
5886;
5887; GFX7-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
5888; GFX7:       ; %bb.0: ; %entry
5889; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5890; GFX7-NEXT:    s_mov_b32 m0, -1
5891; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5892; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5893; GFX7-NEXT:    v_mov_b32_e32 v1, s2
5894; GFX7-NEXT:    v_mov_b32_e32 v2, s1
5895; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5896; GFX7-NEXT:    s_endpgm
5897;
5898; GFX10-WGP-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
5899; GFX10-WGP:       ; %bb.0: ; %entry
5900; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5901; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5902; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5903; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5904; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
5905; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5906; GFX10-WGP-NEXT:    s_endpgm
5907;
5908; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
5909; GFX10-CU:       ; %bb.0: ; %entry
5910; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5911; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5912; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5913; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5914; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
5915; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5916; GFX10-CU-NEXT:    s_endpgm
5917;
5918; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
5919; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5920; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5921; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
5922; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5923; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5924; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
5925; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
5926; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5927; SKIP-CACHE-INV-NEXT:    s_endpgm
5928;
5929; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
5930; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5931; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5932; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5933; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5934; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5935; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5936; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5937; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5938;
5939; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
5940; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5941; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5942; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5943; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5944; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5945; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
5946; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5947; GFX90A-TGSPLIT-NEXT:    s_endpgm
5948    i32 addrspace(3)* %out, i32 %in, i32 %old) {
5949entry:
5950  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
5951  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel acquire
5952  ret void
5953}
5954
5955define amdgpu_kernel void @local_singlethread_one_as_seq_cst_acquire_cmpxchg(
5956; GFX6-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
5957; GFX6:       ; %bb.0: ; %entry
5958; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
5959; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
5960; GFX6-NEXT:    s_mov_b32 m0, -1
5961; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5962; GFX6-NEXT:    v_mov_b32_e32 v0, s2
5963; GFX6-NEXT:    v_mov_b32_e32 v1, s1
5964; GFX6-NEXT:    v_mov_b32_e32 v2, s0
5965; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5966; GFX6-NEXT:    s_endpgm
5967;
5968; GFX7-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
5969; GFX7:       ; %bb.0: ; %entry
5970; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5971; GFX7-NEXT:    s_mov_b32 m0, -1
5972; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5973; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5974; GFX7-NEXT:    v_mov_b32_e32 v1, s2
5975; GFX7-NEXT:    v_mov_b32_e32 v2, s1
5976; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5977; GFX7-NEXT:    s_endpgm
5978;
5979; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
5980; GFX10-WGP:       ; %bb.0: ; %entry
5981; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5982; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5983; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5984; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5985; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
5986; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5987; GFX10-WGP-NEXT:    s_endpgm
5988;
5989; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
5990; GFX10-CU:       ; %bb.0: ; %entry
5991; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5992; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5993; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5994; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5995; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
5996; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
5997; GFX10-CU-NEXT:    s_endpgm
5998;
5999; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
6000; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6001; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6002; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
6003; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6004; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6005; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
6006; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
6007; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6008; SKIP-CACHE-INV-NEXT:    s_endpgm
6009;
6010; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
6011; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6012; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6013; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6014; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6015; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6016; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
6017; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6018; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6019;
6020; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
6021; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6022; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6023; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6024; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6025; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6026; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
6027; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6028; GFX90A-TGSPLIT-NEXT:    s_endpgm
6029    i32 addrspace(3)* %out, i32 %in, i32 %old) {
6030entry:
6031  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
6032  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst acquire
6033  ret void
6034}
6035
6036define amdgpu_kernel void @local_singlethread_one_as_monotonic_seq_cst_cmpxchg(
6037; GFX6-LABEL: local_singlethread_one_as_monotonic_seq_cst_cmpxchg:
6038; GFX6:       ; %bb.0: ; %entry
6039; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
6040; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
6041; GFX6-NEXT:    s_mov_b32 m0, -1
6042; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6043; GFX6-NEXT:    v_mov_b32_e32 v0, s2
6044; GFX6-NEXT:    v_mov_b32_e32 v1, s1
6045; GFX6-NEXT:    v_mov_b32_e32 v2, s0
6046; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6047; GFX6-NEXT:    s_endpgm
6048;
6049; GFX7-LABEL: local_singlethread_one_as_monotonic_seq_cst_cmpxchg:
6050; GFX7:       ; %bb.0: ; %entry
6051; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6052; GFX7-NEXT:    s_mov_b32 m0, -1
6053; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6054; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6055; GFX7-NEXT:    v_mov_b32_e32 v1, s2
6056; GFX7-NEXT:    v_mov_b32_e32 v2, s1
6057; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6058; GFX7-NEXT:    s_endpgm
6059;
6060; GFX10-WGP-LABEL: local_singlethread_one_as_monotonic_seq_cst_cmpxchg:
6061; GFX10-WGP:       ; %bb.0: ; %entry
6062; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6063; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6064; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6065; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
6066; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
6067; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6068; GFX10-WGP-NEXT:    s_endpgm
6069;
6070; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_seq_cst_cmpxchg:
6071; GFX10-CU:       ; %bb.0: ; %entry
6072; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6073; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6074; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6075; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
6076; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
6077; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6078; GFX10-CU-NEXT:    s_endpgm
6079;
6080; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_monotonic_seq_cst_cmpxchg:
6081; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6082; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6083; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
6084; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6085; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6086; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
6087; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
6088; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6089; SKIP-CACHE-INV-NEXT:    s_endpgm
6090;
6091; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_seq_cst_cmpxchg:
6092; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6093; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6094; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6095; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6096; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6097; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
6098; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6099; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6100;
6101; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_seq_cst_cmpxchg:
6102; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6103; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6104; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6105; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6106; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6107; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
6108; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6109; GFX90A-TGSPLIT-NEXT:    s_endpgm
6110    i32 addrspace(3)* %out, i32 %in, i32 %old) {
6111entry:
6112  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
6113  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic seq_cst
6114  ret void
6115}
6116
6117define amdgpu_kernel void @local_singlethread_one_as_acquire_seq_cst_cmpxchg(
6118; GFX6-LABEL: local_singlethread_one_as_acquire_seq_cst_cmpxchg:
6119; GFX6:       ; %bb.0: ; %entry
6120; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
6121; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
6122; GFX6-NEXT:    s_mov_b32 m0, -1
6123; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6124; GFX6-NEXT:    v_mov_b32_e32 v0, s2
6125; GFX6-NEXT:    v_mov_b32_e32 v1, s1
6126; GFX6-NEXT:    v_mov_b32_e32 v2, s0
6127; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6128; GFX6-NEXT:    s_endpgm
6129;
6130; GFX7-LABEL: local_singlethread_one_as_acquire_seq_cst_cmpxchg:
6131; GFX7:       ; %bb.0: ; %entry
6132; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6133; GFX7-NEXT:    s_mov_b32 m0, -1
6134; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6135; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6136; GFX7-NEXT:    v_mov_b32_e32 v1, s2
6137; GFX7-NEXT:    v_mov_b32_e32 v2, s1
6138; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6139; GFX7-NEXT:    s_endpgm
6140;
6141; GFX10-WGP-LABEL: local_singlethread_one_as_acquire_seq_cst_cmpxchg:
6142; GFX10-WGP:       ; %bb.0: ; %entry
6143; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6144; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6145; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6146; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
6147; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
6148; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6149; GFX10-WGP-NEXT:    s_endpgm
6150;
6151; GFX10-CU-LABEL: local_singlethread_one_as_acquire_seq_cst_cmpxchg:
6152; GFX10-CU:       ; %bb.0: ; %entry
6153; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6154; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6155; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6156; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
6157; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
6158; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6159; GFX10-CU-NEXT:    s_endpgm
6160;
6161; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acquire_seq_cst_cmpxchg:
6162; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6163; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6164; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
6165; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6166; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6167; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
6168; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
6169; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6170; SKIP-CACHE-INV-NEXT:    s_endpgm
6171;
6172; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_seq_cst_cmpxchg:
6173; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6174; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6175; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6176; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6177; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6178; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
6179; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6180; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6181;
6182; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acquire_seq_cst_cmpxchg:
6183; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6184; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6185; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6186; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6187; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6188; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
6189; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6190; GFX90A-TGSPLIT-NEXT:    s_endpgm
6191    i32 addrspace(3)* %out, i32 %in, i32 %old) {
6192entry:
6193  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
6194  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire seq_cst
6195  ret void
6196}
6197
6198define amdgpu_kernel void @local_singlethread_one_as_release_seq_cst_cmpxchg(
6199; GFX6-LABEL: local_singlethread_one_as_release_seq_cst_cmpxchg:
6200; GFX6:       ; %bb.0: ; %entry
6201; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
6202; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
6203; GFX6-NEXT:    s_mov_b32 m0, -1
6204; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6205; GFX6-NEXT:    v_mov_b32_e32 v0, s2
6206; GFX6-NEXT:    v_mov_b32_e32 v1, s1
6207; GFX6-NEXT:    v_mov_b32_e32 v2, s0
6208; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6209; GFX6-NEXT:    s_endpgm
6210;
6211; GFX7-LABEL: local_singlethread_one_as_release_seq_cst_cmpxchg:
6212; GFX7:       ; %bb.0: ; %entry
6213; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6214; GFX7-NEXT:    s_mov_b32 m0, -1
6215; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6216; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6217; GFX7-NEXT:    v_mov_b32_e32 v1, s2
6218; GFX7-NEXT:    v_mov_b32_e32 v2, s1
6219; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6220; GFX7-NEXT:    s_endpgm
6221;
6222; GFX10-WGP-LABEL: local_singlethread_one_as_release_seq_cst_cmpxchg:
6223; GFX10-WGP:       ; %bb.0: ; %entry
6224; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6225; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6226; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6227; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
6228; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
6229; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6230; GFX10-WGP-NEXT:    s_endpgm
6231;
6232; GFX10-CU-LABEL: local_singlethread_one_as_release_seq_cst_cmpxchg:
6233; GFX10-CU:       ; %bb.0: ; %entry
6234; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6235; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6236; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6237; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
6238; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
6239; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6240; GFX10-CU-NEXT:    s_endpgm
6241;
6242; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_release_seq_cst_cmpxchg:
6243; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6244; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6245; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
6246; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6247; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6248; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
6249; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
6250; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6251; SKIP-CACHE-INV-NEXT:    s_endpgm
6252;
6253; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_seq_cst_cmpxchg:
6254; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6255; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6256; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6257; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6258; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6259; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
6260; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6261; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6262;
6263; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_release_seq_cst_cmpxchg:
6264; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6265; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6266; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6267; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6268; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6269; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
6270; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6271; GFX90A-TGSPLIT-NEXT:    s_endpgm
6272    i32 addrspace(3)* %out, i32 %in, i32 %old) {
6273entry:
6274  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
6275  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release seq_cst
6276  ret void
6277}
6278
6279define amdgpu_kernel void @local_singlethread_one_as_acq_rel_seq_cst_cmpxchg(
6280; GFX6-LABEL: local_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
6281; GFX6:       ; %bb.0: ; %entry
6282; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
6283; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
6284; GFX6-NEXT:    s_mov_b32 m0, -1
6285; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6286; GFX6-NEXT:    v_mov_b32_e32 v0, s2
6287; GFX6-NEXT:    v_mov_b32_e32 v1, s1
6288; GFX6-NEXT:    v_mov_b32_e32 v2, s0
6289; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6290; GFX6-NEXT:    s_endpgm
6291;
6292; GFX7-LABEL: local_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
6293; GFX7:       ; %bb.0: ; %entry
6294; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6295; GFX7-NEXT:    s_mov_b32 m0, -1
6296; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6297; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6298; GFX7-NEXT:    v_mov_b32_e32 v1, s2
6299; GFX7-NEXT:    v_mov_b32_e32 v2, s1
6300; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6301; GFX7-NEXT:    s_endpgm
6302;
6303; GFX10-WGP-LABEL: local_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
6304; GFX10-WGP:       ; %bb.0: ; %entry
6305; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6306; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6307; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6308; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
6309; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
6310; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6311; GFX10-WGP-NEXT:    s_endpgm
6312;
6313; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
6314; GFX10-CU:       ; %bb.0: ; %entry
6315; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6316; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6317; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6318; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
6319; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
6320; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6321; GFX10-CU-NEXT:    s_endpgm
6322;
6323; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
6324; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6325; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6326; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
6327; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6328; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6329; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
6330; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
6331; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6332; SKIP-CACHE-INV-NEXT:    s_endpgm
6333;
6334; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
6335; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6336; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6337; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6338; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6339; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6340; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
6341; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6342; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6343;
6344; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
6345; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6346; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6347; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6348; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6349; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6350; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
6351; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6352; GFX90A-TGSPLIT-NEXT:    s_endpgm
6353    i32 addrspace(3)* %out, i32 %in, i32 %old) {
6354entry:
6355  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
6356  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel seq_cst
6357  ret void
6358}
6359
6360define amdgpu_kernel void @local_singlethread_one_as_seq_cst_seq_cst_cmpxchg(
6361; GFX6-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
6362; GFX6:       ; %bb.0: ; %entry
6363; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
6364; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
6365; GFX6-NEXT:    s_mov_b32 m0, -1
6366; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6367; GFX6-NEXT:    v_mov_b32_e32 v0, s2
6368; GFX6-NEXT:    v_mov_b32_e32 v1, s1
6369; GFX6-NEXT:    v_mov_b32_e32 v2, s0
6370; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6371; GFX6-NEXT:    s_endpgm
6372;
6373; GFX7-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
6374; GFX7:       ; %bb.0: ; %entry
6375; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6376; GFX7-NEXT:    s_mov_b32 m0, -1
6377; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6378; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6379; GFX7-NEXT:    v_mov_b32_e32 v1, s2
6380; GFX7-NEXT:    v_mov_b32_e32 v2, s1
6381; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6382; GFX7-NEXT:    s_endpgm
6383;
6384; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
6385; GFX10-WGP:       ; %bb.0: ; %entry
6386; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6387; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6388; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6389; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
6390; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
6391; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6392; GFX10-WGP-NEXT:    s_endpgm
6393;
6394; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
6395; GFX10-CU:       ; %bb.0: ; %entry
6396; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6397; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6398; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6399; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
6400; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
6401; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6402; GFX10-CU-NEXT:    s_endpgm
6403;
6404; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
6405; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6406; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6407; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
6408; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6409; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6410; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
6411; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
6412; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6413; SKIP-CACHE-INV-NEXT:    s_endpgm
6414;
6415; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
6416; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6417; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6418; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6419; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6420; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6421; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
6422; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6423; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6424;
6425; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
6426; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6427; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6428; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6429; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6430; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6431; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
6432; GFX90A-TGSPLIT-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
6433; GFX90A-TGSPLIT-NEXT:    s_endpgm
6434    i32 addrspace(3)* %out, i32 %in, i32 %old) {
6435entry:
6436  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
6437  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst seq_cst
6438  ret void
6439}
6440
6441define amdgpu_kernel void @local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg(
6442; GFX6-LABEL: local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
6443; GFX6:       ; %bb.0: ; %entry
6444; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
6445; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
6446; GFX6-NEXT:    s_mov_b32 m0, -1
6447; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6448; GFX6-NEXT:    v_mov_b32_e32 v0, s2
6449; GFX6-NEXT:    v_mov_b32_e32 v1, s1
6450; GFX6-NEXT:    v_mov_b32_e32 v2, s0
6451; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6452; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6453; GFX6-NEXT:    ds_write_b32 v0, v1
6454; GFX6-NEXT:    s_endpgm
6455;
6456; GFX7-LABEL: local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
6457; GFX7:       ; %bb.0: ; %entry
6458; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6459; GFX7-NEXT:    s_mov_b32 m0, -1
6460; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6461; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6462; GFX7-NEXT:    v_mov_b32_e32 v1, s2
6463; GFX7-NEXT:    v_mov_b32_e32 v2, s1
6464; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6465; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6466; GFX7-NEXT:    ds_write_b32 v0, v1
6467; GFX7-NEXT:    s_endpgm
6468;
6469; GFX10-WGP-LABEL: local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
6470; GFX10-WGP:       ; %bb.0: ; %entry
6471; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6472; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6473; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6474; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
6475; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
6476; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6477; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6478; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
6479; GFX10-WGP-NEXT:    s_endpgm
6480;
6481; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
6482; GFX10-CU:       ; %bb.0: ; %entry
6483; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6484; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6485; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6486; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
6487; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
6488; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6489; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6490; GFX10-CU-NEXT:    ds_write_b32 v0, v1
6491; GFX10-CU-NEXT:    s_endpgm
6492;
6493; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
6494; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6495; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6496; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
6497; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6498; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6499; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
6500; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
6501; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6502; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6503; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
6504; SKIP-CACHE-INV-NEXT:    s_endpgm
6505;
6506; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
6507; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6508; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6509; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6510; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6511; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6512; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
6513; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6514; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6515; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
6516; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6517;
6518; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
6519; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6520; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6521; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6522; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6523; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6524; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
6525; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6526; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6527; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
6528; GFX90A-TGSPLIT-NEXT:    s_endpgm
6529    i32 addrspace(3)* %out, i32 %in, i32 %old) {
6530entry:
6531  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
6532  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic monotonic
6533  %val0 = extractvalue { i32, i1 } %val, 0
6534  store i32 %val0, i32 addrspace(3)* %out, align 4
6535  ret void
6536}
6537
6538define amdgpu_kernel void @local_singlethread_one_as_acquire_monotonic_ret_cmpxchg(
6539; GFX6-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
6540; GFX6:       ; %bb.0: ; %entry
6541; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
6542; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
6543; GFX6-NEXT:    s_mov_b32 m0, -1
6544; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6545; GFX6-NEXT:    v_mov_b32_e32 v0, s2
6546; GFX6-NEXT:    v_mov_b32_e32 v1, s1
6547; GFX6-NEXT:    v_mov_b32_e32 v2, s0
6548; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6549; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6550; GFX6-NEXT:    ds_write_b32 v0, v1
6551; GFX6-NEXT:    s_endpgm
6552;
6553; GFX7-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
6554; GFX7:       ; %bb.0: ; %entry
6555; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6556; GFX7-NEXT:    s_mov_b32 m0, -1
6557; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6558; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6559; GFX7-NEXT:    v_mov_b32_e32 v1, s2
6560; GFX7-NEXT:    v_mov_b32_e32 v2, s1
6561; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6562; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6563; GFX7-NEXT:    ds_write_b32 v0, v1
6564; GFX7-NEXT:    s_endpgm
6565;
6566; GFX10-WGP-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
6567; GFX10-WGP:       ; %bb.0: ; %entry
6568; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6569; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6570; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6571; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
6572; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
6573; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6574; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6575; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
6576; GFX10-WGP-NEXT:    s_endpgm
6577;
6578; GFX10-CU-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
6579; GFX10-CU:       ; %bb.0: ; %entry
6580; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6581; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6582; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6583; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
6584; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
6585; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6586; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6587; GFX10-CU-NEXT:    ds_write_b32 v0, v1
6588; GFX10-CU-NEXT:    s_endpgm
6589;
6590; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
6591; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6592; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6593; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
6594; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6595; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6596; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
6597; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
6598; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6599; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6600; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
6601; SKIP-CACHE-INV-NEXT:    s_endpgm
6602;
6603; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
6604; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6605; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6606; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6607; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6608; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6609; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
6610; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6611; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6612; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
6613; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6614;
6615; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
6616; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6617; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6618; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6619; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6620; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6621; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
6622; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6623; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6624; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
6625; GFX90A-TGSPLIT-NEXT:    s_endpgm
6626    i32 addrspace(3)* %out, i32 %in, i32 %old) {
6627entry:
6628  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
6629  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire monotonic
6630  %val0 = extractvalue { i32, i1 } %val, 0
6631  store i32 %val0, i32 addrspace(3)* %out, align 4
6632  ret void
6633}
6634
6635define amdgpu_kernel void @local_singlethread_one_as_release_monotonic_ret_cmpxchg(
6636; GFX6-LABEL: local_singlethread_one_as_release_monotonic_ret_cmpxchg:
6637; GFX6:       ; %bb.0: ; %entry
6638; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
6639; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
6640; GFX6-NEXT:    s_mov_b32 m0, -1
6641; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6642; GFX6-NEXT:    v_mov_b32_e32 v0, s2
6643; GFX6-NEXT:    v_mov_b32_e32 v1, s1
6644; GFX6-NEXT:    v_mov_b32_e32 v2, s0
6645; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6646; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6647; GFX6-NEXT:    ds_write_b32 v0, v1
6648; GFX6-NEXT:    s_endpgm
6649;
6650; GFX7-LABEL: local_singlethread_one_as_release_monotonic_ret_cmpxchg:
6651; GFX7:       ; %bb.0: ; %entry
6652; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6653; GFX7-NEXT:    s_mov_b32 m0, -1
6654; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6655; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6656; GFX7-NEXT:    v_mov_b32_e32 v1, s2
6657; GFX7-NEXT:    v_mov_b32_e32 v2, s1
6658; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6659; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6660; GFX7-NEXT:    ds_write_b32 v0, v1
6661; GFX7-NEXT:    s_endpgm
6662;
6663; GFX10-WGP-LABEL: local_singlethread_one_as_release_monotonic_ret_cmpxchg:
6664; GFX10-WGP:       ; %bb.0: ; %entry
6665; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6666; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6667; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6668; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
6669; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
6670; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6671; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6672; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
6673; GFX10-WGP-NEXT:    s_endpgm
6674;
6675; GFX10-CU-LABEL: local_singlethread_one_as_release_monotonic_ret_cmpxchg:
6676; GFX10-CU:       ; %bb.0: ; %entry
6677; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6678; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6679; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6680; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
6681; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
6682; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6683; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6684; GFX10-CU-NEXT:    ds_write_b32 v0, v1
6685; GFX10-CU-NEXT:    s_endpgm
6686;
6687; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_release_monotonic_ret_cmpxchg:
6688; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6689; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6690; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
6691; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6692; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6693; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
6694; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
6695; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6696; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6697; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
6698; SKIP-CACHE-INV-NEXT:    s_endpgm
6699;
6700; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_monotonic_ret_cmpxchg:
6701; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6702; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6703; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6704; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6705; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6706; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
6707; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6708; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6709; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
6710; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6711;
6712; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_release_monotonic_ret_cmpxchg:
6713; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6714; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6715; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6716; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6717; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6718; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
6719; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6720; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6721; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
6722; GFX90A-TGSPLIT-NEXT:    s_endpgm
6723    i32 addrspace(3)* %out, i32 %in, i32 %old) {
6724entry:
6725  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
6726  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release monotonic
6727  %val0 = extractvalue { i32, i1 } %val, 0
6728  store i32 %val0, i32 addrspace(3)* %out, align 4
6729  ret void
6730}
6731
6732define amdgpu_kernel void @local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg(
6733; GFX6-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
6734; GFX6:       ; %bb.0: ; %entry
6735; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
6736; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
6737; GFX6-NEXT:    s_mov_b32 m0, -1
6738; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6739; GFX6-NEXT:    v_mov_b32_e32 v0, s2
6740; GFX6-NEXT:    v_mov_b32_e32 v1, s1
6741; GFX6-NEXT:    v_mov_b32_e32 v2, s0
6742; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6743; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6744; GFX6-NEXT:    ds_write_b32 v0, v1
6745; GFX6-NEXT:    s_endpgm
6746;
6747; GFX7-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
6748; GFX7:       ; %bb.0: ; %entry
6749; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6750; GFX7-NEXT:    s_mov_b32 m0, -1
6751; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6752; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6753; GFX7-NEXT:    v_mov_b32_e32 v1, s2
6754; GFX7-NEXT:    v_mov_b32_e32 v2, s1
6755; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6756; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6757; GFX7-NEXT:    ds_write_b32 v0, v1
6758; GFX7-NEXT:    s_endpgm
6759;
6760; GFX10-WGP-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
6761; GFX10-WGP:       ; %bb.0: ; %entry
6762; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6763; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6764; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6765; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
6766; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
6767; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6768; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6769; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
6770; GFX10-WGP-NEXT:    s_endpgm
6771;
6772; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
6773; GFX10-CU:       ; %bb.0: ; %entry
6774; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6775; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6776; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6777; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
6778; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
6779; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6780; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6781; GFX10-CU-NEXT:    ds_write_b32 v0, v1
6782; GFX10-CU-NEXT:    s_endpgm
6783;
6784; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
6785; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6786; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6787; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
6788; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6789; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6790; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
6791; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
6792; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6793; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6794; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
6795; SKIP-CACHE-INV-NEXT:    s_endpgm
6796;
6797; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
6798; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6799; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6800; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6801; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6802; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6803; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
6804; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6805; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6806; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
6807; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6808;
6809; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
6810; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6811; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6812; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6813; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6814; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6815; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
6816; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6817; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6818; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
6819; GFX90A-TGSPLIT-NEXT:    s_endpgm
6820    i32 addrspace(3)* %out, i32 %in, i32 %old) {
6821entry:
6822  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
6823  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel monotonic
6824  %val0 = extractvalue { i32, i1 } %val, 0
6825  store i32 %val0, i32 addrspace(3)* %out, align 4
6826  ret void
6827}
6828
6829define amdgpu_kernel void @local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg(
6830; GFX6-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
6831; GFX6:       ; %bb.0: ; %entry
6832; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
6833; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
6834; GFX6-NEXT:    s_mov_b32 m0, -1
6835; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6836; GFX6-NEXT:    v_mov_b32_e32 v0, s2
6837; GFX6-NEXT:    v_mov_b32_e32 v1, s1
6838; GFX6-NEXT:    v_mov_b32_e32 v2, s0
6839; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6840; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6841; GFX6-NEXT:    ds_write_b32 v0, v1
6842; GFX6-NEXT:    s_endpgm
6843;
6844; GFX7-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
6845; GFX7:       ; %bb.0: ; %entry
6846; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6847; GFX7-NEXT:    s_mov_b32 m0, -1
6848; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6849; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6850; GFX7-NEXT:    v_mov_b32_e32 v1, s2
6851; GFX7-NEXT:    v_mov_b32_e32 v2, s1
6852; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6853; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6854; GFX7-NEXT:    ds_write_b32 v0, v1
6855; GFX7-NEXT:    s_endpgm
6856;
6857; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
6858; GFX10-WGP:       ; %bb.0: ; %entry
6859; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6860; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6861; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6862; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
6863; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
6864; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6865; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6866; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
6867; GFX10-WGP-NEXT:    s_endpgm
6868;
6869; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
6870; GFX10-CU:       ; %bb.0: ; %entry
6871; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6872; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6873; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6874; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
6875; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
6876; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6877; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6878; GFX10-CU-NEXT:    ds_write_b32 v0, v1
6879; GFX10-CU-NEXT:    s_endpgm
6880;
6881; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
6882; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6883; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6884; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
6885; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6886; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6887; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
6888; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
6889; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6890; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6891; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
6892; SKIP-CACHE-INV-NEXT:    s_endpgm
6893;
6894; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
6895; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6896; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6897; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6898; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6899; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6900; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
6901; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6902; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6903; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
6904; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6905;
6906; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
6907; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6908; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6909; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6910; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6911; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6912; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
6913; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6914; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6915; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
6916; GFX90A-TGSPLIT-NEXT:    s_endpgm
6917    i32 addrspace(3)* %out, i32 %in, i32 %old) {
6918entry:
6919  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
6920  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst monotonic
6921  %val0 = extractvalue { i32, i1 } %val, 0
6922  store i32 %val0, i32 addrspace(3)* %out, align 4
6923  ret void
6924}
6925
6926define amdgpu_kernel void @local_singlethread_one_as_monotonic_acquire_ret_cmpxchg(
6927; GFX6-LABEL: local_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
6928; GFX6:       ; %bb.0: ; %entry
6929; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
6930; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
6931; GFX6-NEXT:    s_mov_b32 m0, -1
6932; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6933; GFX6-NEXT:    v_mov_b32_e32 v0, s2
6934; GFX6-NEXT:    v_mov_b32_e32 v1, s1
6935; GFX6-NEXT:    v_mov_b32_e32 v2, s0
6936; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6937; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6938; GFX6-NEXT:    ds_write_b32 v0, v1
6939; GFX6-NEXT:    s_endpgm
6940;
6941; GFX7-LABEL: local_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
6942; GFX7:       ; %bb.0: ; %entry
6943; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6944; GFX7-NEXT:    s_mov_b32 m0, -1
6945; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6946; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6947; GFX7-NEXT:    v_mov_b32_e32 v1, s2
6948; GFX7-NEXT:    v_mov_b32_e32 v2, s1
6949; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6950; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6951; GFX7-NEXT:    ds_write_b32 v0, v1
6952; GFX7-NEXT:    s_endpgm
6953;
6954; GFX10-WGP-LABEL: local_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
6955; GFX10-WGP:       ; %bb.0: ; %entry
6956; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6957; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6958; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6959; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
6960; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
6961; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6962; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6963; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
6964; GFX10-WGP-NEXT:    s_endpgm
6965;
6966; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
6967; GFX10-CU:       ; %bb.0: ; %entry
6968; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6969; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6970; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6971; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
6972; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
6973; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6974; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6975; GFX10-CU-NEXT:    ds_write_b32 v0, v1
6976; GFX10-CU-NEXT:    s_endpgm
6977;
6978; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
6979; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6980; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6981; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
6982; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6983; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6984; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
6985; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
6986; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6987; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6988; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
6989; SKIP-CACHE-INV-NEXT:    s_endpgm
6990;
6991; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
6992; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6993; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6994; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6995; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6996; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6997; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
6998; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
6999; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7000; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
7001; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7002;
7003; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
7004; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7005; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7006; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7007; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7008; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7009; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
7010; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7011; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7012; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
7013; GFX90A-TGSPLIT-NEXT:    s_endpgm
7014    i32 addrspace(3)* %out, i32 %in, i32 %old) {
7015entry:
7016  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
7017  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic acquire
7018  %val0 = extractvalue { i32, i1 } %val, 0
7019  store i32 %val0, i32 addrspace(3)* %out, align 4
7020  ret void
7021}
7022
7023define amdgpu_kernel void @local_singlethread_one_as_acquire_acquire_ret_cmpxchg(
7024; GFX6-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
7025; GFX6:       ; %bb.0: ; %entry
7026; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
7027; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
7028; GFX6-NEXT:    s_mov_b32 m0, -1
7029; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7030; GFX6-NEXT:    v_mov_b32_e32 v0, s2
7031; GFX6-NEXT:    v_mov_b32_e32 v1, s1
7032; GFX6-NEXT:    v_mov_b32_e32 v2, s0
7033; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7034; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7035; GFX6-NEXT:    ds_write_b32 v0, v1
7036; GFX6-NEXT:    s_endpgm
7037;
7038; GFX7-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
7039; GFX7:       ; %bb.0: ; %entry
7040; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7041; GFX7-NEXT:    s_mov_b32 m0, -1
7042; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7043; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7044; GFX7-NEXT:    v_mov_b32_e32 v1, s2
7045; GFX7-NEXT:    v_mov_b32_e32 v2, s1
7046; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7047; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7048; GFX7-NEXT:    ds_write_b32 v0, v1
7049; GFX7-NEXT:    s_endpgm
7050;
7051; GFX10-WGP-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
7052; GFX10-WGP:       ; %bb.0: ; %entry
7053; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7054; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7055; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7056; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
7057; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
7058; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7059; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7060; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
7061; GFX10-WGP-NEXT:    s_endpgm
7062;
7063; GFX10-CU-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
7064; GFX10-CU:       ; %bb.0: ; %entry
7065; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7066; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7067; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7068; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
7069; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
7070; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7071; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7072; GFX10-CU-NEXT:    ds_write_b32 v0, v1
7073; GFX10-CU-NEXT:    s_endpgm
7074;
7075; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
7076; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7077; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7078; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
7079; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7080; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7081; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
7082; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
7083; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7084; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7085; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
7086; SKIP-CACHE-INV-NEXT:    s_endpgm
7087;
7088; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
7089; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7090; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7091; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7092; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7093; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7094; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
7095; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7096; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7097; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
7098; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7099;
7100; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
7101; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7102; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7103; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7104; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7105; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7106; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
7107; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7108; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7109; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
7110; GFX90A-TGSPLIT-NEXT:    s_endpgm
7111    i32 addrspace(3)* %out, i32 %in, i32 %old) {
7112entry:
7113  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
7114  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire acquire
7115  %val0 = extractvalue { i32, i1 } %val, 0
7116  store i32 %val0, i32 addrspace(3)* %out, align 4
7117  ret void
7118}
7119
7120define amdgpu_kernel void @local_singlethread_one_as_release_acquire_ret_cmpxchg(
7121; GFX6-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
7122; GFX6:       ; %bb.0: ; %entry
7123; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
7124; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
7125; GFX6-NEXT:    s_mov_b32 m0, -1
7126; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7127; GFX6-NEXT:    v_mov_b32_e32 v0, s2
7128; GFX6-NEXT:    v_mov_b32_e32 v1, s1
7129; GFX6-NEXT:    v_mov_b32_e32 v2, s0
7130; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7131; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7132; GFX6-NEXT:    ds_write_b32 v0, v1
7133; GFX6-NEXT:    s_endpgm
7134;
7135; GFX7-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
7136; GFX7:       ; %bb.0: ; %entry
7137; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7138; GFX7-NEXT:    s_mov_b32 m0, -1
7139; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7140; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7141; GFX7-NEXT:    v_mov_b32_e32 v1, s2
7142; GFX7-NEXT:    v_mov_b32_e32 v2, s1
7143; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7144; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7145; GFX7-NEXT:    ds_write_b32 v0, v1
7146; GFX7-NEXT:    s_endpgm
7147;
7148; GFX10-WGP-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
7149; GFX10-WGP:       ; %bb.0: ; %entry
7150; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7151; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7152; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7153; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
7154; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
7155; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7156; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7157; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
7158; GFX10-WGP-NEXT:    s_endpgm
7159;
7160; GFX10-CU-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
7161; GFX10-CU:       ; %bb.0: ; %entry
7162; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7163; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7164; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7165; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
7166; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
7167; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7168; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7169; GFX10-CU-NEXT:    ds_write_b32 v0, v1
7170; GFX10-CU-NEXT:    s_endpgm
7171;
7172; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
7173; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7174; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7175; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
7176; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7177; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7178; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
7179; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
7180; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7181; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7182; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
7183; SKIP-CACHE-INV-NEXT:    s_endpgm
7184;
7185; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
7186; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7187; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7188; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7189; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7190; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7191; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
7192; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7193; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7194; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
7195; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7196;
7197; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
7198; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7199; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7200; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7201; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7202; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7203; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
7204; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7205; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7206; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
7207; GFX90A-TGSPLIT-NEXT:    s_endpgm
7208    i32 addrspace(3)* %out, i32 %in, i32 %old) {
7209entry:
7210  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
7211  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release acquire
7212  %val0 = extractvalue { i32, i1 } %val, 0
7213  store i32 %val0, i32 addrspace(3)* %out, align 4
7214  ret void
7215}
7216
7217define amdgpu_kernel void @local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg(
7218; GFX6-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
7219; GFX6:       ; %bb.0: ; %entry
7220; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
7221; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
7222; GFX6-NEXT:    s_mov_b32 m0, -1
7223; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7224; GFX6-NEXT:    v_mov_b32_e32 v0, s2
7225; GFX6-NEXT:    v_mov_b32_e32 v1, s1
7226; GFX6-NEXT:    v_mov_b32_e32 v2, s0
7227; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7228; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7229; GFX6-NEXT:    ds_write_b32 v0, v1
7230; GFX6-NEXT:    s_endpgm
7231;
7232; GFX7-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
7233; GFX7:       ; %bb.0: ; %entry
7234; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7235; GFX7-NEXT:    s_mov_b32 m0, -1
7236; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7237; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7238; GFX7-NEXT:    v_mov_b32_e32 v1, s2
7239; GFX7-NEXT:    v_mov_b32_e32 v2, s1
7240; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7241; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7242; GFX7-NEXT:    ds_write_b32 v0, v1
7243; GFX7-NEXT:    s_endpgm
7244;
7245; GFX10-WGP-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
7246; GFX10-WGP:       ; %bb.0: ; %entry
7247; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7248; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7249; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7250; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
7251; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
7252; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7253; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7254; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
7255; GFX10-WGP-NEXT:    s_endpgm
7256;
7257; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
7258; GFX10-CU:       ; %bb.0: ; %entry
7259; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7260; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7261; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7262; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
7263; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
7264; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7265; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7266; GFX10-CU-NEXT:    ds_write_b32 v0, v1
7267; GFX10-CU-NEXT:    s_endpgm
7268;
7269; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
7270; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7271; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7272; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
7273; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7274; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7275; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
7276; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
7277; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7278; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7279; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
7280; SKIP-CACHE-INV-NEXT:    s_endpgm
7281;
7282; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
7283; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7284; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7285; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7286; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7287; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7288; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
7289; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7290; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7291; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
7292; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7293;
7294; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
7295; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7296; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7297; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7298; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7299; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7300; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
7301; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7302; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7303; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
7304; GFX90A-TGSPLIT-NEXT:    s_endpgm
7305    i32 addrspace(3)* %out, i32 %in, i32 %old) {
7306entry:
7307  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
7308  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel acquire
7309  %val0 = extractvalue { i32, i1 } %val, 0
7310  store i32 %val0, i32 addrspace(3)* %out, align 4
7311  ret void
7312}
7313
7314define amdgpu_kernel void @local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg(
7315; GFX6-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
7316; GFX6:       ; %bb.0: ; %entry
7317; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
7318; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
7319; GFX6-NEXT:    s_mov_b32 m0, -1
7320; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7321; GFX6-NEXT:    v_mov_b32_e32 v0, s2
7322; GFX6-NEXT:    v_mov_b32_e32 v1, s1
7323; GFX6-NEXT:    v_mov_b32_e32 v2, s0
7324; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7325; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7326; GFX6-NEXT:    ds_write_b32 v0, v1
7327; GFX6-NEXT:    s_endpgm
7328;
7329; GFX7-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
7330; GFX7:       ; %bb.0: ; %entry
7331; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7332; GFX7-NEXT:    s_mov_b32 m0, -1
7333; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7334; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7335; GFX7-NEXT:    v_mov_b32_e32 v1, s2
7336; GFX7-NEXT:    v_mov_b32_e32 v2, s1
7337; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7338; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7339; GFX7-NEXT:    ds_write_b32 v0, v1
7340; GFX7-NEXT:    s_endpgm
7341;
7342; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
7343; GFX10-WGP:       ; %bb.0: ; %entry
7344; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7345; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7346; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7347; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
7348; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
7349; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7350; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7351; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
7352; GFX10-WGP-NEXT:    s_endpgm
7353;
7354; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
7355; GFX10-CU:       ; %bb.0: ; %entry
7356; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7357; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7358; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7359; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
7360; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
7361; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7362; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7363; GFX10-CU-NEXT:    ds_write_b32 v0, v1
7364; GFX10-CU-NEXT:    s_endpgm
7365;
7366; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
7367; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7368; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7369; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
7370; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7371; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7372; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
7373; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
7374; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7375; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7376; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
7377; SKIP-CACHE-INV-NEXT:    s_endpgm
7378;
7379; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
7380; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7381; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7382; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7383; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7384; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7385; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
7386; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7387; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7388; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
7389; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7390;
7391; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
7392; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7393; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7394; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7395; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7396; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7397; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
7398; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7399; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7400; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
7401; GFX90A-TGSPLIT-NEXT:    s_endpgm
7402    i32 addrspace(3)* %out, i32 %in, i32 %old) {
7403entry:
7404  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
7405  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst acquire
7406  %val0 = extractvalue { i32, i1 } %val, 0
7407  store i32 %val0, i32 addrspace(3)* %out, align 4
7408  ret void
7409}
7410
7411define amdgpu_kernel void @local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg(
7412; GFX6-LABEL: local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
7413; GFX6:       ; %bb.0: ; %entry
7414; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
7415; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
7416; GFX6-NEXT:    s_mov_b32 m0, -1
7417; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7418; GFX6-NEXT:    v_mov_b32_e32 v0, s2
7419; GFX6-NEXT:    v_mov_b32_e32 v1, s1
7420; GFX6-NEXT:    v_mov_b32_e32 v2, s0
7421; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7422; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7423; GFX6-NEXT:    ds_write_b32 v0, v1
7424; GFX6-NEXT:    s_endpgm
7425;
7426; GFX7-LABEL: local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
7427; GFX7:       ; %bb.0: ; %entry
7428; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7429; GFX7-NEXT:    s_mov_b32 m0, -1
7430; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7431; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7432; GFX7-NEXT:    v_mov_b32_e32 v1, s2
7433; GFX7-NEXT:    v_mov_b32_e32 v2, s1
7434; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7435; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7436; GFX7-NEXT:    ds_write_b32 v0, v1
7437; GFX7-NEXT:    s_endpgm
7438;
7439; GFX10-WGP-LABEL: local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
7440; GFX10-WGP:       ; %bb.0: ; %entry
7441; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7442; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7443; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7444; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
7445; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
7446; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7447; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7448; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
7449; GFX10-WGP-NEXT:    s_endpgm
7450;
7451; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
7452; GFX10-CU:       ; %bb.0: ; %entry
7453; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7454; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7455; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7456; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
7457; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
7458; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7459; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7460; GFX10-CU-NEXT:    ds_write_b32 v0, v1
7461; GFX10-CU-NEXT:    s_endpgm
7462;
7463; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
7464; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7465; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7466; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
7467; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7468; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7469; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
7470; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
7471; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7472; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7473; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
7474; SKIP-CACHE-INV-NEXT:    s_endpgm
7475;
7476; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
7477; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7478; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7479; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7480; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7481; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7482; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
7483; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7484; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7485; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
7486; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7487;
7488; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
7489; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7490; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7491; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7492; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7493; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7494; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
7495; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7496; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7497; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
7498; GFX90A-TGSPLIT-NEXT:    s_endpgm
7499    i32 addrspace(3)* %out, i32 %in, i32 %old) {
7500entry:
7501  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
7502  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic seq_cst
7503  %val0 = extractvalue { i32, i1 } %val, 0
7504  store i32 %val0, i32 addrspace(3)* %out, align 4
7505  ret void
7506}
7507
7508define amdgpu_kernel void @local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg(
7509; GFX6-LABEL: local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
7510; GFX6:       ; %bb.0: ; %entry
7511; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
7512; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
7513; GFX6-NEXT:    s_mov_b32 m0, -1
7514; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7515; GFX6-NEXT:    v_mov_b32_e32 v0, s2
7516; GFX6-NEXT:    v_mov_b32_e32 v1, s1
7517; GFX6-NEXT:    v_mov_b32_e32 v2, s0
7518; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7519; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7520; GFX6-NEXT:    ds_write_b32 v0, v1
7521; GFX6-NEXT:    s_endpgm
7522;
7523; GFX7-LABEL: local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
7524; GFX7:       ; %bb.0: ; %entry
7525; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7526; GFX7-NEXT:    s_mov_b32 m0, -1
7527; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7528; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7529; GFX7-NEXT:    v_mov_b32_e32 v1, s2
7530; GFX7-NEXT:    v_mov_b32_e32 v2, s1
7531; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7532; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7533; GFX7-NEXT:    ds_write_b32 v0, v1
7534; GFX7-NEXT:    s_endpgm
7535;
7536; GFX10-WGP-LABEL: local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
7537; GFX10-WGP:       ; %bb.0: ; %entry
7538; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7539; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7540; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7541; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
7542; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
7543; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7544; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7545; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
7546; GFX10-WGP-NEXT:    s_endpgm
7547;
7548; GFX10-CU-LABEL: local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
7549; GFX10-CU:       ; %bb.0: ; %entry
7550; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7551; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7552; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7553; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
7554; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
7555; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7556; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7557; GFX10-CU-NEXT:    ds_write_b32 v0, v1
7558; GFX10-CU-NEXT:    s_endpgm
7559;
7560; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
7561; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7562; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7563; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
7564; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7565; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7566; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
7567; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
7568; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7569; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7570; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
7571; SKIP-CACHE-INV-NEXT:    s_endpgm
7572;
7573; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
7574; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7575; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7576; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7577; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7578; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7579; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
7580; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7581; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7582; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
7583; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7584;
7585; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
7586; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7587; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7588; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7589; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7590; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7591; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
7592; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7593; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7594; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
7595; GFX90A-TGSPLIT-NEXT:    s_endpgm
7596    i32 addrspace(3)* %out, i32 %in, i32 %old) {
7597entry:
7598  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
7599  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire seq_cst
7600  %val0 = extractvalue { i32, i1 } %val, 0
7601  store i32 %val0, i32 addrspace(3)* %out, align 4
7602  ret void
7603}
7604
7605define amdgpu_kernel void @local_singlethread_one_as_release_seq_cst_ret_cmpxchg(
7606; GFX6-LABEL: local_singlethread_one_as_release_seq_cst_ret_cmpxchg:
7607; GFX6:       ; %bb.0: ; %entry
7608; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
7609; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
7610; GFX6-NEXT:    s_mov_b32 m0, -1
7611; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7612; GFX6-NEXT:    v_mov_b32_e32 v0, s2
7613; GFX6-NEXT:    v_mov_b32_e32 v1, s1
7614; GFX6-NEXT:    v_mov_b32_e32 v2, s0
7615; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7616; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7617; GFX6-NEXT:    ds_write_b32 v0, v1
7618; GFX6-NEXT:    s_endpgm
7619;
7620; GFX7-LABEL: local_singlethread_one_as_release_seq_cst_ret_cmpxchg:
7621; GFX7:       ; %bb.0: ; %entry
7622; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7623; GFX7-NEXT:    s_mov_b32 m0, -1
7624; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7625; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7626; GFX7-NEXT:    v_mov_b32_e32 v1, s2
7627; GFX7-NEXT:    v_mov_b32_e32 v2, s1
7628; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7629; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7630; GFX7-NEXT:    ds_write_b32 v0, v1
7631; GFX7-NEXT:    s_endpgm
7632;
7633; GFX10-WGP-LABEL: local_singlethread_one_as_release_seq_cst_ret_cmpxchg:
7634; GFX10-WGP:       ; %bb.0: ; %entry
7635; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7636; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7637; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7638; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
7639; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
7640; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7641; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7642; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
7643; GFX10-WGP-NEXT:    s_endpgm
7644;
7645; GFX10-CU-LABEL: local_singlethread_one_as_release_seq_cst_ret_cmpxchg:
7646; GFX10-CU:       ; %bb.0: ; %entry
7647; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7648; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7649; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7650; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
7651; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
7652; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7653; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7654; GFX10-CU-NEXT:    ds_write_b32 v0, v1
7655; GFX10-CU-NEXT:    s_endpgm
7656;
7657; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_release_seq_cst_ret_cmpxchg:
7658; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7659; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7660; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
7661; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7662; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7663; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
7664; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
7665; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7666; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7667; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
7668; SKIP-CACHE-INV-NEXT:    s_endpgm
7669;
7670; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_release_seq_cst_ret_cmpxchg:
7671; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7672; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7673; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7674; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7675; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7676; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
7677; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7678; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7679; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
7680; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7681;
7682; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_release_seq_cst_ret_cmpxchg:
7683; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7684; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7685; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7686; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7687; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7688; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
7689; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7690; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7691; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
7692; GFX90A-TGSPLIT-NEXT:    s_endpgm
7693    i32 addrspace(3)* %out, i32 %in, i32 %old) {
7694entry:
7695  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
7696  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release seq_cst
7697  %val0 = extractvalue { i32, i1 } %val, 0
7698  store i32 %val0, i32 addrspace(3)* %out, align 4
7699  ret void
7700}
7701
7702define amdgpu_kernel void @local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg(
7703; GFX6-LABEL: local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
7704; GFX6:       ; %bb.0: ; %entry
7705; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
7706; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
7707; GFX6-NEXT:    s_mov_b32 m0, -1
7708; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7709; GFX6-NEXT:    v_mov_b32_e32 v0, s2
7710; GFX6-NEXT:    v_mov_b32_e32 v1, s1
7711; GFX6-NEXT:    v_mov_b32_e32 v2, s0
7712; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7713; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7714; GFX6-NEXT:    ds_write_b32 v0, v1
7715; GFX6-NEXT:    s_endpgm
7716;
7717; GFX7-LABEL: local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
7718; GFX7:       ; %bb.0: ; %entry
7719; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7720; GFX7-NEXT:    s_mov_b32 m0, -1
7721; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7722; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7723; GFX7-NEXT:    v_mov_b32_e32 v1, s2
7724; GFX7-NEXT:    v_mov_b32_e32 v2, s1
7725; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7726; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7727; GFX7-NEXT:    ds_write_b32 v0, v1
7728; GFX7-NEXT:    s_endpgm
7729;
7730; GFX10-WGP-LABEL: local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
7731; GFX10-WGP:       ; %bb.0: ; %entry
7732; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7733; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7734; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7735; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
7736; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
7737; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7738; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7739; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
7740; GFX10-WGP-NEXT:    s_endpgm
7741;
7742; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
7743; GFX10-CU:       ; %bb.0: ; %entry
7744; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7745; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7746; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7747; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
7748; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
7749; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7750; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7751; GFX10-CU-NEXT:    ds_write_b32 v0, v1
7752; GFX10-CU-NEXT:    s_endpgm
7753;
7754; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
7755; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7756; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7757; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
7758; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7759; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7760; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
7761; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
7762; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7763; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7764; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
7765; SKIP-CACHE-INV-NEXT:    s_endpgm
7766;
7767; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
7768; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7769; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7770; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7771; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7772; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7773; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
7774; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7775; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7776; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
7777; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7778;
7779; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
7780; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7781; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7782; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7783; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7784; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7785; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
7786; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7787; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7788; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
7789; GFX90A-TGSPLIT-NEXT:    s_endpgm
7790    i32 addrspace(3)* %out, i32 %in, i32 %old) {
7791entry:
7792  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
7793  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel seq_cst
7794  %val0 = extractvalue { i32, i1 } %val, 0
7795  store i32 %val0, i32 addrspace(3)* %out, align 4
7796  ret void
7797}
7798
7799define amdgpu_kernel void @local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg(
7800; GFX6-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
7801; GFX6:       ; %bb.0: ; %entry
7802; GFX6-NEXT:    s_load_dword s2, s[4:5], 0x0
7803; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x1
7804; GFX6-NEXT:    s_mov_b32 m0, -1
7805; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7806; GFX6-NEXT:    v_mov_b32_e32 v0, s2
7807; GFX6-NEXT:    v_mov_b32_e32 v1, s1
7808; GFX6-NEXT:    v_mov_b32_e32 v2, s0
7809; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7810; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7811; GFX6-NEXT:    ds_write_b32 v0, v1
7812; GFX6-NEXT:    s_endpgm
7813;
7814; GFX7-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
7815; GFX7:       ; %bb.0: ; %entry
7816; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7817; GFX7-NEXT:    s_mov_b32 m0, -1
7818; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7819; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7820; GFX7-NEXT:    v_mov_b32_e32 v1, s2
7821; GFX7-NEXT:    v_mov_b32_e32 v2, s1
7822; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7823; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7824; GFX7-NEXT:    ds_write_b32 v0, v1
7825; GFX7-NEXT:    s_endpgm
7826;
7827; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
7828; GFX10-WGP:       ; %bb.0: ; %entry
7829; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7830; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7831; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7832; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
7833; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
7834; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7835; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7836; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
7837; GFX10-WGP-NEXT:    s_endpgm
7838;
7839; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
7840; GFX10-CU:       ; %bb.0: ; %entry
7841; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7842; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7843; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7844; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
7845; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
7846; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7847; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7848; GFX10-CU-NEXT:    ds_write_b32 v0, v1
7849; GFX10-CU-NEXT:    s_endpgm
7850;
7851; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
7852; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7853; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7854; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
7855; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7856; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7857; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s2
7858; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
7859; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7860; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7861; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
7862; SKIP-CACHE-INV-NEXT:    s_endpgm
7863;
7864; GFX90A-NOTTGSPLIT-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
7865; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7866; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7867; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7868; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7869; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7870; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
7871; GFX90A-NOTTGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7872; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7873; GFX90A-NOTTGSPLIT-NEXT:    ds_write_b32 v0, v1
7874; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7875;
7876; GFX90A-TGSPLIT-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
7877; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7878; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7879; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7880; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7881; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7882; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s1
7883; GFX90A-TGSPLIT-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
7884; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7885; GFX90A-TGSPLIT-NEXT:    ds_write_b32 v0, v1
7886; GFX90A-TGSPLIT-NEXT:    s_endpgm
7887    i32 addrspace(3)* %out, i32 %in, i32 %old) {
7888entry:
7889  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
7890  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst seq_cst
7891  %val0 = extractvalue { i32, i1 } %val, 0
7892  store i32 %val0, i32 addrspace(3)* %out, align 4
7893  ret void
7894}
7895
7896