1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx600 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX6 %s
3; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
4; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
5; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
6; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
7; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-NOTTGSPLIT %s
8; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-TGSPLIT %s
9
10define amdgpu_kernel void @global_singlethread_unordered_load(
11; GFX6-LABEL: global_singlethread_unordered_load:
12; GFX6:       ; %bb.0: ; %entry
13; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
14; GFX6-NEXT:    s_mov_b32 s7, 0x100f000
15; GFX6-NEXT:    s_mov_b32 s6, -1
16; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
17; GFX6-NEXT:    s_mov_b32 s4, s0
18; GFX6-NEXT:    s_mov_b32 s5, s1
19; GFX6-NEXT:    buffer_load_dword v0, off, s[4:7], 0
20; GFX6-NEXT:    s_mov_b32 s4, s2
21; GFX6-NEXT:    s_mov_b32 s5, s3
22; GFX6-NEXT:    s_waitcnt vmcnt(0)
23; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
24; GFX6-NEXT:    s_endpgm
25;
26; GFX7-LABEL: global_singlethread_unordered_load:
27; GFX7:       ; %bb.0: ; %entry
28; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
29; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
30; GFX7-NEXT:    v_mov_b32_e32 v0, s0
31; GFX7-NEXT:    v_mov_b32_e32 v1, s1
32; GFX7-NEXT:    flat_load_dword v2, v[0:1]
33; GFX7-NEXT:    v_mov_b32_e32 v0, s2
34; GFX7-NEXT:    v_mov_b32_e32 v1, s3
35; GFX7-NEXT:    s_waitcnt vmcnt(0)
36; GFX7-NEXT:    flat_store_dword v[0:1], v2
37; GFX7-NEXT:    s_endpgm
38;
39; GFX10-WGP-LABEL: global_singlethread_unordered_load:
40; GFX10-WGP:       ; %bb.0: ; %entry
41; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
42; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
43; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
44; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
45; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
46; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
47; GFX10-WGP-NEXT:    s_endpgm
48;
49; GFX10-CU-LABEL: global_singlethread_unordered_load:
50; GFX10-CU:       ; %bb.0: ; %entry
51; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
52; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
53; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
54; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
55; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
56; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
57; GFX10-CU-NEXT:    s_endpgm
58;
59; SKIP-CACHE-INV-LABEL: global_singlethread_unordered_load:
60; SKIP-CACHE-INV:       ; %bb.0: ; %entry
61; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
62; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
63; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
64; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
65; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s0
66; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s1
67; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[4:7], 0
68; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s2
69; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s3
70; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
71; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
72; SKIP-CACHE-INV-NEXT:    s_endpgm
73;
74; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_unordered_load:
75; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
76; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
77; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
78; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
79; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
80; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
81; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
82; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
83;
84; GFX90A-TGSPLIT-LABEL: global_singlethread_unordered_load:
85; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
86; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
87; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
88; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
89; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
90; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
91; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
92; GFX90A-TGSPLIT-NEXT:    s_endpgm
93    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
94entry:
95  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread") unordered, align 4
96  store i32 %val, i32 addrspace(1)* %out
97  ret void
98}
99
100define amdgpu_kernel void @global_singlethread_monotonic_load(
101; GFX6-LABEL: global_singlethread_monotonic_load:
102; GFX6:       ; %bb.0: ; %entry
103; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
104; GFX6-NEXT:    s_mov_b32 s7, 0x100f000
105; GFX6-NEXT:    s_mov_b32 s6, -1
106; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
107; GFX6-NEXT:    s_mov_b32 s4, s0
108; GFX6-NEXT:    s_mov_b32 s5, s1
109; GFX6-NEXT:    buffer_load_dword v0, off, s[4:7], 0
110; GFX6-NEXT:    s_mov_b32 s4, s2
111; GFX6-NEXT:    s_mov_b32 s5, s3
112; GFX6-NEXT:    s_waitcnt vmcnt(0)
113; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
114; GFX6-NEXT:    s_endpgm
115;
116; GFX7-LABEL: global_singlethread_monotonic_load:
117; GFX7:       ; %bb.0: ; %entry
118; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
119; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
120; GFX7-NEXT:    v_mov_b32_e32 v0, s0
121; GFX7-NEXT:    v_mov_b32_e32 v1, s1
122; GFX7-NEXT:    flat_load_dword v2, v[0:1]
123; GFX7-NEXT:    v_mov_b32_e32 v0, s2
124; GFX7-NEXT:    v_mov_b32_e32 v1, s3
125; GFX7-NEXT:    s_waitcnt vmcnt(0)
126; GFX7-NEXT:    flat_store_dword v[0:1], v2
127; GFX7-NEXT:    s_endpgm
128;
129; GFX10-WGP-LABEL: global_singlethread_monotonic_load:
130; GFX10-WGP:       ; %bb.0: ; %entry
131; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
132; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
133; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
134; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
135; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
136; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
137; GFX10-WGP-NEXT:    s_endpgm
138;
139; GFX10-CU-LABEL: global_singlethread_monotonic_load:
140; GFX10-CU:       ; %bb.0: ; %entry
141; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
142; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
143; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
144; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
145; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
146; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
147; GFX10-CU-NEXT:    s_endpgm
148;
149; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_load:
150; SKIP-CACHE-INV:       ; %bb.0: ; %entry
151; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
152; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
153; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
154; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
155; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s0
156; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s1
157; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[4:7], 0
158; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s2
159; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s3
160; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
161; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
162; SKIP-CACHE-INV-NEXT:    s_endpgm
163;
164; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_load:
165; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
166; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
167; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
168; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
169; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
170; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
171; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
172; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
173;
174; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_load:
175; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
176; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
177; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
178; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
179; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
180; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
181; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
182; GFX90A-TGSPLIT-NEXT:    s_endpgm
183    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
184entry:
185  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread") monotonic, align 4
186  store i32 %val, i32 addrspace(1)* %out
187  ret void
188}
189
190define amdgpu_kernel void @global_singlethread_acquire_load(
191; GFX6-LABEL: global_singlethread_acquire_load:
192; GFX6:       ; %bb.0: ; %entry
193; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
194; GFX6-NEXT:    s_mov_b32 s7, 0x100f000
195; GFX6-NEXT:    s_mov_b32 s6, -1
196; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
197; GFX6-NEXT:    s_mov_b32 s4, s0
198; GFX6-NEXT:    s_mov_b32 s5, s1
199; GFX6-NEXT:    buffer_load_dword v0, off, s[4:7], 0
200; GFX6-NEXT:    s_mov_b32 s4, s2
201; GFX6-NEXT:    s_mov_b32 s5, s3
202; GFX6-NEXT:    s_waitcnt vmcnt(0)
203; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
204; GFX6-NEXT:    s_endpgm
205;
206; GFX7-LABEL: global_singlethread_acquire_load:
207; GFX7:       ; %bb.0: ; %entry
208; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
209; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
210; GFX7-NEXT:    v_mov_b32_e32 v0, s0
211; GFX7-NEXT:    v_mov_b32_e32 v1, s1
212; GFX7-NEXT:    flat_load_dword v2, v[0:1]
213; GFX7-NEXT:    v_mov_b32_e32 v0, s2
214; GFX7-NEXT:    v_mov_b32_e32 v1, s3
215; GFX7-NEXT:    s_waitcnt vmcnt(0)
216; GFX7-NEXT:    flat_store_dword v[0:1], v2
217; GFX7-NEXT:    s_endpgm
218;
219; GFX10-WGP-LABEL: global_singlethread_acquire_load:
220; GFX10-WGP:       ; %bb.0: ; %entry
221; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
222; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
223; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
224; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
225; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
226; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
227; GFX10-WGP-NEXT:    s_endpgm
228;
229; GFX10-CU-LABEL: global_singlethread_acquire_load:
230; GFX10-CU:       ; %bb.0: ; %entry
231; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
232; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
233; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
234; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
235; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
236; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
237; GFX10-CU-NEXT:    s_endpgm
238;
239; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_load:
240; SKIP-CACHE-INV:       ; %bb.0: ; %entry
241; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
242; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
243; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
244; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
245; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s0
246; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s1
247; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[4:7], 0
248; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s2
249; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s3
250; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
251; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
252; SKIP-CACHE-INV-NEXT:    s_endpgm
253;
254; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_load:
255; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
256; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
257; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
258; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
259; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
260; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
261; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
262; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
263;
264; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_load:
265; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
266; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
267; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
268; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
269; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
270; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
271; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
272; GFX90A-TGSPLIT-NEXT:    s_endpgm
273    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
274entry:
275  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread") acquire, align 4
276  store i32 %val, i32 addrspace(1)* %out
277  ret void
278}
279
280define amdgpu_kernel void @global_singlethread_seq_cst_load(
281; GFX6-LABEL: global_singlethread_seq_cst_load:
282; GFX6:       ; %bb.0: ; %entry
283; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
284; GFX6-NEXT:    s_mov_b32 s7, 0x100f000
285; GFX6-NEXT:    s_mov_b32 s6, -1
286; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
287; GFX6-NEXT:    s_mov_b32 s4, s0
288; GFX6-NEXT:    s_mov_b32 s5, s1
289; GFX6-NEXT:    buffer_load_dword v0, off, s[4:7], 0
290; GFX6-NEXT:    s_mov_b32 s4, s2
291; GFX6-NEXT:    s_mov_b32 s5, s3
292; GFX6-NEXT:    s_waitcnt vmcnt(0)
293; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
294; GFX6-NEXT:    s_endpgm
295;
296; GFX7-LABEL: global_singlethread_seq_cst_load:
297; GFX7:       ; %bb.0: ; %entry
298; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
299; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
300; GFX7-NEXT:    v_mov_b32_e32 v0, s0
301; GFX7-NEXT:    v_mov_b32_e32 v1, s1
302; GFX7-NEXT:    flat_load_dword v2, v[0:1]
303; GFX7-NEXT:    v_mov_b32_e32 v0, s2
304; GFX7-NEXT:    v_mov_b32_e32 v1, s3
305; GFX7-NEXT:    s_waitcnt vmcnt(0)
306; GFX7-NEXT:    flat_store_dword v[0:1], v2
307; GFX7-NEXT:    s_endpgm
308;
309; GFX10-WGP-LABEL: global_singlethread_seq_cst_load:
310; GFX10-WGP:       ; %bb.0: ; %entry
311; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
312; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
313; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
314; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
315; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
316; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
317; GFX10-WGP-NEXT:    s_endpgm
318;
319; GFX10-CU-LABEL: global_singlethread_seq_cst_load:
320; GFX10-CU:       ; %bb.0: ; %entry
321; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
322; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
323; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
324; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
325; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
326; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
327; GFX10-CU-NEXT:    s_endpgm
328;
329; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_load:
330; SKIP-CACHE-INV:       ; %bb.0: ; %entry
331; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
332; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
333; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
334; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
335; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s0
336; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s1
337; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[4:7], 0
338; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s2
339; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s3
340; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
341; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
342; SKIP-CACHE-INV-NEXT:    s_endpgm
343;
344; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_load:
345; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
346; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
347; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
348; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
349; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
350; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
351; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
352; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
353;
354; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_load:
355; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
356; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
357; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
358; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
359; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
360; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
361; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
362; GFX90A-TGSPLIT-NEXT:    s_endpgm
363    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
364entry:
365  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread") seq_cst, align 4
366  store i32 %val, i32 addrspace(1)* %out
367  ret void
368}
369
370define amdgpu_kernel void @global_singlethread_unordered_store(
371; GFX6-LABEL: global_singlethread_unordered_store:
372; GFX6:       ; %bb.0: ; %entry
373; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
374; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
375; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
376; GFX6-NEXT:    s_mov_b32 s2, -1
377; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
378; GFX6-NEXT:    v_mov_b32_e32 v0, s6
379; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
380; GFX6-NEXT:    s_endpgm
381;
382; GFX7-LABEL: global_singlethread_unordered_store:
383; GFX7:       ; %bb.0: ; %entry
384; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
385; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
386; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
387; GFX7-NEXT:    v_mov_b32_e32 v0, s0
388; GFX7-NEXT:    v_mov_b32_e32 v1, s1
389; GFX7-NEXT:    v_mov_b32_e32 v2, s2
390; GFX7-NEXT:    flat_store_dword v[0:1], v2
391; GFX7-NEXT:    s_endpgm
392;
393; GFX10-WGP-LABEL: global_singlethread_unordered_store:
394; GFX10-WGP:       ; %bb.0: ; %entry
395; GFX10-WGP-NEXT:    s_clause 0x1
396; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
397; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
398; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
399; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
400; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
401; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
402; GFX10-WGP-NEXT:    s_endpgm
403;
404; GFX10-CU-LABEL: global_singlethread_unordered_store:
405; GFX10-CU:       ; %bb.0: ; %entry
406; GFX10-CU-NEXT:    s_clause 0x1
407; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
408; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
409; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
410; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
411; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
412; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
413; GFX10-CU-NEXT:    s_endpgm
414;
415; SKIP-CACHE-INV-LABEL: global_singlethread_unordered_store:
416; SKIP-CACHE-INV:       ; %bb.0: ; %entry
417; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x0
418; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
419; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
420; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
421; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
422; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
423; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
424; SKIP-CACHE-INV-NEXT:    s_endpgm
425;
426; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_unordered_store:
427; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
428; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
429; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
430; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
431; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
432; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
433; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
434; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
435;
436; GFX90A-TGSPLIT-LABEL: global_singlethread_unordered_store:
437; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
438; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
439; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
440; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
441; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
442; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
443; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
444; GFX90A-TGSPLIT-NEXT:    s_endpgm
445    i32 %in, i32 addrspace(1)* %out) {
446entry:
447  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread") unordered, align 4
448  ret void
449}
450
451define amdgpu_kernel void @global_singlethread_monotonic_store(
452; GFX6-LABEL: global_singlethread_monotonic_store:
453; GFX6:       ; %bb.0: ; %entry
454; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
455; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
456; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
457; GFX6-NEXT:    s_mov_b32 s2, -1
458; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
459; GFX6-NEXT:    v_mov_b32_e32 v0, s6
460; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
461; GFX6-NEXT:    s_endpgm
462;
463; GFX7-LABEL: global_singlethread_monotonic_store:
464; GFX7:       ; %bb.0: ; %entry
465; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
466; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
467; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
468; GFX7-NEXT:    v_mov_b32_e32 v0, s0
469; GFX7-NEXT:    v_mov_b32_e32 v1, s1
470; GFX7-NEXT:    v_mov_b32_e32 v2, s2
471; GFX7-NEXT:    flat_store_dword v[0:1], v2
472; GFX7-NEXT:    s_endpgm
473;
474; GFX10-WGP-LABEL: global_singlethread_monotonic_store:
475; GFX10-WGP:       ; %bb.0: ; %entry
476; GFX10-WGP-NEXT:    s_clause 0x1
477; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
478; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
479; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
480; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
481; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
482; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
483; GFX10-WGP-NEXT:    s_endpgm
484;
485; GFX10-CU-LABEL: global_singlethread_monotonic_store:
486; GFX10-CU:       ; %bb.0: ; %entry
487; GFX10-CU-NEXT:    s_clause 0x1
488; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
489; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
490; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
491; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
492; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
493; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
494; GFX10-CU-NEXT:    s_endpgm
495;
496; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_store:
497; SKIP-CACHE-INV:       ; %bb.0: ; %entry
498; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x0
499; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
500; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
501; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
502; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
503; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
504; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
505; SKIP-CACHE-INV-NEXT:    s_endpgm
506;
507; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_store:
508; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
509; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
510; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
511; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
512; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
513; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
514; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
515; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
516;
517; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_store:
518; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
519; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
520; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
521; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
522; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
523; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
524; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
525; GFX90A-TGSPLIT-NEXT:    s_endpgm
526    i32 %in, i32 addrspace(1)* %out) {
527entry:
528  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread") monotonic, align 4
529  ret void
530}
531
532define amdgpu_kernel void @global_singlethread_release_store(
533; GFX6-LABEL: global_singlethread_release_store:
534; GFX6:       ; %bb.0: ; %entry
535; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
536; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
537; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
538; GFX6-NEXT:    s_mov_b32 s2, -1
539; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
540; GFX6-NEXT:    v_mov_b32_e32 v0, s6
541; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
542; GFX6-NEXT:    s_endpgm
543;
544; GFX7-LABEL: global_singlethread_release_store:
545; GFX7:       ; %bb.0: ; %entry
546; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
547; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
548; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
549; GFX7-NEXT:    v_mov_b32_e32 v0, s0
550; GFX7-NEXT:    v_mov_b32_e32 v1, s1
551; GFX7-NEXT:    v_mov_b32_e32 v2, s2
552; GFX7-NEXT:    flat_store_dword v[0:1], v2
553; GFX7-NEXT:    s_endpgm
554;
555; GFX10-WGP-LABEL: global_singlethread_release_store:
556; GFX10-WGP:       ; %bb.0: ; %entry
557; GFX10-WGP-NEXT:    s_clause 0x1
558; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
559; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
560; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
561; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
562; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
563; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
564; GFX10-WGP-NEXT:    s_endpgm
565;
566; GFX10-CU-LABEL: global_singlethread_release_store:
567; GFX10-CU:       ; %bb.0: ; %entry
568; GFX10-CU-NEXT:    s_clause 0x1
569; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
570; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
571; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
572; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
573; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
574; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
575; GFX10-CU-NEXT:    s_endpgm
576;
577; SKIP-CACHE-INV-LABEL: global_singlethread_release_store:
578; SKIP-CACHE-INV:       ; %bb.0: ; %entry
579; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x0
580; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
581; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
582; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
583; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
584; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
585; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
586; SKIP-CACHE-INV-NEXT:    s_endpgm
587;
588; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_release_store:
589; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
590; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
591; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
592; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
593; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
594; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
595; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
596; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
597;
598; GFX90A-TGSPLIT-LABEL: global_singlethread_release_store:
599; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
600; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
601; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
602; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
603; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
604; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
605; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
606; GFX90A-TGSPLIT-NEXT:    s_endpgm
607    i32 %in, i32 addrspace(1)* %out) {
608entry:
609  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread") release, align 4
610  ret void
611}
612
613define amdgpu_kernel void @global_singlethread_seq_cst_store(
614; GFX6-LABEL: global_singlethread_seq_cst_store:
615; GFX6:       ; %bb.0: ; %entry
616; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
617; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
618; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
619; GFX6-NEXT:    s_mov_b32 s2, -1
620; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
621; GFX6-NEXT:    v_mov_b32_e32 v0, s6
622; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
623; GFX6-NEXT:    s_endpgm
624;
625; GFX7-LABEL: global_singlethread_seq_cst_store:
626; GFX7:       ; %bb.0: ; %entry
627; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
628; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
629; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
630; GFX7-NEXT:    v_mov_b32_e32 v0, s0
631; GFX7-NEXT:    v_mov_b32_e32 v1, s1
632; GFX7-NEXT:    v_mov_b32_e32 v2, s2
633; GFX7-NEXT:    flat_store_dword v[0:1], v2
634; GFX7-NEXT:    s_endpgm
635;
636; GFX10-WGP-LABEL: global_singlethread_seq_cst_store:
637; GFX10-WGP:       ; %bb.0: ; %entry
638; GFX10-WGP-NEXT:    s_clause 0x1
639; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
640; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
641; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
642; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
643; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
644; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
645; GFX10-WGP-NEXT:    s_endpgm
646;
647; GFX10-CU-LABEL: global_singlethread_seq_cst_store:
648; GFX10-CU:       ; %bb.0: ; %entry
649; GFX10-CU-NEXT:    s_clause 0x1
650; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
651; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
652; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
653; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
654; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
655; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
656; GFX10-CU-NEXT:    s_endpgm
657;
658; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_store:
659; SKIP-CACHE-INV:       ; %bb.0: ; %entry
660; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x0
661; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
662; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
663; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
664; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
665; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
666; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
667; SKIP-CACHE-INV-NEXT:    s_endpgm
668;
669; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_store:
670; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
671; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
672; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
673; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
674; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
675; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
676; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
677; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
678;
679; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_store:
680; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
681; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
682; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
683; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
684; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
685; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
686; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
687; GFX90A-TGSPLIT-NEXT:    s_endpgm
688    i32 %in, i32 addrspace(1)* %out) {
689entry:
690  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread") seq_cst, align 4
691  ret void
692}
693
694define amdgpu_kernel void @global_singlethread_monotonic_atomicrmw(
695; GFX6-LABEL: global_singlethread_monotonic_atomicrmw:
696; GFX6:       ; %bb.0: ; %entry
697; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
698; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
699; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
700; GFX6-NEXT:    s_mov_b32 s2, -1
701; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
702; GFX6-NEXT:    v_mov_b32_e32 v0, s6
703; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
704; GFX6-NEXT:    s_endpgm
705;
706; GFX7-LABEL: global_singlethread_monotonic_atomicrmw:
707; GFX7:       ; %bb.0: ; %entry
708; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
709; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
710; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
711; GFX7-NEXT:    v_mov_b32_e32 v0, s0
712; GFX7-NEXT:    v_mov_b32_e32 v1, s1
713; GFX7-NEXT:    v_mov_b32_e32 v2, s2
714; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
715; GFX7-NEXT:    s_endpgm
716;
717; GFX10-WGP-LABEL: global_singlethread_monotonic_atomicrmw:
718; GFX10-WGP:       ; %bb.0: ; %entry
719; GFX10-WGP-NEXT:    s_clause 0x1
720; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
721; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
722; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
723; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
724; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
725; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
726; GFX10-WGP-NEXT:    s_endpgm
727;
728; GFX10-CU-LABEL: global_singlethread_monotonic_atomicrmw:
729; GFX10-CU:       ; %bb.0: ; %entry
730; GFX10-CU-NEXT:    s_clause 0x1
731; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
732; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
733; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
734; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
735; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
736; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
737; GFX10-CU-NEXT:    s_endpgm
738;
739; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_atomicrmw:
740; SKIP-CACHE-INV:       ; %bb.0: ; %entry
741; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
742; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
743; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
744; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
745; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
746; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
747; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
748; SKIP-CACHE-INV-NEXT:    s_endpgm
749;
750; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_atomicrmw:
751; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
752; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
753; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
754; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
755; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
756; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
757; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
758; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
759;
760; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_atomicrmw:
761; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
762; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
763; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
764; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
765; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
766; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
767; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
768; GFX90A-TGSPLIT-NEXT:    s_endpgm
769    i32 addrspace(1)* %out, i32 %in) {
770entry:
771  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") monotonic
772  ret void
773}
774
775define amdgpu_kernel void @global_singlethread_acquire_atomicrmw(
776; GFX6-LABEL: global_singlethread_acquire_atomicrmw:
777; GFX6:       ; %bb.0: ; %entry
778; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
779; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
780; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
781; GFX6-NEXT:    s_mov_b32 s2, -1
782; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
783; GFX6-NEXT:    v_mov_b32_e32 v0, s6
784; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
785; GFX6-NEXT:    s_endpgm
786;
787; GFX7-LABEL: global_singlethread_acquire_atomicrmw:
788; GFX7:       ; %bb.0: ; %entry
789; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
790; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
791; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
792; GFX7-NEXT:    v_mov_b32_e32 v0, s0
793; GFX7-NEXT:    v_mov_b32_e32 v1, s1
794; GFX7-NEXT:    v_mov_b32_e32 v2, s2
795; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
796; GFX7-NEXT:    s_endpgm
797;
798; GFX10-WGP-LABEL: global_singlethread_acquire_atomicrmw:
799; GFX10-WGP:       ; %bb.0: ; %entry
800; GFX10-WGP-NEXT:    s_clause 0x1
801; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
802; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
803; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
804; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
805; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
806; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
807; GFX10-WGP-NEXT:    s_endpgm
808;
809; GFX10-CU-LABEL: global_singlethread_acquire_atomicrmw:
810; GFX10-CU:       ; %bb.0: ; %entry
811; GFX10-CU-NEXT:    s_clause 0x1
812; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
813; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
814; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
815; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
816; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
817; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
818; GFX10-CU-NEXT:    s_endpgm
819;
820; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_atomicrmw:
821; SKIP-CACHE-INV:       ; %bb.0: ; %entry
822; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
823; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
824; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
825; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
826; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
827; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
828; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
829; SKIP-CACHE-INV-NEXT:    s_endpgm
830;
831; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_atomicrmw:
832; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
833; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
834; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
835; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
836; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
837; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
838; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
839; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
840;
841; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_atomicrmw:
842; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
843; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
844; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
845; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
846; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
847; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
848; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
849; GFX90A-TGSPLIT-NEXT:    s_endpgm
850    i32 addrspace(1)* %out, i32 %in) {
851entry:
852  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") acquire
853  ret void
854}
855
856define amdgpu_kernel void @global_singlethread_release_atomicrmw(
857; GFX6-LABEL: global_singlethread_release_atomicrmw:
858; GFX6:       ; %bb.0: ; %entry
859; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
860; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
861; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
862; GFX6-NEXT:    s_mov_b32 s2, -1
863; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
864; GFX6-NEXT:    v_mov_b32_e32 v0, s6
865; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
866; GFX6-NEXT:    s_endpgm
867;
868; GFX7-LABEL: global_singlethread_release_atomicrmw:
869; GFX7:       ; %bb.0: ; %entry
870; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
871; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
872; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
873; GFX7-NEXT:    v_mov_b32_e32 v0, s0
874; GFX7-NEXT:    v_mov_b32_e32 v1, s1
875; GFX7-NEXT:    v_mov_b32_e32 v2, s2
876; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
877; GFX7-NEXT:    s_endpgm
878;
879; GFX10-WGP-LABEL: global_singlethread_release_atomicrmw:
880; GFX10-WGP:       ; %bb.0: ; %entry
881; GFX10-WGP-NEXT:    s_clause 0x1
882; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
883; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
884; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
885; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
886; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
887; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
888; GFX10-WGP-NEXT:    s_endpgm
889;
890; GFX10-CU-LABEL: global_singlethread_release_atomicrmw:
891; GFX10-CU:       ; %bb.0: ; %entry
892; GFX10-CU-NEXT:    s_clause 0x1
893; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
894; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
895; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
896; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
897; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
898; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
899; GFX10-CU-NEXT:    s_endpgm
900;
901; SKIP-CACHE-INV-LABEL: global_singlethread_release_atomicrmw:
902; SKIP-CACHE-INV:       ; %bb.0: ; %entry
903; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
904; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
905; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
906; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
907; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
908; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
909; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
910; SKIP-CACHE-INV-NEXT:    s_endpgm
911;
912; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_release_atomicrmw:
913; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
914; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
915; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
916; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
917; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
918; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
919; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
920; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
921;
922; GFX90A-TGSPLIT-LABEL: global_singlethread_release_atomicrmw:
923; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
924; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
925; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
926; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
927; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
928; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
929; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
930; GFX90A-TGSPLIT-NEXT:    s_endpgm
931    i32 addrspace(1)* %out, i32 %in) {
932entry:
933  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") release
934  ret void
935}
936
937define amdgpu_kernel void @global_singlethread_acq_rel_atomicrmw(
938; GFX6-LABEL: global_singlethread_acq_rel_atomicrmw:
939; GFX6:       ; %bb.0: ; %entry
940; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
941; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
942; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
943; GFX6-NEXT:    s_mov_b32 s2, -1
944; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
945; GFX6-NEXT:    v_mov_b32_e32 v0, s6
946; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
947; GFX6-NEXT:    s_endpgm
948;
949; GFX7-LABEL: global_singlethread_acq_rel_atomicrmw:
950; GFX7:       ; %bb.0: ; %entry
951; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
952; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
953; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
954; GFX7-NEXT:    v_mov_b32_e32 v0, s0
955; GFX7-NEXT:    v_mov_b32_e32 v1, s1
956; GFX7-NEXT:    v_mov_b32_e32 v2, s2
957; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
958; GFX7-NEXT:    s_endpgm
959;
960; GFX10-WGP-LABEL: global_singlethread_acq_rel_atomicrmw:
961; GFX10-WGP:       ; %bb.0: ; %entry
962; GFX10-WGP-NEXT:    s_clause 0x1
963; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
964; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
965; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
966; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
967; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
968; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
969; GFX10-WGP-NEXT:    s_endpgm
970;
971; GFX10-CU-LABEL: global_singlethread_acq_rel_atomicrmw:
972; GFX10-CU:       ; %bb.0: ; %entry
973; GFX10-CU-NEXT:    s_clause 0x1
974; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
975; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
976; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
977; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
978; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
979; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
980; GFX10-CU-NEXT:    s_endpgm
981;
982; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_atomicrmw:
983; SKIP-CACHE-INV:       ; %bb.0: ; %entry
984; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
985; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
986; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
987; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
988; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
989; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
990; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
991; SKIP-CACHE-INV-NEXT:    s_endpgm
992;
993; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_atomicrmw:
994; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
995; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
996; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
997; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
998; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
999; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1000; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1001; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1002;
1003; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_atomicrmw:
1004; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1005; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1006; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1007; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1008; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1009; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1010; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1011; GFX90A-TGSPLIT-NEXT:    s_endpgm
1012    i32 addrspace(1)* %out, i32 %in) {
1013entry:
1014  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") acq_rel
1015  ret void
1016}
1017
1018define amdgpu_kernel void @global_singlethread_seq_cst_atomicrmw(
1019; GFX6-LABEL: global_singlethread_seq_cst_atomicrmw:
1020; GFX6:       ; %bb.0: ; %entry
1021; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
1022; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1023; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1024; GFX6-NEXT:    s_mov_b32 s2, -1
1025; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1026; GFX6-NEXT:    v_mov_b32_e32 v0, s6
1027; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
1028; GFX6-NEXT:    s_endpgm
1029;
1030; GFX7-LABEL: global_singlethread_seq_cst_atomicrmw:
1031; GFX7:       ; %bb.0: ; %entry
1032; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1033; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1034; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1035; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1036; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1037; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1038; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1039; GFX7-NEXT:    s_endpgm
1040;
1041; GFX10-WGP-LABEL: global_singlethread_seq_cst_atomicrmw:
1042; GFX10-WGP:       ; %bb.0: ; %entry
1043; GFX10-WGP-NEXT:    s_clause 0x1
1044; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1045; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1046; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
1047; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1048; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1049; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
1050; GFX10-WGP-NEXT:    s_endpgm
1051;
1052; GFX10-CU-LABEL: global_singlethread_seq_cst_atomicrmw:
1053; GFX10-CU:       ; %bb.0: ; %entry
1054; GFX10-CU-NEXT:    s_clause 0x1
1055; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1056; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1057; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
1058; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1059; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1060; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
1061; GFX10-CU-NEXT:    s_endpgm
1062;
1063; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_atomicrmw:
1064; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1065; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
1066; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1067; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
1068; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
1069; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1070; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
1071; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
1072; SKIP-CACHE-INV-NEXT:    s_endpgm
1073;
1074; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_atomicrmw:
1075; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1076; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1077; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1078; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1079; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1080; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1081; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1082; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1083;
1084; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_atomicrmw:
1085; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1086; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1087; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1088; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1089; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1090; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1091; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1092; GFX90A-TGSPLIT-NEXT:    s_endpgm
1093    i32 addrspace(1)* %out, i32 %in) {
1094entry:
1095  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") seq_cst
1096  ret void
1097}
1098
1099define amdgpu_kernel void @global_singlethread_acquire_ret_atomicrmw(
1100; GFX6-LABEL: global_singlethread_acquire_ret_atomicrmw:
1101; GFX6:       ; %bb.0: ; %entry
1102; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
1103; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1104; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1105; GFX6-NEXT:    s_mov_b32 s2, -1
1106; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1107; GFX6-NEXT:    v_mov_b32_e32 v0, s6
1108; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
1109; GFX6-NEXT:    s_waitcnt vmcnt(0)
1110; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1111; GFX6-NEXT:    s_endpgm
1112;
1113; GFX7-LABEL: global_singlethread_acquire_ret_atomicrmw:
1114; GFX7:       ; %bb.0: ; %entry
1115; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1116; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1117; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1118; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1119; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1120; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1121; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1122; GFX7-NEXT:    s_waitcnt vmcnt(0)
1123; GFX7-NEXT:    flat_store_dword v[0:1], v2
1124; GFX7-NEXT:    s_endpgm
1125;
1126; GFX10-WGP-LABEL: global_singlethread_acquire_ret_atomicrmw:
1127; GFX10-WGP:       ; %bb.0: ; %entry
1128; GFX10-WGP-NEXT:    s_clause 0x1
1129; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1130; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1131; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
1132; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1133; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1134; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1135; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
1136; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
1137; GFX10-WGP-NEXT:    s_endpgm
1138;
1139; GFX10-CU-LABEL: global_singlethread_acquire_ret_atomicrmw:
1140; GFX10-CU:       ; %bb.0: ; %entry
1141; GFX10-CU-NEXT:    s_clause 0x1
1142; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1143; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1144; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
1145; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1146; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1147; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1148; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
1149; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
1150; GFX10-CU-NEXT:    s_endpgm
1151;
1152; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_ret_atomicrmw:
1153; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1154; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
1155; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1156; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
1157; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
1158; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1159; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
1160; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
1161; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
1162; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1163; SKIP-CACHE-INV-NEXT:    s_endpgm
1164;
1165; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_ret_atomicrmw:
1166; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1167; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1168; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1169; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1170; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1171; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1172; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1173; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1174; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
1175; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1176;
1177; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_ret_atomicrmw:
1178; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1179; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1180; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1181; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1182; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1183; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1184; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1185; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1186; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
1187; GFX90A-TGSPLIT-NEXT:    s_endpgm
1188    i32 addrspace(1)* %out, i32 %in) {
1189entry:
1190  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") acquire
1191  store i32 %val, i32 addrspace(1)* %out, align 4
1192  ret void
1193}
1194
1195define amdgpu_kernel void @global_singlethread_acq_rel_ret_atomicrmw(
1196; GFX6-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1197; GFX6:       ; %bb.0: ; %entry
1198; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
1199; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1200; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1201; GFX6-NEXT:    s_mov_b32 s2, -1
1202; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1203; GFX6-NEXT:    v_mov_b32_e32 v0, s6
1204; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
1205; GFX6-NEXT:    s_waitcnt vmcnt(0)
1206; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1207; GFX6-NEXT:    s_endpgm
1208;
1209; GFX7-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1210; GFX7:       ; %bb.0: ; %entry
1211; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1212; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1213; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1214; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1215; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1216; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1217; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1218; GFX7-NEXT:    s_waitcnt vmcnt(0)
1219; GFX7-NEXT:    flat_store_dword v[0:1], v2
1220; GFX7-NEXT:    s_endpgm
1221;
1222; GFX10-WGP-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1223; GFX10-WGP:       ; %bb.0: ; %entry
1224; GFX10-WGP-NEXT:    s_clause 0x1
1225; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1226; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1227; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
1228; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1229; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1230; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1231; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
1232; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
1233; GFX10-WGP-NEXT:    s_endpgm
1234;
1235; GFX10-CU-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1236; GFX10-CU:       ; %bb.0: ; %entry
1237; GFX10-CU-NEXT:    s_clause 0x1
1238; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1239; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1240; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
1241; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1242; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1243; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1244; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
1245; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
1246; GFX10-CU-NEXT:    s_endpgm
1247;
1248; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1249; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1250; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
1251; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1252; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
1253; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
1254; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1255; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
1256; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
1257; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
1258; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1259; SKIP-CACHE-INV-NEXT:    s_endpgm
1260;
1261; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1262; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1263; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1264; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1265; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1266; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1267; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1268; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1269; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1270; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
1271; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1272;
1273; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1274; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1275; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1276; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1277; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1278; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1279; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1280; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1281; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1282; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
1283; GFX90A-TGSPLIT-NEXT:    s_endpgm
1284    i32 addrspace(1)* %out, i32 %in) {
1285entry:
1286  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") acq_rel
1287  store i32 %val, i32 addrspace(1)* %out, align 4
1288  ret void
1289}
1290
1291define amdgpu_kernel void @global_singlethread_seq_cst_ret_atomicrmw(
1292; GFX6-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
1293; GFX6:       ; %bb.0: ; %entry
1294; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
1295; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1296; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1297; GFX6-NEXT:    s_mov_b32 s2, -1
1298; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1299; GFX6-NEXT:    v_mov_b32_e32 v0, s6
1300; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
1301; GFX6-NEXT:    s_waitcnt vmcnt(0)
1302; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1303; GFX6-NEXT:    s_endpgm
1304;
1305; GFX7-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
1306; GFX7:       ; %bb.0: ; %entry
1307; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1308; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1309; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1310; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1311; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1312; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1313; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1314; GFX7-NEXT:    s_waitcnt vmcnt(0)
1315; GFX7-NEXT:    flat_store_dword v[0:1], v2
1316; GFX7-NEXT:    s_endpgm
1317;
1318; GFX10-WGP-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
1319; GFX10-WGP:       ; %bb.0: ; %entry
1320; GFX10-WGP-NEXT:    s_clause 0x1
1321; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1322; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1323; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
1324; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1325; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1326; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1327; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
1328; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
1329; GFX10-WGP-NEXT:    s_endpgm
1330;
1331; GFX10-CU-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
1332; GFX10-CU:       ; %bb.0: ; %entry
1333; GFX10-CU-NEXT:    s_clause 0x1
1334; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1335; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1336; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
1337; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1338; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1339; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1340; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
1341; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
1342; GFX10-CU-NEXT:    s_endpgm
1343;
1344; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
1345; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1346; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
1347; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1348; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
1349; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
1350; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1351; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
1352; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
1353; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
1354; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1355; SKIP-CACHE-INV-NEXT:    s_endpgm
1356;
1357; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
1358; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1359; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1360; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1361; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1362; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1363; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1364; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1365; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1366; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
1367; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1368;
1369; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
1370; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1371; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1372; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1373; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1374; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1375; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1376; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1377; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1378; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
1379; GFX90A-TGSPLIT-NEXT:    s_endpgm
1380    i32 addrspace(1)* %out, i32 %in) {
1381entry:
1382  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") seq_cst
1383  store i32 %val, i32 addrspace(1)* %out, align 4
1384  ret void
1385}
1386
1387define amdgpu_kernel void @global_singlethread_monotonic_monotonic_cmpxchg(
1388; GFX6-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
1389; GFX6:       ; %bb.0: ; %entry
1390; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
1391; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1392; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1393; GFX6-NEXT:    s_mov_b32 s2, -1
1394; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1395; GFX6-NEXT:    v_mov_b32_e32 v0, s6
1396; GFX6-NEXT:    v_mov_b32_e32 v1, s7
1397; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
1398; GFX6-NEXT:    s_endpgm
1399;
1400; GFX7-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
1401; GFX7:       ; %bb.0: ; %entry
1402; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1403; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1404; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1405; GFX7-NEXT:    s_add_u32 s0, s0, 16
1406; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1407; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1408; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1409; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1410; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1411; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1412; GFX7-NEXT:    s_endpgm
1413;
1414; GFX10-WGP-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
1415; GFX10-WGP:       ; %bb.0: ; %entry
1416; GFX10-WGP-NEXT:    s_clause 0x1
1417; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1418; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1419; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
1420; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1421; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1422; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1423; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1424; GFX10-WGP-NEXT:    s_endpgm
1425;
1426; GFX10-CU-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
1427; GFX10-CU:       ; %bb.0: ; %entry
1428; GFX10-CU-NEXT:    s_clause 0x1
1429; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1430; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1431; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
1432; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1433; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1434; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1435; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1436; GFX10-CU-NEXT:    s_endpgm
1437;
1438; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
1439; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1440; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
1441; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1442; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
1443; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
1444; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1445; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
1446; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
1447; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
1448; SKIP-CACHE-INV-NEXT:    s_endpgm
1449;
1450; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
1451; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1452; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1453; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1454; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1455; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1456; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1457; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1458; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1459;
1460; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
1461; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1462; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1463; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1464; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1465; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1466; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1467; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1468; GFX90A-TGSPLIT-NEXT:    s_endpgm
1469    i32 addrspace(1)* %out, i32 %in, i32 %old) {
1470entry:
1471  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
1472  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic monotonic
1473  ret void
1474}
1475
1476define amdgpu_kernel void @global_singlethread_acquire_monotonic_cmpxchg(
1477; GFX6-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
1478; GFX6:       ; %bb.0: ; %entry
1479; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
1480; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1481; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1482; GFX6-NEXT:    s_mov_b32 s2, -1
1483; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1484; GFX6-NEXT:    v_mov_b32_e32 v0, s6
1485; GFX6-NEXT:    v_mov_b32_e32 v1, s7
1486; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
1487; GFX6-NEXT:    s_endpgm
1488;
1489; GFX7-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
1490; GFX7:       ; %bb.0: ; %entry
1491; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1492; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1493; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1494; GFX7-NEXT:    s_add_u32 s0, s0, 16
1495; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1496; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1497; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1498; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1499; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1500; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1501; GFX7-NEXT:    s_endpgm
1502;
1503; GFX10-WGP-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
1504; GFX10-WGP:       ; %bb.0: ; %entry
1505; GFX10-WGP-NEXT:    s_clause 0x1
1506; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1507; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1508; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
1509; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1510; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1511; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1512; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1513; GFX10-WGP-NEXT:    s_endpgm
1514;
1515; GFX10-CU-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
1516; GFX10-CU:       ; %bb.0: ; %entry
1517; GFX10-CU-NEXT:    s_clause 0x1
1518; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1519; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1520; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
1521; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1522; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1523; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1524; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1525; GFX10-CU-NEXT:    s_endpgm
1526;
1527; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
1528; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1529; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
1530; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1531; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
1532; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
1533; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1534; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
1535; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
1536; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
1537; SKIP-CACHE-INV-NEXT:    s_endpgm
1538;
1539; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
1540; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1541; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1542; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1543; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1544; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1545; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1546; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1547; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1548;
1549; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
1550; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1551; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1552; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1553; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1554; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1555; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1556; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1557; GFX90A-TGSPLIT-NEXT:    s_endpgm
1558    i32 addrspace(1)* %out, i32 %in, i32 %old) {
1559entry:
1560  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
1561  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire monotonic
1562  ret void
1563}
1564
1565define amdgpu_kernel void @global_singlethread_release_monotonic_cmpxchg(
1566; GFX6-LABEL: global_singlethread_release_monotonic_cmpxchg:
1567; GFX6:       ; %bb.0: ; %entry
1568; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
1569; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1570; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1571; GFX6-NEXT:    s_mov_b32 s2, -1
1572; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1573; GFX6-NEXT:    v_mov_b32_e32 v0, s6
1574; GFX6-NEXT:    v_mov_b32_e32 v1, s7
1575; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
1576; GFX6-NEXT:    s_endpgm
1577;
1578; GFX7-LABEL: global_singlethread_release_monotonic_cmpxchg:
1579; GFX7:       ; %bb.0: ; %entry
1580; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1581; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1582; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1583; GFX7-NEXT:    s_add_u32 s0, s0, 16
1584; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1585; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1586; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1587; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1588; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1589; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1590; GFX7-NEXT:    s_endpgm
1591;
1592; GFX10-WGP-LABEL: global_singlethread_release_monotonic_cmpxchg:
1593; GFX10-WGP:       ; %bb.0: ; %entry
1594; GFX10-WGP-NEXT:    s_clause 0x1
1595; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1596; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1597; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
1598; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1599; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1600; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1601; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1602; GFX10-WGP-NEXT:    s_endpgm
1603;
1604; GFX10-CU-LABEL: global_singlethread_release_monotonic_cmpxchg:
1605; GFX10-CU:       ; %bb.0: ; %entry
1606; GFX10-CU-NEXT:    s_clause 0x1
1607; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1608; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1609; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
1610; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1611; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1612; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1613; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1614; GFX10-CU-NEXT:    s_endpgm
1615;
1616; SKIP-CACHE-INV-LABEL: global_singlethread_release_monotonic_cmpxchg:
1617; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1618; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
1619; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1620; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
1621; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
1622; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1623; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
1624; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
1625; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
1626; SKIP-CACHE-INV-NEXT:    s_endpgm
1627;
1628; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_release_monotonic_cmpxchg:
1629; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1630; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1631; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1632; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1633; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1634; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1635; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1636; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1637;
1638; GFX90A-TGSPLIT-LABEL: global_singlethread_release_monotonic_cmpxchg:
1639; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1640; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1641; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1642; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1643; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1644; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1645; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1646; GFX90A-TGSPLIT-NEXT:    s_endpgm
1647    i32 addrspace(1)* %out, i32 %in, i32 %old) {
1648entry:
1649  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
1650  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") release monotonic
1651  ret void
1652}
1653
1654define amdgpu_kernel void @global_singlethread_acq_rel_monotonic_cmpxchg(
1655; GFX6-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
1656; GFX6:       ; %bb.0: ; %entry
1657; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
1658; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1659; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1660; GFX6-NEXT:    s_mov_b32 s2, -1
1661; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1662; GFX6-NEXT:    v_mov_b32_e32 v0, s6
1663; GFX6-NEXT:    v_mov_b32_e32 v1, s7
1664; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
1665; GFX6-NEXT:    s_endpgm
1666;
1667; GFX7-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
1668; GFX7:       ; %bb.0: ; %entry
1669; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1670; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1671; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1672; GFX7-NEXT:    s_add_u32 s0, s0, 16
1673; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1674; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1675; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1676; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1677; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1678; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1679; GFX7-NEXT:    s_endpgm
1680;
1681; GFX10-WGP-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
1682; GFX10-WGP:       ; %bb.0: ; %entry
1683; GFX10-WGP-NEXT:    s_clause 0x1
1684; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1685; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1686; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
1687; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1688; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1689; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1690; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1691; GFX10-WGP-NEXT:    s_endpgm
1692;
1693; GFX10-CU-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
1694; GFX10-CU:       ; %bb.0: ; %entry
1695; GFX10-CU-NEXT:    s_clause 0x1
1696; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1697; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1698; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
1699; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1700; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1701; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1702; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1703; GFX10-CU-NEXT:    s_endpgm
1704;
1705; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
1706; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1707; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
1708; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1709; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
1710; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
1711; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1712; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
1713; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
1714; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
1715; SKIP-CACHE-INV-NEXT:    s_endpgm
1716;
1717; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
1718; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1719; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1720; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1721; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1722; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1723; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1724; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1725; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1726;
1727; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
1728; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1729; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1730; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1731; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1732; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1733; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1734; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1735; GFX90A-TGSPLIT-NEXT:    s_endpgm
1736    i32 addrspace(1)* %out, i32 %in, i32 %old) {
1737entry:
1738  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
1739  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel monotonic
1740  ret void
1741}
1742
1743define amdgpu_kernel void @global_singlethread_seq_cst_monotonic_cmpxchg(
1744; GFX6-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
1745; GFX6:       ; %bb.0: ; %entry
1746; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
1747; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1748; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1749; GFX6-NEXT:    s_mov_b32 s2, -1
1750; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1751; GFX6-NEXT:    v_mov_b32_e32 v0, s6
1752; GFX6-NEXT:    v_mov_b32_e32 v1, s7
1753; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
1754; GFX6-NEXT:    s_endpgm
1755;
1756; GFX7-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
1757; GFX7:       ; %bb.0: ; %entry
1758; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1759; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1760; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1761; GFX7-NEXT:    s_add_u32 s0, s0, 16
1762; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1763; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1764; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1765; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1766; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1767; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1768; GFX7-NEXT:    s_endpgm
1769;
1770; GFX10-WGP-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
1771; GFX10-WGP:       ; %bb.0: ; %entry
1772; GFX10-WGP-NEXT:    s_clause 0x1
1773; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1774; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1775; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
1776; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1777; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1778; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1779; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1780; GFX10-WGP-NEXT:    s_endpgm
1781;
1782; GFX10-CU-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
1783; GFX10-CU:       ; %bb.0: ; %entry
1784; GFX10-CU-NEXT:    s_clause 0x1
1785; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1786; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1787; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
1788; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1789; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1790; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1791; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1792; GFX10-CU-NEXT:    s_endpgm
1793;
1794; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
1795; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1796; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
1797; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1798; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
1799; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
1800; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1801; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
1802; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
1803; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
1804; SKIP-CACHE-INV-NEXT:    s_endpgm
1805;
1806; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
1807; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1808; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1809; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1810; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1811; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1812; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1813; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1814; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1815;
1816; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
1817; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1818; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1819; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1820; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1821; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1822; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1823; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1824; GFX90A-TGSPLIT-NEXT:    s_endpgm
1825    i32 addrspace(1)* %out, i32 %in, i32 %old) {
1826entry:
1827  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
1828  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst monotonic
1829  ret void
1830}
1831
1832define amdgpu_kernel void @global_singlethread_monotonic_acquire_cmpxchg(
1833; GFX6-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
1834; GFX6:       ; %bb.0: ; %entry
1835; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
1836; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1837; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1838; GFX6-NEXT:    s_mov_b32 s2, -1
1839; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1840; GFX6-NEXT:    v_mov_b32_e32 v0, s6
1841; GFX6-NEXT:    v_mov_b32_e32 v1, s7
1842; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
1843; GFX6-NEXT:    s_endpgm
1844;
1845; GFX7-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
1846; GFX7:       ; %bb.0: ; %entry
1847; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1848; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1849; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1850; GFX7-NEXT:    s_add_u32 s0, s0, 16
1851; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1852; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1853; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1854; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1855; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1856; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1857; GFX7-NEXT:    s_endpgm
1858;
1859; GFX10-WGP-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
1860; GFX10-WGP:       ; %bb.0: ; %entry
1861; GFX10-WGP-NEXT:    s_clause 0x1
1862; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1863; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1864; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
1865; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1866; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1867; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1868; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1869; GFX10-WGP-NEXT:    s_endpgm
1870;
1871; GFX10-CU-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
1872; GFX10-CU:       ; %bb.0: ; %entry
1873; GFX10-CU-NEXT:    s_clause 0x1
1874; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1875; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1876; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
1877; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1878; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1879; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1880; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1881; GFX10-CU-NEXT:    s_endpgm
1882;
1883; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
1884; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1885; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
1886; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1887; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
1888; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
1889; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1890; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
1891; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
1892; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
1893; SKIP-CACHE-INV-NEXT:    s_endpgm
1894;
1895; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
1896; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1897; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1898; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1899; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1900; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1901; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1902; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1903; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1904;
1905; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
1906; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1907; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1908; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1909; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1910; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1911; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1912; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1913; GFX90A-TGSPLIT-NEXT:    s_endpgm
1914    i32 addrspace(1)* %out, i32 %in, i32 %old) {
1915entry:
1916  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
1917  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic acquire
1918  ret void
1919}
1920
1921define amdgpu_kernel void @global_singlethread_acquire_acquire_cmpxchg(
1922; GFX6-LABEL: global_singlethread_acquire_acquire_cmpxchg:
1923; GFX6:       ; %bb.0: ; %entry
1924; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
1925; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1926; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1927; GFX6-NEXT:    s_mov_b32 s2, -1
1928; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1929; GFX6-NEXT:    v_mov_b32_e32 v0, s6
1930; GFX6-NEXT:    v_mov_b32_e32 v1, s7
1931; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
1932; GFX6-NEXT:    s_endpgm
1933;
1934; GFX7-LABEL: global_singlethread_acquire_acquire_cmpxchg:
1935; GFX7:       ; %bb.0: ; %entry
1936; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1937; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1938; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1939; GFX7-NEXT:    s_add_u32 s0, s0, 16
1940; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1941; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1942; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1943; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1944; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1945; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1946; GFX7-NEXT:    s_endpgm
1947;
1948; GFX10-WGP-LABEL: global_singlethread_acquire_acquire_cmpxchg:
1949; GFX10-WGP:       ; %bb.0: ; %entry
1950; GFX10-WGP-NEXT:    s_clause 0x1
1951; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1952; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1953; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
1954; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1955; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1956; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1957; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1958; GFX10-WGP-NEXT:    s_endpgm
1959;
1960; GFX10-CU-LABEL: global_singlethread_acquire_acquire_cmpxchg:
1961; GFX10-CU:       ; %bb.0: ; %entry
1962; GFX10-CU-NEXT:    s_clause 0x1
1963; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1964; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1965; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
1966; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1967; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1968; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1969; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1970; GFX10-CU-NEXT:    s_endpgm
1971;
1972; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_acquire_cmpxchg:
1973; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1974; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
1975; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1976; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
1977; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
1978; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1979; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
1980; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
1981; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
1982; SKIP-CACHE-INV-NEXT:    s_endpgm
1983;
1984; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_acquire_cmpxchg:
1985; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1986; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1987; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1988; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1989; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1990; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1991; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1992; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1993;
1994; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_acquire_cmpxchg:
1995; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1996; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1997; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1998; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1999; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2000; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2001; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2002; GFX90A-TGSPLIT-NEXT:    s_endpgm
2003    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2004entry:
2005  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2006  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire acquire
2007  ret void
2008}
2009
2010define amdgpu_kernel void @global_singlethread_release_acquire_cmpxchg(
2011; GFX6-LABEL: global_singlethread_release_acquire_cmpxchg:
2012; GFX6:       ; %bb.0: ; %entry
2013; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
2014; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2015; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2016; GFX6-NEXT:    s_mov_b32 s2, -1
2017; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2018; GFX6-NEXT:    v_mov_b32_e32 v0, s6
2019; GFX6-NEXT:    v_mov_b32_e32 v1, s7
2020; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2021; GFX6-NEXT:    s_endpgm
2022;
2023; GFX7-LABEL: global_singlethread_release_acquire_cmpxchg:
2024; GFX7:       ; %bb.0: ; %entry
2025; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2026; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2027; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2028; GFX7-NEXT:    s_add_u32 s0, s0, 16
2029; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2030; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2031; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2032; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2033; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2034; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2035; GFX7-NEXT:    s_endpgm
2036;
2037; GFX10-WGP-LABEL: global_singlethread_release_acquire_cmpxchg:
2038; GFX10-WGP:       ; %bb.0: ; %entry
2039; GFX10-WGP-NEXT:    s_clause 0x1
2040; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2041; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2042; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2043; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2044; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2045; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2046; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2047; GFX10-WGP-NEXT:    s_endpgm
2048;
2049; GFX10-CU-LABEL: global_singlethread_release_acquire_cmpxchg:
2050; GFX10-CU:       ; %bb.0: ; %entry
2051; GFX10-CU-NEXT:    s_clause 0x1
2052; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2053; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2054; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2055; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2056; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2057; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2058; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2059; GFX10-CU-NEXT:    s_endpgm
2060;
2061; SKIP-CACHE-INV-LABEL: global_singlethread_release_acquire_cmpxchg:
2062; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2063; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
2064; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
2065; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
2066; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
2067; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2068; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2069; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2070; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2071; SKIP-CACHE-INV-NEXT:    s_endpgm
2072;
2073; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_release_acquire_cmpxchg:
2074; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2075; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2076; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2077; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2078; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2079; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2080; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2081; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2082;
2083; GFX90A-TGSPLIT-LABEL: global_singlethread_release_acquire_cmpxchg:
2084; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2085; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2086; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2087; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2088; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2089; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2090; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2091; GFX90A-TGSPLIT-NEXT:    s_endpgm
2092    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2093entry:
2094  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2095  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") release acquire
2096  ret void
2097}
2098
2099define amdgpu_kernel void @global_singlethread_acq_rel_acquire_cmpxchg(
2100; GFX6-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
2101; GFX6:       ; %bb.0: ; %entry
2102; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
2103; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2104; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2105; GFX6-NEXT:    s_mov_b32 s2, -1
2106; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2107; GFX6-NEXT:    v_mov_b32_e32 v0, s6
2108; GFX6-NEXT:    v_mov_b32_e32 v1, s7
2109; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2110; GFX6-NEXT:    s_endpgm
2111;
2112; GFX7-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
2113; GFX7:       ; %bb.0: ; %entry
2114; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2115; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2116; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2117; GFX7-NEXT:    s_add_u32 s0, s0, 16
2118; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2119; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2120; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2121; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2122; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2123; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2124; GFX7-NEXT:    s_endpgm
2125;
2126; GFX10-WGP-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
2127; GFX10-WGP:       ; %bb.0: ; %entry
2128; GFX10-WGP-NEXT:    s_clause 0x1
2129; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2130; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2131; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2132; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2133; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2134; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2135; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2136; GFX10-WGP-NEXT:    s_endpgm
2137;
2138; GFX10-CU-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
2139; GFX10-CU:       ; %bb.0: ; %entry
2140; GFX10-CU-NEXT:    s_clause 0x1
2141; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2142; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2143; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2144; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2145; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2146; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2147; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2148; GFX10-CU-NEXT:    s_endpgm
2149;
2150; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
2151; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2152; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
2153; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
2154; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
2155; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
2156; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2157; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2158; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2159; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2160; SKIP-CACHE-INV-NEXT:    s_endpgm
2161;
2162; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
2163; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2164; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2165; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2166; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2167; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2168; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2169; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2170; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2171;
2172; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
2173; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2174; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2175; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2176; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2177; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2178; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2179; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2180; GFX90A-TGSPLIT-NEXT:    s_endpgm
2181    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2182entry:
2183  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2184  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel acquire
2185  ret void
2186}
2187
2188define amdgpu_kernel void @global_singlethread_seq_cst_acquire_cmpxchg(
2189; GFX6-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
2190; GFX6:       ; %bb.0: ; %entry
2191; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
2192; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2193; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2194; GFX6-NEXT:    s_mov_b32 s2, -1
2195; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2196; GFX6-NEXT:    v_mov_b32_e32 v0, s6
2197; GFX6-NEXT:    v_mov_b32_e32 v1, s7
2198; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2199; GFX6-NEXT:    s_endpgm
2200;
2201; GFX7-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
2202; GFX7:       ; %bb.0: ; %entry
2203; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2204; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2205; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2206; GFX7-NEXT:    s_add_u32 s0, s0, 16
2207; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2208; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2209; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2210; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2211; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2212; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2213; GFX7-NEXT:    s_endpgm
2214;
2215; GFX10-WGP-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
2216; GFX10-WGP:       ; %bb.0: ; %entry
2217; GFX10-WGP-NEXT:    s_clause 0x1
2218; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2219; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2220; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2221; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2222; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2223; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2224; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2225; GFX10-WGP-NEXT:    s_endpgm
2226;
2227; GFX10-CU-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
2228; GFX10-CU:       ; %bb.0: ; %entry
2229; GFX10-CU-NEXT:    s_clause 0x1
2230; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2231; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2232; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2233; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2234; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2235; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2236; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2237; GFX10-CU-NEXT:    s_endpgm
2238;
2239; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
2240; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2241; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
2242; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
2243; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
2244; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
2245; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2246; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2247; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2248; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2249; SKIP-CACHE-INV-NEXT:    s_endpgm
2250;
2251; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
2252; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2253; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2254; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2255; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2256; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2257; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2258; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2259; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2260;
2261; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
2262; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2263; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2264; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2265; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2266; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2267; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2268; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2269; GFX90A-TGSPLIT-NEXT:    s_endpgm
2270    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2271entry:
2272  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2273  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst acquire
2274  ret void
2275}
2276
2277define amdgpu_kernel void @global_singlethread_monotonic_seq_cst_cmpxchg(
2278; GFX6-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
2279; GFX6:       ; %bb.0: ; %entry
2280; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
2281; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2282; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2283; GFX6-NEXT:    s_mov_b32 s2, -1
2284; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2285; GFX6-NEXT:    v_mov_b32_e32 v0, s6
2286; GFX6-NEXT:    v_mov_b32_e32 v1, s7
2287; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2288; GFX6-NEXT:    s_endpgm
2289;
2290; GFX7-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
2291; GFX7:       ; %bb.0: ; %entry
2292; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2293; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2294; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2295; GFX7-NEXT:    s_add_u32 s0, s0, 16
2296; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2297; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2298; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2299; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2300; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2301; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2302; GFX7-NEXT:    s_endpgm
2303;
2304; GFX10-WGP-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
2305; GFX10-WGP:       ; %bb.0: ; %entry
2306; GFX10-WGP-NEXT:    s_clause 0x1
2307; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2308; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2309; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2310; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2311; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2312; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2313; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2314; GFX10-WGP-NEXT:    s_endpgm
2315;
2316; GFX10-CU-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
2317; GFX10-CU:       ; %bb.0: ; %entry
2318; GFX10-CU-NEXT:    s_clause 0x1
2319; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2320; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2321; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2322; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2323; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2324; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2325; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2326; GFX10-CU-NEXT:    s_endpgm
2327;
2328; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
2329; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2330; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
2331; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
2332; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
2333; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
2334; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2335; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2336; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2337; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2338; SKIP-CACHE-INV-NEXT:    s_endpgm
2339;
2340; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
2341; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2342; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2343; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2344; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2345; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2346; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2347; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2348; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2349;
2350; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
2351; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2352; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2353; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2354; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2355; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2356; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2357; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2358; GFX90A-TGSPLIT-NEXT:    s_endpgm
2359    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2360entry:
2361  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2362  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic seq_cst
2363  ret void
2364}
2365
2366define amdgpu_kernel void @global_singlethread_acquire_seq_cst_cmpxchg(
2367; GFX6-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
2368; GFX6:       ; %bb.0: ; %entry
2369; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
2370; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2371; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2372; GFX6-NEXT:    s_mov_b32 s2, -1
2373; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2374; GFX6-NEXT:    v_mov_b32_e32 v0, s6
2375; GFX6-NEXT:    v_mov_b32_e32 v1, s7
2376; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2377; GFX6-NEXT:    s_endpgm
2378;
2379; GFX7-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
2380; GFX7:       ; %bb.0: ; %entry
2381; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2382; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2383; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2384; GFX7-NEXT:    s_add_u32 s0, s0, 16
2385; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2386; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2387; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2388; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2389; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2390; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2391; GFX7-NEXT:    s_endpgm
2392;
2393; GFX10-WGP-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
2394; GFX10-WGP:       ; %bb.0: ; %entry
2395; GFX10-WGP-NEXT:    s_clause 0x1
2396; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2397; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2398; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2399; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2400; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2401; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2402; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2403; GFX10-WGP-NEXT:    s_endpgm
2404;
2405; GFX10-CU-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
2406; GFX10-CU:       ; %bb.0: ; %entry
2407; GFX10-CU-NEXT:    s_clause 0x1
2408; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2409; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2410; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2411; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2412; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2413; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2414; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2415; GFX10-CU-NEXT:    s_endpgm
2416;
2417; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
2418; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2419; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
2420; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
2421; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
2422; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
2423; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2424; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2425; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2426; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2427; SKIP-CACHE-INV-NEXT:    s_endpgm
2428;
2429; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
2430; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2431; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2432; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2433; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2434; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2435; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2436; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2437; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2438;
2439; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
2440; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2441; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2442; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2443; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2444; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2445; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2446; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2447; GFX90A-TGSPLIT-NEXT:    s_endpgm
2448    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2449entry:
2450  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2451  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire seq_cst
2452  ret void
2453}
2454
2455define amdgpu_kernel void @global_singlethread_release_seq_cst_cmpxchg(
2456; GFX6-LABEL: global_singlethread_release_seq_cst_cmpxchg:
2457; GFX6:       ; %bb.0: ; %entry
2458; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
2459; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2460; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2461; GFX6-NEXT:    s_mov_b32 s2, -1
2462; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2463; GFX6-NEXT:    v_mov_b32_e32 v0, s6
2464; GFX6-NEXT:    v_mov_b32_e32 v1, s7
2465; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2466; GFX6-NEXT:    s_endpgm
2467;
2468; GFX7-LABEL: global_singlethread_release_seq_cst_cmpxchg:
2469; GFX7:       ; %bb.0: ; %entry
2470; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2471; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2472; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2473; GFX7-NEXT:    s_add_u32 s0, s0, 16
2474; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2475; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2476; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2477; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2478; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2479; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2480; GFX7-NEXT:    s_endpgm
2481;
2482; GFX10-WGP-LABEL: global_singlethread_release_seq_cst_cmpxchg:
2483; GFX10-WGP:       ; %bb.0: ; %entry
2484; GFX10-WGP-NEXT:    s_clause 0x1
2485; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2486; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2487; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2488; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2489; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2490; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2491; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2492; GFX10-WGP-NEXT:    s_endpgm
2493;
2494; GFX10-CU-LABEL: global_singlethread_release_seq_cst_cmpxchg:
2495; GFX10-CU:       ; %bb.0: ; %entry
2496; GFX10-CU-NEXT:    s_clause 0x1
2497; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2498; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2499; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2500; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2501; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2502; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2503; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2504; GFX10-CU-NEXT:    s_endpgm
2505;
2506; SKIP-CACHE-INV-LABEL: global_singlethread_release_seq_cst_cmpxchg:
2507; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2508; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
2509; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
2510; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
2511; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
2512; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2513; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2514; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2515; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2516; SKIP-CACHE-INV-NEXT:    s_endpgm
2517;
2518; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_release_seq_cst_cmpxchg:
2519; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2520; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2521; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2522; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2523; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2524; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2525; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2526; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2527;
2528; GFX90A-TGSPLIT-LABEL: global_singlethread_release_seq_cst_cmpxchg:
2529; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2530; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2531; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2532; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2533; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2534; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2535; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2536; GFX90A-TGSPLIT-NEXT:    s_endpgm
2537    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2538entry:
2539  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2540  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") release seq_cst
2541  ret void
2542}
2543
2544define amdgpu_kernel void @global_singlethread_acq_rel_seq_cst_cmpxchg(
2545; GFX6-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
2546; GFX6:       ; %bb.0: ; %entry
2547; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
2548; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2549; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2550; GFX6-NEXT:    s_mov_b32 s2, -1
2551; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2552; GFX6-NEXT:    v_mov_b32_e32 v0, s6
2553; GFX6-NEXT:    v_mov_b32_e32 v1, s7
2554; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2555; GFX6-NEXT:    s_endpgm
2556;
2557; GFX7-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
2558; GFX7:       ; %bb.0: ; %entry
2559; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2560; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2561; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2562; GFX7-NEXT:    s_add_u32 s0, s0, 16
2563; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2564; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2565; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2566; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2567; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2568; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2569; GFX7-NEXT:    s_endpgm
2570;
2571; GFX10-WGP-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
2572; GFX10-WGP:       ; %bb.0: ; %entry
2573; GFX10-WGP-NEXT:    s_clause 0x1
2574; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2575; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2576; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2577; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2578; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2579; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2580; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2581; GFX10-WGP-NEXT:    s_endpgm
2582;
2583; GFX10-CU-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
2584; GFX10-CU:       ; %bb.0: ; %entry
2585; GFX10-CU-NEXT:    s_clause 0x1
2586; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2587; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2588; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2589; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2590; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2591; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2592; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2593; GFX10-CU-NEXT:    s_endpgm
2594;
2595; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
2596; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2597; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
2598; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
2599; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
2600; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
2601; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2602; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2603; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2604; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2605; SKIP-CACHE-INV-NEXT:    s_endpgm
2606;
2607; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
2608; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2609; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2610; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2611; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2612; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2613; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2614; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2615; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2616;
2617; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
2618; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2619; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2620; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2621; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2622; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2623; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2624; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2625; GFX90A-TGSPLIT-NEXT:    s_endpgm
2626    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2627entry:
2628  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2629  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel seq_cst
2630  ret void
2631}
2632
2633define amdgpu_kernel void @global_singlethread_seq_cst_seq_cst_cmpxchg(
2634; GFX6-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
2635; GFX6:       ; %bb.0: ; %entry
2636; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
2637; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2638; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2639; GFX6-NEXT:    s_mov_b32 s2, -1
2640; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2641; GFX6-NEXT:    v_mov_b32_e32 v0, s6
2642; GFX6-NEXT:    v_mov_b32_e32 v1, s7
2643; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2644; GFX6-NEXT:    s_endpgm
2645;
2646; GFX7-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
2647; GFX7:       ; %bb.0: ; %entry
2648; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2649; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2650; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2651; GFX7-NEXT:    s_add_u32 s0, s0, 16
2652; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2653; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2654; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2655; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2656; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2657; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2658; GFX7-NEXT:    s_endpgm
2659;
2660; GFX10-WGP-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
2661; GFX10-WGP:       ; %bb.0: ; %entry
2662; GFX10-WGP-NEXT:    s_clause 0x1
2663; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2664; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2665; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2666; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2667; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2668; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2669; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2670; GFX10-WGP-NEXT:    s_endpgm
2671;
2672; GFX10-CU-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
2673; GFX10-CU:       ; %bb.0: ; %entry
2674; GFX10-CU-NEXT:    s_clause 0x1
2675; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2676; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2677; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2678; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2679; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2680; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2681; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2682; GFX10-CU-NEXT:    s_endpgm
2683;
2684; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
2685; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2686; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
2687; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
2688; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
2689; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
2690; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2691; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2692; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2693; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2694; SKIP-CACHE-INV-NEXT:    s_endpgm
2695;
2696; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
2697; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2698; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2699; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2700; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2701; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2702; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2703; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2704; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2705;
2706; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
2707; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2708; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2709; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2710; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2711; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2712; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2713; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2714; GFX90A-TGSPLIT-NEXT:    s_endpgm
2715    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2716entry:
2717  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2718  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst seq_cst
2719  ret void
2720}
2721
2722define amdgpu_kernel void @global_singlethread_monotonic_monotonic_ret_cmpxchg(
2723; GFX6-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
2724; GFX6:       ; %bb.0: ; %entry
2725; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
2726; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2727; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2728; GFX6-NEXT:    s_mov_b32 s2, -1
2729; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2730; GFX6-NEXT:    v_mov_b32_e32 v0, s6
2731; GFX6-NEXT:    v_mov_b32_e32 v1, s7
2732; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
2733; GFX6-NEXT:    s_waitcnt vmcnt(0)
2734; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
2735; GFX6-NEXT:    s_endpgm
2736;
2737; GFX7-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
2738; GFX7:       ; %bb.0: ; %entry
2739; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2740; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2741; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2742; GFX7-NEXT:    s_add_u32 s4, s0, 16
2743; GFX7-NEXT:    s_addc_u32 s5, s1, 0
2744; GFX7-NEXT:    v_mov_b32_e32 v0, s4
2745; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2746; GFX7-NEXT:    v_mov_b32_e32 v1, s5
2747; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2748; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2749; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2750; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2751; GFX7-NEXT:    s_waitcnt vmcnt(0)
2752; GFX7-NEXT:    flat_store_dword v[0:1], v2
2753; GFX7-NEXT:    s_endpgm
2754;
2755; GFX10-WGP-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
2756; GFX10-WGP:       ; %bb.0: ; %entry
2757; GFX10-WGP-NEXT:    s_clause 0x1
2758; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2759; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2760; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2761; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2762; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2763; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2764; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
2765; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
2766; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
2767; GFX10-WGP-NEXT:    s_endpgm
2768;
2769; GFX10-CU-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
2770; GFX10-CU:       ; %bb.0: ; %entry
2771; GFX10-CU-NEXT:    s_clause 0x1
2772; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2773; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2774; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2775; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2776; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2777; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2778; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
2779; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
2780; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
2781; GFX10-CU-NEXT:    s_endpgm
2782;
2783; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
2784; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2785; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
2786; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
2787; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
2788; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
2789; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2790; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2791; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2792; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
2793; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
2794; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
2795; SKIP-CACHE-INV-NEXT:    s_endpgm
2796;
2797; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
2798; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2799; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2800; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2801; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2802; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2803; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2804; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
2805; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2806; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
2807; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2808;
2809; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
2810; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2811; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2812; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2813; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2814; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2815; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2816; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
2817; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2818; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
2819; GFX90A-TGSPLIT-NEXT:    s_endpgm
2820    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2821entry:
2822  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2823  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic monotonic
2824  %val0 = extractvalue { i32, i1 } %val, 0
2825  store i32 %val0, i32 addrspace(1)* %out, align 4
2826  ret void
2827}
2828
2829define amdgpu_kernel void @global_singlethread_acquire_monotonic_ret_cmpxchg(
2830; GFX6-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
2831; GFX6:       ; %bb.0: ; %entry
2832; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
2833; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2834; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2835; GFX6-NEXT:    s_mov_b32 s2, -1
2836; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2837; GFX6-NEXT:    v_mov_b32_e32 v0, s6
2838; GFX6-NEXT:    v_mov_b32_e32 v1, s7
2839; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
2840; GFX6-NEXT:    s_waitcnt vmcnt(0)
2841; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
2842; GFX6-NEXT:    s_endpgm
2843;
2844; GFX7-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
2845; GFX7:       ; %bb.0: ; %entry
2846; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2847; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2848; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2849; GFX7-NEXT:    s_add_u32 s4, s0, 16
2850; GFX7-NEXT:    s_addc_u32 s5, s1, 0
2851; GFX7-NEXT:    v_mov_b32_e32 v0, s4
2852; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2853; GFX7-NEXT:    v_mov_b32_e32 v1, s5
2854; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2855; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2856; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2857; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2858; GFX7-NEXT:    s_waitcnt vmcnt(0)
2859; GFX7-NEXT:    flat_store_dword v[0:1], v2
2860; GFX7-NEXT:    s_endpgm
2861;
2862; GFX10-WGP-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
2863; GFX10-WGP:       ; %bb.0: ; %entry
2864; GFX10-WGP-NEXT:    s_clause 0x1
2865; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2866; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2867; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2868; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2869; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2870; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2871; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
2872; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
2873; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
2874; GFX10-WGP-NEXT:    s_endpgm
2875;
2876; GFX10-CU-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
2877; GFX10-CU:       ; %bb.0: ; %entry
2878; GFX10-CU-NEXT:    s_clause 0x1
2879; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2880; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2881; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2882; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2883; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2884; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2885; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
2886; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
2887; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
2888; GFX10-CU-NEXT:    s_endpgm
2889;
2890; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
2891; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2892; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
2893; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
2894; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
2895; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
2896; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2897; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2898; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2899; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
2900; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
2901; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
2902; SKIP-CACHE-INV-NEXT:    s_endpgm
2903;
2904; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
2905; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2906; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2907; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2908; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2909; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2910; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2911; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
2912; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2913; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
2914; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2915;
2916; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
2917; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2918; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2919; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2920; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2921; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2922; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2923; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
2924; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2925; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
2926; GFX90A-TGSPLIT-NEXT:    s_endpgm
2927    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2928entry:
2929  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2930  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire monotonic
2931  %val0 = extractvalue { i32, i1 } %val, 0
2932  store i32 %val0, i32 addrspace(1)* %out, align 4
2933  ret void
2934}
2935
2936define amdgpu_kernel void @global_singlethread_release_monotonic_ret_cmpxchg(
2937; GFX6-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
2938; GFX6:       ; %bb.0: ; %entry
2939; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
2940; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2941; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2942; GFX6-NEXT:    s_mov_b32 s2, -1
2943; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2944; GFX6-NEXT:    v_mov_b32_e32 v0, s6
2945; GFX6-NEXT:    v_mov_b32_e32 v1, s7
2946; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
2947; GFX6-NEXT:    s_waitcnt vmcnt(0)
2948; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
2949; GFX6-NEXT:    s_endpgm
2950;
2951; GFX7-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
2952; GFX7:       ; %bb.0: ; %entry
2953; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2954; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2955; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2956; GFX7-NEXT:    s_add_u32 s4, s0, 16
2957; GFX7-NEXT:    s_addc_u32 s5, s1, 0
2958; GFX7-NEXT:    v_mov_b32_e32 v0, s4
2959; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2960; GFX7-NEXT:    v_mov_b32_e32 v1, s5
2961; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2962; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2963; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2964; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2965; GFX7-NEXT:    s_waitcnt vmcnt(0)
2966; GFX7-NEXT:    flat_store_dword v[0:1], v2
2967; GFX7-NEXT:    s_endpgm
2968;
2969; GFX10-WGP-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
2970; GFX10-WGP:       ; %bb.0: ; %entry
2971; GFX10-WGP-NEXT:    s_clause 0x1
2972; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2973; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2974; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2975; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2976; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2977; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2978; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
2979; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
2980; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
2981; GFX10-WGP-NEXT:    s_endpgm
2982;
2983; GFX10-CU-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
2984; GFX10-CU:       ; %bb.0: ; %entry
2985; GFX10-CU-NEXT:    s_clause 0x1
2986; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2987; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2988; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2989; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2990; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2991; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2992; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
2993; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
2994; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
2995; GFX10-CU-NEXT:    s_endpgm
2996;
2997; SKIP-CACHE-INV-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
2998; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2999; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
3000; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
3001; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3002; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3003; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3004; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3005; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3006; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3007; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3008; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3009; SKIP-CACHE-INV-NEXT:    s_endpgm
3010;
3011; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
3012; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3013; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3014; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3015; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3016; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3017; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3018; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3019; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3020; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
3021; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3022;
3023; GFX90A-TGSPLIT-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
3024; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3025; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3026; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3027; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3028; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3029; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3030; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3031; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3032; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
3033; GFX90A-TGSPLIT-NEXT:    s_endpgm
3034    i32 addrspace(1)* %out, i32 %in, i32 %old) {
3035entry:
3036  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3037  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") release monotonic
3038  %val0 = extractvalue { i32, i1 } %val, 0
3039  store i32 %val0, i32 addrspace(1)* %out, align 4
3040  ret void
3041}
3042
3043define amdgpu_kernel void @global_singlethread_acq_rel_monotonic_ret_cmpxchg(
3044; GFX6-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
3045; GFX6:       ; %bb.0: ; %entry
3046; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
3047; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3048; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3049; GFX6-NEXT:    s_mov_b32 s2, -1
3050; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3051; GFX6-NEXT:    v_mov_b32_e32 v0, s6
3052; GFX6-NEXT:    v_mov_b32_e32 v1, s7
3053; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3054; GFX6-NEXT:    s_waitcnt vmcnt(0)
3055; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3056; GFX6-NEXT:    s_endpgm
3057;
3058; GFX7-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
3059; GFX7:       ; %bb.0: ; %entry
3060; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3061; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3062; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3063; GFX7-NEXT:    s_add_u32 s4, s0, 16
3064; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3065; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3066; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3067; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3068; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3069; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3070; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3071; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3072; GFX7-NEXT:    s_waitcnt vmcnt(0)
3073; GFX7-NEXT:    flat_store_dword v[0:1], v2
3074; GFX7-NEXT:    s_endpgm
3075;
3076; GFX10-WGP-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
3077; GFX10-WGP:       ; %bb.0: ; %entry
3078; GFX10-WGP-NEXT:    s_clause 0x1
3079; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3080; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3081; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3082; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3083; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3084; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3085; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3086; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3087; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
3088; GFX10-WGP-NEXT:    s_endpgm
3089;
3090; GFX10-CU-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
3091; GFX10-CU:       ; %bb.0: ; %entry
3092; GFX10-CU-NEXT:    s_clause 0x1
3093; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3094; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3095; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3096; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3097; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3098; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3099; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3100; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3101; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
3102; GFX10-CU-NEXT:    s_endpgm
3103;
3104; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
3105; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3106; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
3107; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
3108; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3109; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3110; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3111; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3112; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3113; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3114; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3115; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3116; SKIP-CACHE-INV-NEXT:    s_endpgm
3117;
3118; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
3119; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3120; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3121; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3122; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3123; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3124; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3125; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3126; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3127; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
3128; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3129;
3130; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
3131; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3132; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3133; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3134; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3135; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3136; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3137; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3138; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3139; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
3140; GFX90A-TGSPLIT-NEXT:    s_endpgm
3141    i32 addrspace(1)* %out, i32 %in, i32 %old) {
3142entry:
3143  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3144  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel monotonic
3145  %val0 = extractvalue { i32, i1 } %val, 0
3146  store i32 %val0, i32 addrspace(1)* %out, align 4
3147  ret void
3148}
3149
3150define amdgpu_kernel void @global_singlethread_seq_cst_monotonic_ret_cmpxchg(
3151; GFX6-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
3152; GFX6:       ; %bb.0: ; %entry
3153; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
3154; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3155; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3156; GFX6-NEXT:    s_mov_b32 s2, -1
3157; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3158; GFX6-NEXT:    v_mov_b32_e32 v0, s6
3159; GFX6-NEXT:    v_mov_b32_e32 v1, s7
3160; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3161; GFX6-NEXT:    s_waitcnt vmcnt(0)
3162; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3163; GFX6-NEXT:    s_endpgm
3164;
3165; GFX7-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
3166; GFX7:       ; %bb.0: ; %entry
3167; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3168; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3169; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3170; GFX7-NEXT:    s_add_u32 s4, s0, 16
3171; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3172; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3173; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3174; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3175; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3176; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3177; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3178; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3179; GFX7-NEXT:    s_waitcnt vmcnt(0)
3180; GFX7-NEXT:    flat_store_dword v[0:1], v2
3181; GFX7-NEXT:    s_endpgm
3182;
3183; GFX10-WGP-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
3184; GFX10-WGP:       ; %bb.0: ; %entry
3185; GFX10-WGP-NEXT:    s_clause 0x1
3186; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3187; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3188; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3189; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3190; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3191; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3192; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3193; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3194; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
3195; GFX10-WGP-NEXT:    s_endpgm
3196;
3197; GFX10-CU-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
3198; GFX10-CU:       ; %bb.0: ; %entry
3199; GFX10-CU-NEXT:    s_clause 0x1
3200; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3201; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3202; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3203; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3204; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3205; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3206; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3207; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3208; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
3209; GFX10-CU-NEXT:    s_endpgm
3210;
3211; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
3212; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3213; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
3214; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
3215; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3216; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3217; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3218; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3219; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3220; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3221; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3222; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3223; SKIP-CACHE-INV-NEXT:    s_endpgm
3224;
3225; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
3226; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3227; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3228; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3229; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3230; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3231; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3232; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3233; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3234; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
3235; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3236;
3237; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
3238; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3239; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3240; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3241; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3242; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3243; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3244; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3245; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3246; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
3247; GFX90A-TGSPLIT-NEXT:    s_endpgm
3248    i32 addrspace(1)* %out, i32 %in, i32 %old) {
3249entry:
3250  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3251  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst monotonic
3252  %val0 = extractvalue { i32, i1 } %val, 0
3253  store i32 %val0, i32 addrspace(1)* %out, align 4
3254  ret void
3255}
3256
3257define amdgpu_kernel void @global_singlethread_monotonic_acquire_ret_cmpxchg(
3258; GFX6-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
3259; GFX6:       ; %bb.0: ; %entry
3260; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
3261; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3262; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3263; GFX6-NEXT:    s_mov_b32 s2, -1
3264; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3265; GFX6-NEXT:    v_mov_b32_e32 v0, s6
3266; GFX6-NEXT:    v_mov_b32_e32 v1, s7
3267; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3268; GFX6-NEXT:    s_waitcnt vmcnt(0)
3269; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3270; GFX6-NEXT:    s_endpgm
3271;
3272; GFX7-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
3273; GFX7:       ; %bb.0: ; %entry
3274; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3275; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3276; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3277; GFX7-NEXT:    s_add_u32 s4, s0, 16
3278; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3279; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3280; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3281; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3282; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3283; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3284; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3285; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3286; GFX7-NEXT:    s_waitcnt vmcnt(0)
3287; GFX7-NEXT:    flat_store_dword v[0:1], v2
3288; GFX7-NEXT:    s_endpgm
3289;
3290; GFX10-WGP-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
3291; GFX10-WGP:       ; %bb.0: ; %entry
3292; GFX10-WGP-NEXT:    s_clause 0x1
3293; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3294; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3295; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3296; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3297; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3298; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3299; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3300; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3301; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
3302; GFX10-WGP-NEXT:    s_endpgm
3303;
3304; GFX10-CU-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
3305; GFX10-CU:       ; %bb.0: ; %entry
3306; GFX10-CU-NEXT:    s_clause 0x1
3307; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3308; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3309; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3310; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3311; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3312; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3313; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3314; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3315; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
3316; GFX10-CU-NEXT:    s_endpgm
3317;
3318; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
3319; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3320; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
3321; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
3322; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3323; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3324; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3325; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3326; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3327; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3328; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3329; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3330; SKIP-CACHE-INV-NEXT:    s_endpgm
3331;
3332; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
3333; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3334; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3335; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3336; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3337; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3338; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3339; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3340; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3341; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
3342; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3343;
3344; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
3345; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3346; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3347; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3348; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3349; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3350; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3351; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3352; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3353; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
3354; GFX90A-TGSPLIT-NEXT:    s_endpgm
3355    i32 addrspace(1)* %out, i32 %in, i32 %old) {
3356entry:
3357  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3358  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic acquire
3359  %val0 = extractvalue { i32, i1 } %val, 0
3360  store i32 %val0, i32 addrspace(1)* %out, align 4
3361  ret void
3362}
3363
3364define amdgpu_kernel void @global_singlethread_acquire_acquire_ret_cmpxchg(
3365; GFX6-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
3366; GFX6:       ; %bb.0: ; %entry
3367; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
3368; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3369; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3370; GFX6-NEXT:    s_mov_b32 s2, -1
3371; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3372; GFX6-NEXT:    v_mov_b32_e32 v0, s6
3373; GFX6-NEXT:    v_mov_b32_e32 v1, s7
3374; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3375; GFX6-NEXT:    s_waitcnt vmcnt(0)
3376; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3377; GFX6-NEXT:    s_endpgm
3378;
3379; GFX7-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
3380; GFX7:       ; %bb.0: ; %entry
3381; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3382; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3383; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3384; GFX7-NEXT:    s_add_u32 s4, s0, 16
3385; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3386; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3387; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3388; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3389; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3390; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3391; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3392; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3393; GFX7-NEXT:    s_waitcnt vmcnt(0)
3394; GFX7-NEXT:    flat_store_dword v[0:1], v2
3395; GFX7-NEXT:    s_endpgm
3396;
3397; GFX10-WGP-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
3398; GFX10-WGP:       ; %bb.0: ; %entry
3399; GFX10-WGP-NEXT:    s_clause 0x1
3400; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3401; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3402; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3403; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3404; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3405; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3406; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3407; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3408; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
3409; GFX10-WGP-NEXT:    s_endpgm
3410;
3411; GFX10-CU-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
3412; GFX10-CU:       ; %bb.0: ; %entry
3413; GFX10-CU-NEXT:    s_clause 0x1
3414; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3415; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3416; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3417; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3418; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3419; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3420; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3421; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3422; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
3423; GFX10-CU-NEXT:    s_endpgm
3424;
3425; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
3426; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3427; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
3428; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
3429; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3430; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3431; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3432; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3433; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3434; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3435; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3436; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3437; SKIP-CACHE-INV-NEXT:    s_endpgm
3438;
3439; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
3440; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3441; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3442; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3443; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3444; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3445; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3446; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3447; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3448; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
3449; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3450;
3451; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
3452; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3453; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3454; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3455; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3456; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3457; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3458; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3459; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3460; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
3461; GFX90A-TGSPLIT-NEXT:    s_endpgm
3462    i32 addrspace(1)* %out, i32 %in, i32 %old) {
3463entry:
3464  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3465  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire acquire
3466  %val0 = extractvalue { i32, i1 } %val, 0
3467  store i32 %val0, i32 addrspace(1)* %out, align 4
3468  ret void
3469}
3470
3471define amdgpu_kernel void @global_singlethread_release_acquire_ret_cmpxchg(
3472; GFX6-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
3473; GFX6:       ; %bb.0: ; %entry
3474; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
3475; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3476; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3477; GFX6-NEXT:    s_mov_b32 s2, -1
3478; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3479; GFX6-NEXT:    v_mov_b32_e32 v0, s6
3480; GFX6-NEXT:    v_mov_b32_e32 v1, s7
3481; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3482; GFX6-NEXT:    s_waitcnt vmcnt(0)
3483; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3484; GFX6-NEXT:    s_endpgm
3485;
3486; GFX7-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
3487; GFX7:       ; %bb.0: ; %entry
3488; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3489; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3490; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3491; GFX7-NEXT:    s_add_u32 s4, s0, 16
3492; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3493; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3494; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3495; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3496; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3497; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3498; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3499; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3500; GFX7-NEXT:    s_waitcnt vmcnt(0)
3501; GFX7-NEXT:    flat_store_dword v[0:1], v2
3502; GFX7-NEXT:    s_endpgm
3503;
3504; GFX10-WGP-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
3505; GFX10-WGP:       ; %bb.0: ; %entry
3506; GFX10-WGP-NEXT:    s_clause 0x1
3507; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3508; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3509; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3510; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3511; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3512; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3513; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3514; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3515; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
3516; GFX10-WGP-NEXT:    s_endpgm
3517;
3518; GFX10-CU-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
3519; GFX10-CU:       ; %bb.0: ; %entry
3520; GFX10-CU-NEXT:    s_clause 0x1
3521; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3522; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3523; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3524; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3525; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3526; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3527; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3528; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3529; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
3530; GFX10-CU-NEXT:    s_endpgm
3531;
3532; SKIP-CACHE-INV-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
3533; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3534; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
3535; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
3536; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3537; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3538; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3539; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3540; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3541; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3542; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3543; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3544; SKIP-CACHE-INV-NEXT:    s_endpgm
3545;
3546; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
3547; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3548; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3549; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3550; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3551; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3552; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3553; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3554; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3555; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
3556; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3557;
3558; GFX90A-TGSPLIT-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
3559; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3560; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3561; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3562; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3563; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3564; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3565; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3566; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3567; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
3568; GFX90A-TGSPLIT-NEXT:    s_endpgm
3569    i32 addrspace(1)* %out, i32 %in, i32 %old) {
3570entry:
3571  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3572  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") release acquire
3573  %val0 = extractvalue { i32, i1 } %val, 0
3574  store i32 %val0, i32 addrspace(1)* %out, align 4
3575  ret void
3576}
3577
3578define amdgpu_kernel void @global_singlethread_acq_rel_acquire_ret_cmpxchg(
3579; GFX6-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
3580; GFX6:       ; %bb.0: ; %entry
3581; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
3582; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3583; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3584; GFX6-NEXT:    s_mov_b32 s2, -1
3585; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3586; GFX6-NEXT:    v_mov_b32_e32 v0, s6
3587; GFX6-NEXT:    v_mov_b32_e32 v1, s7
3588; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3589; GFX6-NEXT:    s_waitcnt vmcnt(0)
3590; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3591; GFX6-NEXT:    s_endpgm
3592;
3593; GFX7-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
3594; GFX7:       ; %bb.0: ; %entry
3595; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3596; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3597; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3598; GFX7-NEXT:    s_add_u32 s4, s0, 16
3599; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3600; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3601; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3602; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3603; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3604; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3605; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3606; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3607; GFX7-NEXT:    s_waitcnt vmcnt(0)
3608; GFX7-NEXT:    flat_store_dword v[0:1], v2
3609; GFX7-NEXT:    s_endpgm
3610;
3611; GFX10-WGP-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
3612; GFX10-WGP:       ; %bb.0: ; %entry
3613; GFX10-WGP-NEXT:    s_clause 0x1
3614; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3615; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3616; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3617; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3618; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3619; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3620; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3621; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3622; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
3623; GFX10-WGP-NEXT:    s_endpgm
3624;
3625; GFX10-CU-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
3626; GFX10-CU:       ; %bb.0: ; %entry
3627; GFX10-CU-NEXT:    s_clause 0x1
3628; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3629; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3630; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3631; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3632; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3633; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3634; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3635; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3636; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
3637; GFX10-CU-NEXT:    s_endpgm
3638;
3639; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
3640; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3641; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
3642; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
3643; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3644; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3645; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3646; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3647; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3648; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3649; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3650; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3651; SKIP-CACHE-INV-NEXT:    s_endpgm
3652;
3653; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
3654; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3655; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3656; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3657; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3658; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3659; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3660; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3661; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3662; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
3663; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3664;
3665; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
3666; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3667; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3668; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3669; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3670; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3671; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3672; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3673; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3674; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
3675; GFX90A-TGSPLIT-NEXT:    s_endpgm
3676    i32 addrspace(1)* %out, i32 %in, i32 %old) {
3677entry:
3678  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3679  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel acquire
3680  %val0 = extractvalue { i32, i1 } %val, 0
3681  store i32 %val0, i32 addrspace(1)* %out, align 4
3682  ret void
3683}
3684
3685define amdgpu_kernel void @global_singlethread_seq_cst_acquire_ret_cmpxchg(
3686; GFX6-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
3687; GFX6:       ; %bb.0: ; %entry
3688; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
3689; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3690; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3691; GFX6-NEXT:    s_mov_b32 s2, -1
3692; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3693; GFX6-NEXT:    v_mov_b32_e32 v0, s6
3694; GFX6-NEXT:    v_mov_b32_e32 v1, s7
3695; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3696; GFX6-NEXT:    s_waitcnt vmcnt(0)
3697; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3698; GFX6-NEXT:    s_endpgm
3699;
3700; GFX7-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
3701; GFX7:       ; %bb.0: ; %entry
3702; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3703; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3704; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3705; GFX7-NEXT:    s_add_u32 s4, s0, 16
3706; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3707; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3708; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3709; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3710; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3711; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3712; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3713; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3714; GFX7-NEXT:    s_waitcnt vmcnt(0)
3715; GFX7-NEXT:    flat_store_dword v[0:1], v2
3716; GFX7-NEXT:    s_endpgm
3717;
3718; GFX10-WGP-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
3719; GFX10-WGP:       ; %bb.0: ; %entry
3720; GFX10-WGP-NEXT:    s_clause 0x1
3721; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3722; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3723; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3724; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3725; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3726; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3727; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3728; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3729; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
3730; GFX10-WGP-NEXT:    s_endpgm
3731;
3732; GFX10-CU-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
3733; GFX10-CU:       ; %bb.0: ; %entry
3734; GFX10-CU-NEXT:    s_clause 0x1
3735; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3736; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3737; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3738; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3739; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3740; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3741; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3742; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3743; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
3744; GFX10-CU-NEXT:    s_endpgm
3745;
3746; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
3747; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3748; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
3749; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
3750; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3751; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3752; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3753; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3754; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3755; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3756; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3757; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3758; SKIP-CACHE-INV-NEXT:    s_endpgm
3759;
3760; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
3761; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3762; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3763; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3764; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3765; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3766; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3767; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3768; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3769; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
3770; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3771;
3772; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
3773; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3774; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3775; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3776; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3777; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3778; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3779; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3780; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3781; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
3782; GFX90A-TGSPLIT-NEXT:    s_endpgm
3783    i32 addrspace(1)* %out, i32 %in, i32 %old) {
3784entry:
3785  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3786  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst acquire
3787  %val0 = extractvalue { i32, i1 } %val, 0
3788  store i32 %val0, i32 addrspace(1)* %out, align 4
3789  ret void
3790}
3791
3792define amdgpu_kernel void @global_singlethread_monotonic_seq_cst_ret_cmpxchg(
3793; GFX6-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
3794; GFX6:       ; %bb.0: ; %entry
3795; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
3796; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3797; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3798; GFX6-NEXT:    s_mov_b32 s2, -1
3799; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3800; GFX6-NEXT:    v_mov_b32_e32 v0, s6
3801; GFX6-NEXT:    v_mov_b32_e32 v1, s7
3802; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3803; GFX6-NEXT:    s_waitcnt vmcnt(0)
3804; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3805; GFX6-NEXT:    s_endpgm
3806;
3807; GFX7-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
3808; GFX7:       ; %bb.0: ; %entry
3809; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3810; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3811; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3812; GFX7-NEXT:    s_add_u32 s4, s0, 16
3813; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3814; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3815; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3816; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3817; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3818; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3819; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3820; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3821; GFX7-NEXT:    s_waitcnt vmcnt(0)
3822; GFX7-NEXT:    flat_store_dword v[0:1], v2
3823; GFX7-NEXT:    s_endpgm
3824;
3825; GFX10-WGP-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
3826; GFX10-WGP:       ; %bb.0: ; %entry
3827; GFX10-WGP-NEXT:    s_clause 0x1
3828; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3829; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3830; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3831; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3832; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3833; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3834; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3835; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3836; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
3837; GFX10-WGP-NEXT:    s_endpgm
3838;
3839; GFX10-CU-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
3840; GFX10-CU:       ; %bb.0: ; %entry
3841; GFX10-CU-NEXT:    s_clause 0x1
3842; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3843; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3844; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3845; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3846; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3847; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3848; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3849; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3850; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
3851; GFX10-CU-NEXT:    s_endpgm
3852;
3853; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
3854; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3855; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
3856; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
3857; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3858; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3859; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3860; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3861; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3862; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3863; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3864; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3865; SKIP-CACHE-INV-NEXT:    s_endpgm
3866;
3867; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
3868; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3869; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3870; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3871; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3872; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3873; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3874; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3875; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3876; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
3877; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3878;
3879; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
3880; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3881; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3882; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3883; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3884; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3885; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3886; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3887; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3888; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
3889; GFX90A-TGSPLIT-NEXT:    s_endpgm
3890    i32 addrspace(1)* %out, i32 %in, i32 %old) {
3891entry:
3892  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3893  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic seq_cst
3894  %val0 = extractvalue { i32, i1 } %val, 0
3895  store i32 %val0, i32 addrspace(1)* %out, align 4
3896  ret void
3897}
3898
3899define amdgpu_kernel void @global_singlethread_acquire_seq_cst_ret_cmpxchg(
3900; GFX6-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
3901; GFX6:       ; %bb.0: ; %entry
3902; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
3903; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3904; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3905; GFX6-NEXT:    s_mov_b32 s2, -1
3906; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3907; GFX6-NEXT:    v_mov_b32_e32 v0, s6
3908; GFX6-NEXT:    v_mov_b32_e32 v1, s7
3909; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3910; GFX6-NEXT:    s_waitcnt vmcnt(0)
3911; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3912; GFX6-NEXT:    s_endpgm
3913;
3914; GFX7-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
3915; GFX7:       ; %bb.0: ; %entry
3916; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3917; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3918; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3919; GFX7-NEXT:    s_add_u32 s4, s0, 16
3920; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3921; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3922; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3923; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3924; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3925; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3926; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3927; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3928; GFX7-NEXT:    s_waitcnt vmcnt(0)
3929; GFX7-NEXT:    flat_store_dword v[0:1], v2
3930; GFX7-NEXT:    s_endpgm
3931;
3932; GFX10-WGP-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
3933; GFX10-WGP:       ; %bb.0: ; %entry
3934; GFX10-WGP-NEXT:    s_clause 0x1
3935; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3936; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3937; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3938; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3939; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3940; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3941; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3942; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3943; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
3944; GFX10-WGP-NEXT:    s_endpgm
3945;
3946; GFX10-CU-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
3947; GFX10-CU:       ; %bb.0: ; %entry
3948; GFX10-CU-NEXT:    s_clause 0x1
3949; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3950; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3951; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3952; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3953; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3954; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3955; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3956; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3957; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
3958; GFX10-CU-NEXT:    s_endpgm
3959;
3960; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
3961; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3962; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
3963; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
3964; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3965; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3966; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3967; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3968; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3969; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3970; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3971; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3972; SKIP-CACHE-INV-NEXT:    s_endpgm
3973;
3974; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
3975; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3976; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3977; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3978; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3979; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3980; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3981; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3982; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3983; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
3984; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3985;
3986; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
3987; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3988; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3989; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3990; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3991; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3992; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3993; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3994; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3995; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
3996; GFX90A-TGSPLIT-NEXT:    s_endpgm
3997    i32 addrspace(1)* %out, i32 %in, i32 %old) {
3998entry:
3999  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
4000  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire seq_cst
4001  %val0 = extractvalue { i32, i1 } %val, 0
4002  store i32 %val0, i32 addrspace(1)* %out, align 4
4003  ret void
4004}
4005
4006define amdgpu_kernel void @global_singlethread_release_seq_cst_ret_cmpxchg(
4007; GFX6-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
4008; GFX6:       ; %bb.0: ; %entry
4009; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
4010; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4011; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4012; GFX6-NEXT:    s_mov_b32 s2, -1
4013; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4014; GFX6-NEXT:    v_mov_b32_e32 v0, s6
4015; GFX6-NEXT:    v_mov_b32_e32 v1, s7
4016; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4017; GFX6-NEXT:    s_waitcnt vmcnt(0)
4018; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4019; GFX6-NEXT:    s_endpgm
4020;
4021; GFX7-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
4022; GFX7:       ; %bb.0: ; %entry
4023; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4024; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4025; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4026; GFX7-NEXT:    s_add_u32 s4, s0, 16
4027; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4028; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4029; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4030; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4031; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4032; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4033; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4034; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4035; GFX7-NEXT:    s_waitcnt vmcnt(0)
4036; GFX7-NEXT:    flat_store_dword v[0:1], v2
4037; GFX7-NEXT:    s_endpgm
4038;
4039; GFX10-WGP-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
4040; GFX10-WGP:       ; %bb.0: ; %entry
4041; GFX10-WGP-NEXT:    s_clause 0x1
4042; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4043; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4044; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
4045; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4046; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4047; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4048; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4049; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4050; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
4051; GFX10-WGP-NEXT:    s_endpgm
4052;
4053; GFX10-CU-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
4054; GFX10-CU:       ; %bb.0: ; %entry
4055; GFX10-CU-NEXT:    s_clause 0x1
4056; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4057; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4058; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
4059; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4060; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4061; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4062; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4063; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4064; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
4065; GFX10-CU-NEXT:    s_endpgm
4066;
4067; SKIP-CACHE-INV-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
4068; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4069; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
4070; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
4071; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
4072; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
4073; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4074; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4075; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4076; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4077; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4078; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4079; SKIP-CACHE-INV-NEXT:    s_endpgm
4080;
4081; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
4082; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4083; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4084; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4085; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4086; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4087; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4088; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4089; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4090; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4091; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4092;
4093; GFX90A-TGSPLIT-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
4094; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4095; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4096; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4097; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4098; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4099; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4100; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4101; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4102; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4103; GFX90A-TGSPLIT-NEXT:    s_endpgm
4104    i32 addrspace(1)* %out, i32 %in, i32 %old) {
4105entry:
4106  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
4107  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") release seq_cst
4108  %val0 = extractvalue { i32, i1 } %val, 0
4109  store i32 %val0, i32 addrspace(1)* %out, align 4
4110  ret void
4111}
4112
4113define amdgpu_kernel void @global_singlethread_acq_rel_seq_cst_ret_cmpxchg(
4114; GFX6-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
4115; GFX6:       ; %bb.0: ; %entry
4116; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
4117; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4118; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4119; GFX6-NEXT:    s_mov_b32 s2, -1
4120; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4121; GFX6-NEXT:    v_mov_b32_e32 v0, s6
4122; GFX6-NEXT:    v_mov_b32_e32 v1, s7
4123; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4124; GFX6-NEXT:    s_waitcnt vmcnt(0)
4125; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4126; GFX6-NEXT:    s_endpgm
4127;
4128; GFX7-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
4129; GFX7:       ; %bb.0: ; %entry
4130; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4131; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4132; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4133; GFX7-NEXT:    s_add_u32 s4, s0, 16
4134; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4135; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4136; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4137; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4138; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4139; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4140; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4141; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4142; GFX7-NEXT:    s_waitcnt vmcnt(0)
4143; GFX7-NEXT:    flat_store_dword v[0:1], v2
4144; GFX7-NEXT:    s_endpgm
4145;
4146; GFX10-WGP-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
4147; GFX10-WGP:       ; %bb.0: ; %entry
4148; GFX10-WGP-NEXT:    s_clause 0x1
4149; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4150; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4151; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
4152; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4153; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4154; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4155; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4156; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4157; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
4158; GFX10-WGP-NEXT:    s_endpgm
4159;
4160; GFX10-CU-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
4161; GFX10-CU:       ; %bb.0: ; %entry
4162; GFX10-CU-NEXT:    s_clause 0x1
4163; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4164; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4165; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
4166; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4167; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4168; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4169; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4170; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4171; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
4172; GFX10-CU-NEXT:    s_endpgm
4173;
4174; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
4175; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4176; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
4177; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
4178; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
4179; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
4180; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4181; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4182; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4183; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4184; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4185; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4186; SKIP-CACHE-INV-NEXT:    s_endpgm
4187;
4188; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
4189; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4190; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4191; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4192; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4193; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4194; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4195; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4196; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4197; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4198; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4199;
4200; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
4201; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4202; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4203; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4204; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4205; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4206; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4207; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4208; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4209; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4210; GFX90A-TGSPLIT-NEXT:    s_endpgm
4211    i32 addrspace(1)* %out, i32 %in, i32 %old) {
4212entry:
4213  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
4214  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel seq_cst
4215  %val0 = extractvalue { i32, i1 } %val, 0
4216  store i32 %val0, i32 addrspace(1)* %out, align 4
4217  ret void
4218}
4219
4220define amdgpu_kernel void @global_singlethread_seq_cst_seq_cst_ret_cmpxchg(
4221; GFX6-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
4222; GFX6:       ; %bb.0: ; %entry
4223; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
4224; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4225; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4226; GFX6-NEXT:    s_mov_b32 s2, -1
4227; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4228; GFX6-NEXT:    v_mov_b32_e32 v0, s6
4229; GFX6-NEXT:    v_mov_b32_e32 v1, s7
4230; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4231; GFX6-NEXT:    s_waitcnt vmcnt(0)
4232; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4233; GFX6-NEXT:    s_endpgm
4234;
4235; GFX7-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
4236; GFX7:       ; %bb.0: ; %entry
4237; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4238; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4239; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4240; GFX7-NEXT:    s_add_u32 s4, s0, 16
4241; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4242; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4243; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4244; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4245; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4246; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4247; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4248; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4249; GFX7-NEXT:    s_waitcnt vmcnt(0)
4250; GFX7-NEXT:    flat_store_dword v[0:1], v2
4251; GFX7-NEXT:    s_endpgm
4252;
4253; GFX10-WGP-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
4254; GFX10-WGP:       ; %bb.0: ; %entry
4255; GFX10-WGP-NEXT:    s_clause 0x1
4256; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4257; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4258; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
4259; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4260; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4261; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4262; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4263; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4264; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
4265; GFX10-WGP-NEXT:    s_endpgm
4266;
4267; GFX10-CU-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
4268; GFX10-CU:       ; %bb.0: ; %entry
4269; GFX10-CU-NEXT:    s_clause 0x1
4270; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4271; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4272; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
4273; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4274; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4275; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4276; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4277; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4278; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
4279; GFX10-CU-NEXT:    s_endpgm
4280;
4281; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
4282; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4283; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
4284; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
4285; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
4286; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
4287; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4288; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4289; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4290; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4291; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4292; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4293; SKIP-CACHE-INV-NEXT:    s_endpgm
4294;
4295; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
4296; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4297; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4298; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4299; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4300; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4301; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4302; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4303; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4304; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4305; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4306;
4307; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
4308; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4309; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4310; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4311; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4312; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4313; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4314; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4315; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4316; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4317; GFX90A-TGSPLIT-NEXT:    s_endpgm
4318    i32 addrspace(1)* %out, i32 %in, i32 %old) {
4319entry:
4320  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
4321  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst seq_cst
4322  %val0 = extractvalue { i32, i1 } %val, 0
4323  store i32 %val0, i32 addrspace(1)* %out, align 4
4324  ret void
4325}
4326
4327define amdgpu_kernel void @global_singlethread_one_as_unordered_load(
4328; GFX6-LABEL: global_singlethread_one_as_unordered_load:
4329; GFX6:       ; %bb.0: ; %entry
4330; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4331; GFX6-NEXT:    s_mov_b32 s7, 0x100f000
4332; GFX6-NEXT:    s_mov_b32 s6, -1
4333; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4334; GFX6-NEXT:    s_mov_b32 s4, s0
4335; GFX6-NEXT:    s_mov_b32 s5, s1
4336; GFX6-NEXT:    buffer_load_dword v0, off, s[4:7], 0
4337; GFX6-NEXT:    s_mov_b32 s4, s2
4338; GFX6-NEXT:    s_mov_b32 s5, s3
4339; GFX6-NEXT:    s_waitcnt vmcnt(0)
4340; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
4341; GFX6-NEXT:    s_endpgm
4342;
4343; GFX7-LABEL: global_singlethread_one_as_unordered_load:
4344; GFX7:       ; %bb.0: ; %entry
4345; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4346; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4347; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4348; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4349; GFX7-NEXT:    flat_load_dword v2, v[0:1]
4350; GFX7-NEXT:    v_mov_b32_e32 v0, s2
4351; GFX7-NEXT:    v_mov_b32_e32 v1, s3
4352; GFX7-NEXT:    s_waitcnt vmcnt(0)
4353; GFX7-NEXT:    flat_store_dword v[0:1], v2
4354; GFX7-NEXT:    s_endpgm
4355;
4356; GFX10-WGP-LABEL: global_singlethread_one_as_unordered_load:
4357; GFX10-WGP:       ; %bb.0: ; %entry
4358; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4359; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
4360; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4361; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
4362; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4363; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
4364; GFX10-WGP-NEXT:    s_endpgm
4365;
4366; GFX10-CU-LABEL: global_singlethread_one_as_unordered_load:
4367; GFX10-CU:       ; %bb.0: ; %entry
4368; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4369; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
4370; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4371; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
4372; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4373; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
4374; GFX10-CU-NEXT:    s_endpgm
4375;
4376; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_unordered_load:
4377; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4378; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4379; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
4380; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
4381; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4382; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s0
4383; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s1
4384; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[4:7], 0
4385; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s2
4386; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s3
4387; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4388; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
4389; SKIP-CACHE-INV-NEXT:    s_endpgm
4390;
4391; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_unordered_load:
4392; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4393; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4394; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4395; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4396; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
4397; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4398; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
4399; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4400;
4401; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_unordered_load:
4402; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4403; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4404; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4405; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4406; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
4407; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4408; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
4409; GFX90A-TGSPLIT-NEXT:    s_endpgm
4410    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
4411entry:
4412  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread-one-as") unordered, align 4
4413  store i32 %val, i32 addrspace(1)* %out
4414  ret void
4415}
4416
4417define amdgpu_kernel void @global_singlethread_one_as_monotonic_load(
4418; GFX6-LABEL: global_singlethread_one_as_monotonic_load:
4419; GFX6:       ; %bb.0: ; %entry
4420; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4421; GFX6-NEXT:    s_mov_b32 s7, 0x100f000
4422; GFX6-NEXT:    s_mov_b32 s6, -1
4423; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4424; GFX6-NEXT:    s_mov_b32 s4, s0
4425; GFX6-NEXT:    s_mov_b32 s5, s1
4426; GFX6-NEXT:    buffer_load_dword v0, off, s[4:7], 0
4427; GFX6-NEXT:    s_mov_b32 s4, s2
4428; GFX6-NEXT:    s_mov_b32 s5, s3
4429; GFX6-NEXT:    s_waitcnt vmcnt(0)
4430; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
4431; GFX6-NEXT:    s_endpgm
4432;
4433; GFX7-LABEL: global_singlethread_one_as_monotonic_load:
4434; GFX7:       ; %bb.0: ; %entry
4435; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4436; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4437; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4438; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4439; GFX7-NEXT:    flat_load_dword v2, v[0:1]
4440; GFX7-NEXT:    v_mov_b32_e32 v0, s2
4441; GFX7-NEXT:    v_mov_b32_e32 v1, s3
4442; GFX7-NEXT:    s_waitcnt vmcnt(0)
4443; GFX7-NEXT:    flat_store_dword v[0:1], v2
4444; GFX7-NEXT:    s_endpgm
4445;
4446; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_load:
4447; GFX10-WGP:       ; %bb.0: ; %entry
4448; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4449; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
4450; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4451; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
4452; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4453; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
4454; GFX10-WGP-NEXT:    s_endpgm
4455;
4456; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_load:
4457; GFX10-CU:       ; %bb.0: ; %entry
4458; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4459; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
4460; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4461; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
4462; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4463; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
4464; GFX10-CU-NEXT:    s_endpgm
4465;
4466; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_load:
4467; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4468; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4469; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
4470; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
4471; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4472; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s0
4473; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s1
4474; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[4:7], 0
4475; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s2
4476; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s3
4477; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4478; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
4479; SKIP-CACHE-INV-NEXT:    s_endpgm
4480;
4481; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_load:
4482; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4483; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4484; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4485; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4486; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
4487; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4488; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
4489; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4490;
4491; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_load:
4492; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4493; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4494; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4495; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4496; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
4497; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4498; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
4499; GFX90A-TGSPLIT-NEXT:    s_endpgm
4500    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
4501entry:
4502  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread-one-as") monotonic, align 4
4503  store i32 %val, i32 addrspace(1)* %out
4504  ret void
4505}
4506
4507define amdgpu_kernel void @global_singlethread_one_as_acquire_load(
4508; GFX6-LABEL: global_singlethread_one_as_acquire_load:
4509; GFX6:       ; %bb.0: ; %entry
4510; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4511; GFX6-NEXT:    s_mov_b32 s7, 0x100f000
4512; GFX6-NEXT:    s_mov_b32 s6, -1
4513; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4514; GFX6-NEXT:    s_mov_b32 s4, s0
4515; GFX6-NEXT:    s_mov_b32 s5, s1
4516; GFX6-NEXT:    buffer_load_dword v0, off, s[4:7], 0
4517; GFX6-NEXT:    s_mov_b32 s4, s2
4518; GFX6-NEXT:    s_mov_b32 s5, s3
4519; GFX6-NEXT:    s_waitcnt vmcnt(0)
4520; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
4521; GFX6-NEXT:    s_endpgm
4522;
4523; GFX7-LABEL: global_singlethread_one_as_acquire_load:
4524; GFX7:       ; %bb.0: ; %entry
4525; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4526; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4527; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4528; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4529; GFX7-NEXT:    flat_load_dword v2, v[0:1]
4530; GFX7-NEXT:    v_mov_b32_e32 v0, s2
4531; GFX7-NEXT:    v_mov_b32_e32 v1, s3
4532; GFX7-NEXT:    s_waitcnt vmcnt(0)
4533; GFX7-NEXT:    flat_store_dword v[0:1], v2
4534; GFX7-NEXT:    s_endpgm
4535;
4536; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_load:
4537; GFX10-WGP:       ; %bb.0: ; %entry
4538; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4539; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
4540; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4541; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
4542; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4543; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
4544; GFX10-WGP-NEXT:    s_endpgm
4545;
4546; GFX10-CU-LABEL: global_singlethread_one_as_acquire_load:
4547; GFX10-CU:       ; %bb.0: ; %entry
4548; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4549; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
4550; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4551; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
4552; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4553; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
4554; GFX10-CU-NEXT:    s_endpgm
4555;
4556; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_load:
4557; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4558; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4559; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
4560; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
4561; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4562; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s0
4563; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s1
4564; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[4:7], 0
4565; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s2
4566; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s3
4567; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4568; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
4569; SKIP-CACHE-INV-NEXT:    s_endpgm
4570;
4571; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_load:
4572; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4573; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4574; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4575; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4576; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
4577; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4578; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
4579; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4580;
4581; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_load:
4582; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4583; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4584; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4585; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4586; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
4587; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4588; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
4589; GFX90A-TGSPLIT-NEXT:    s_endpgm
4590    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
4591entry:
4592  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread-one-as") acquire, align 4
4593  store i32 %val, i32 addrspace(1)* %out
4594  ret void
4595}
4596
4597define amdgpu_kernel void @global_singlethread_one_as_seq_cst_load(
4598; GFX6-LABEL: global_singlethread_one_as_seq_cst_load:
4599; GFX6:       ; %bb.0: ; %entry
4600; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4601; GFX6-NEXT:    s_mov_b32 s7, 0x100f000
4602; GFX6-NEXT:    s_mov_b32 s6, -1
4603; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4604; GFX6-NEXT:    s_mov_b32 s4, s0
4605; GFX6-NEXT:    s_mov_b32 s5, s1
4606; GFX6-NEXT:    buffer_load_dword v0, off, s[4:7], 0
4607; GFX6-NEXT:    s_mov_b32 s4, s2
4608; GFX6-NEXT:    s_mov_b32 s5, s3
4609; GFX6-NEXT:    s_waitcnt vmcnt(0)
4610; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
4611; GFX6-NEXT:    s_endpgm
4612;
4613; GFX7-LABEL: global_singlethread_one_as_seq_cst_load:
4614; GFX7:       ; %bb.0: ; %entry
4615; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4616; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4617; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4618; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4619; GFX7-NEXT:    flat_load_dword v2, v[0:1]
4620; GFX7-NEXT:    v_mov_b32_e32 v0, s2
4621; GFX7-NEXT:    v_mov_b32_e32 v1, s3
4622; GFX7-NEXT:    s_waitcnt vmcnt(0)
4623; GFX7-NEXT:    flat_store_dword v[0:1], v2
4624; GFX7-NEXT:    s_endpgm
4625;
4626; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_load:
4627; GFX10-WGP:       ; %bb.0: ; %entry
4628; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4629; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
4630; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4631; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
4632; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4633; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
4634; GFX10-WGP-NEXT:    s_endpgm
4635;
4636; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_load:
4637; GFX10-CU:       ; %bb.0: ; %entry
4638; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4639; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
4640; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4641; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
4642; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4643; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
4644; GFX10-CU-NEXT:    s_endpgm
4645;
4646; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_load:
4647; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4648; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4649; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
4650; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
4651; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4652; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s0
4653; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s1
4654; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[4:7], 0
4655; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s2
4656; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s3
4657; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4658; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
4659; SKIP-CACHE-INV-NEXT:    s_endpgm
4660;
4661; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_load:
4662; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4663; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4664; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4665; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4666; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
4667; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4668; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
4669; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4670;
4671; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_load:
4672; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4673; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4674; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4675; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4676; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
4677; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4678; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
4679; GFX90A-TGSPLIT-NEXT:    s_endpgm
4680    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
4681entry:
4682  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread-one-as") seq_cst, align 4
4683  store i32 %val, i32 addrspace(1)* %out
4684  ret void
4685}
4686
4687define amdgpu_kernel void @global_singlethread_one_as_unordered_store(
4688; GFX6-LABEL: global_singlethread_one_as_unordered_store:
4689; GFX6:       ; %bb.0: ; %entry
4690; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
4691; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
4692; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4693; GFX6-NEXT:    s_mov_b32 s2, -1
4694; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4695; GFX6-NEXT:    v_mov_b32_e32 v0, s6
4696; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4697; GFX6-NEXT:    s_endpgm
4698;
4699; GFX7-LABEL: global_singlethread_one_as_unordered_store:
4700; GFX7:       ; %bb.0: ; %entry
4701; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
4702; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
4703; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4704; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4705; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4706; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4707; GFX7-NEXT:    flat_store_dword v[0:1], v2
4708; GFX7-NEXT:    s_endpgm
4709;
4710; GFX10-WGP-LABEL: global_singlethread_one_as_unordered_store:
4711; GFX10-WGP:       ; %bb.0: ; %entry
4712; GFX10-WGP-NEXT:    s_clause 0x1
4713; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
4714; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4715; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
4716; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4717; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
4718; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
4719; GFX10-WGP-NEXT:    s_endpgm
4720;
4721; GFX10-CU-LABEL: global_singlethread_one_as_unordered_store:
4722; GFX10-CU:       ; %bb.0: ; %entry
4723; GFX10-CU-NEXT:    s_clause 0x1
4724; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
4725; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4726; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
4727; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4728; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
4729; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
4730; GFX10-CU-NEXT:    s_endpgm
4731;
4732; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_unordered_store:
4733; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4734; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x0
4735; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4736; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
4737; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
4738; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4739; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4740; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4741; SKIP-CACHE-INV-NEXT:    s_endpgm
4742;
4743; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_unordered_store:
4744; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4745; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
4746; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4747; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4748; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4749; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4750; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
4751; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4752;
4753; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_unordered_store:
4754; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4755; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
4756; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4757; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4758; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4759; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4760; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
4761; GFX90A-TGSPLIT-NEXT:    s_endpgm
4762    i32 %in, i32 addrspace(1)* %out) {
4763entry:
4764  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread-one-as") unordered, align 4
4765  ret void
4766}
4767
4768define amdgpu_kernel void @global_singlethread_one_as_monotonic_store(
4769; GFX6-LABEL: global_singlethread_one_as_monotonic_store:
4770; GFX6:       ; %bb.0: ; %entry
4771; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
4772; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
4773; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4774; GFX6-NEXT:    s_mov_b32 s2, -1
4775; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4776; GFX6-NEXT:    v_mov_b32_e32 v0, s6
4777; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4778; GFX6-NEXT:    s_endpgm
4779;
4780; GFX7-LABEL: global_singlethread_one_as_monotonic_store:
4781; GFX7:       ; %bb.0: ; %entry
4782; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
4783; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
4784; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4785; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4786; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4787; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4788; GFX7-NEXT:    flat_store_dword v[0:1], v2
4789; GFX7-NEXT:    s_endpgm
4790;
4791; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_store:
4792; GFX10-WGP:       ; %bb.0: ; %entry
4793; GFX10-WGP-NEXT:    s_clause 0x1
4794; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
4795; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4796; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
4797; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4798; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
4799; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
4800; GFX10-WGP-NEXT:    s_endpgm
4801;
4802; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_store:
4803; GFX10-CU:       ; %bb.0: ; %entry
4804; GFX10-CU-NEXT:    s_clause 0x1
4805; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
4806; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4807; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
4808; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4809; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
4810; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
4811; GFX10-CU-NEXT:    s_endpgm
4812;
4813; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_store:
4814; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4815; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x0
4816; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4817; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
4818; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
4819; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4820; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4821; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4822; SKIP-CACHE-INV-NEXT:    s_endpgm
4823;
4824; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_store:
4825; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4826; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
4827; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4828; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4829; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4830; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4831; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
4832; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4833;
4834; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_store:
4835; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4836; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
4837; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4838; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4839; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4840; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4841; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
4842; GFX90A-TGSPLIT-NEXT:    s_endpgm
4843    i32 %in, i32 addrspace(1)* %out) {
4844entry:
4845  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread-one-as") monotonic, align 4
4846  ret void
4847}
4848
4849define amdgpu_kernel void @global_singlethread_one_as_release_store(
4850; GFX6-LABEL: global_singlethread_one_as_release_store:
4851; GFX6:       ; %bb.0: ; %entry
4852; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
4853; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
4854; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4855; GFX6-NEXT:    s_mov_b32 s2, -1
4856; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4857; GFX6-NEXT:    v_mov_b32_e32 v0, s6
4858; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4859; GFX6-NEXT:    s_endpgm
4860;
4861; GFX7-LABEL: global_singlethread_one_as_release_store:
4862; GFX7:       ; %bb.0: ; %entry
4863; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
4864; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
4865; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4866; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4867; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4868; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4869; GFX7-NEXT:    flat_store_dword v[0:1], v2
4870; GFX7-NEXT:    s_endpgm
4871;
4872; GFX10-WGP-LABEL: global_singlethread_one_as_release_store:
4873; GFX10-WGP:       ; %bb.0: ; %entry
4874; GFX10-WGP-NEXT:    s_clause 0x1
4875; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
4876; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4877; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
4878; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4879; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
4880; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
4881; GFX10-WGP-NEXT:    s_endpgm
4882;
4883; GFX10-CU-LABEL: global_singlethread_one_as_release_store:
4884; GFX10-CU:       ; %bb.0: ; %entry
4885; GFX10-CU-NEXT:    s_clause 0x1
4886; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
4887; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4888; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
4889; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4890; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
4891; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
4892; GFX10-CU-NEXT:    s_endpgm
4893;
4894; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_store:
4895; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4896; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x0
4897; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4898; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
4899; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
4900; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4901; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4902; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4903; SKIP-CACHE-INV-NEXT:    s_endpgm
4904;
4905; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_store:
4906; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4907; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
4908; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4909; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4910; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4911; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4912; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
4913; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4914;
4915; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_release_store:
4916; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4917; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
4918; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4919; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4920; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4921; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4922; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
4923; GFX90A-TGSPLIT-NEXT:    s_endpgm
4924    i32 %in, i32 addrspace(1)* %out) {
4925entry:
4926  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread-one-as") release, align 4
4927  ret void
4928}
4929
4930define amdgpu_kernel void @global_singlethread_one_as_seq_cst_store(
4931; GFX6-LABEL: global_singlethread_one_as_seq_cst_store:
4932; GFX6:       ; %bb.0: ; %entry
4933; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
4934; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
4935; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4936; GFX6-NEXT:    s_mov_b32 s2, -1
4937; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4938; GFX6-NEXT:    v_mov_b32_e32 v0, s6
4939; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4940; GFX6-NEXT:    s_endpgm
4941;
4942; GFX7-LABEL: global_singlethread_one_as_seq_cst_store:
4943; GFX7:       ; %bb.0: ; %entry
4944; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
4945; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
4946; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4947; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4948; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4949; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4950; GFX7-NEXT:    flat_store_dword v[0:1], v2
4951; GFX7-NEXT:    s_endpgm
4952;
4953; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_store:
4954; GFX10-WGP:       ; %bb.0: ; %entry
4955; GFX10-WGP-NEXT:    s_clause 0x1
4956; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
4957; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4958; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
4959; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4960; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
4961; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
4962; GFX10-WGP-NEXT:    s_endpgm
4963;
4964; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_store:
4965; GFX10-CU:       ; %bb.0: ; %entry
4966; GFX10-CU-NEXT:    s_clause 0x1
4967; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
4968; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4969; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
4970; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4971; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
4972; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
4973; GFX10-CU-NEXT:    s_endpgm
4974;
4975; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_store:
4976; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4977; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x0
4978; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4979; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
4980; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
4981; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4982; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4983; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4984; SKIP-CACHE-INV-NEXT:    s_endpgm
4985;
4986; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_store:
4987; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4988; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
4989; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4990; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4991; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4992; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4993; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
4994; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4995;
4996; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_store:
4997; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4998; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
4999; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5000; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5001; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5002; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5003; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
5004; GFX90A-TGSPLIT-NEXT:    s_endpgm
5005    i32 %in, i32 addrspace(1)* %out) {
5006entry:
5007  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread-one-as") seq_cst, align 4
5008  ret void
5009}
5010
5011define amdgpu_kernel void @global_singlethread_one_as_monotonic_atomicrmw(
5012; GFX6-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
5013; GFX6:       ; %bb.0: ; %entry
5014; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
5015; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5016; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5017; GFX6-NEXT:    s_mov_b32 s2, -1
5018; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5019; GFX6-NEXT:    v_mov_b32_e32 v0, s6
5020; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
5021; GFX6-NEXT:    s_endpgm
5022;
5023; GFX7-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
5024; GFX7:       ; %bb.0: ; %entry
5025; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5026; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
5027; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5028; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5029; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5030; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5031; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
5032; GFX7-NEXT:    s_endpgm
5033;
5034; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
5035; GFX10-WGP:       ; %bb.0: ; %entry
5036; GFX10-WGP-NEXT:    s_clause 0x1
5037; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
5038; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5039; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
5040; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5041; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5042; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
5043; GFX10-WGP-NEXT:    s_endpgm
5044;
5045; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
5046; GFX10-CU:       ; %bb.0: ; %entry
5047; GFX10-CU-NEXT:    s_clause 0x1
5048; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
5049; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5050; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
5051; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5052; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5053; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
5054; GFX10-CU-NEXT:    s_endpgm
5055;
5056; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
5057; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5058; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
5059; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
5060; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
5061; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
5062; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5063; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5064; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
5065; SKIP-CACHE-INV-NEXT:    s_endpgm
5066;
5067; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
5068; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5069; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5070; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5071; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5072; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5073; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5074; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
5075; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5076;
5077; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
5078; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5079; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5080; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5081; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5082; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5083; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5084; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
5085; GFX90A-TGSPLIT-NEXT:    s_endpgm
5086    i32 addrspace(1)* %out, i32 %in) {
5087entry:
5088  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") monotonic
5089  ret void
5090}
5091
5092define amdgpu_kernel void @global_singlethread_one_as_acquire_atomicrmw(
5093; GFX6-LABEL: global_singlethread_one_as_acquire_atomicrmw:
5094; GFX6:       ; %bb.0: ; %entry
5095; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
5096; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5097; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5098; GFX6-NEXT:    s_mov_b32 s2, -1
5099; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5100; GFX6-NEXT:    v_mov_b32_e32 v0, s6
5101; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
5102; GFX6-NEXT:    s_endpgm
5103;
5104; GFX7-LABEL: global_singlethread_one_as_acquire_atomicrmw:
5105; GFX7:       ; %bb.0: ; %entry
5106; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5107; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
5108; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5109; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5110; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5111; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5112; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
5113; GFX7-NEXT:    s_endpgm
5114;
5115; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_atomicrmw:
5116; GFX10-WGP:       ; %bb.0: ; %entry
5117; GFX10-WGP-NEXT:    s_clause 0x1
5118; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
5119; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5120; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
5121; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5122; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5123; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
5124; GFX10-WGP-NEXT:    s_endpgm
5125;
5126; GFX10-CU-LABEL: global_singlethread_one_as_acquire_atomicrmw:
5127; GFX10-CU:       ; %bb.0: ; %entry
5128; GFX10-CU-NEXT:    s_clause 0x1
5129; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
5130; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5131; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
5132; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5133; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5134; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
5135; GFX10-CU-NEXT:    s_endpgm
5136;
5137; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_atomicrmw:
5138; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5139; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
5140; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
5141; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
5142; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
5143; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5144; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5145; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
5146; SKIP-CACHE-INV-NEXT:    s_endpgm
5147;
5148; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_atomicrmw:
5149; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5150; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5151; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5152; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5153; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5154; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5155; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
5156; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5157;
5158; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_atomicrmw:
5159; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5160; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5161; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5162; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5163; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5164; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5165; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
5166; GFX90A-TGSPLIT-NEXT:    s_endpgm
5167    i32 addrspace(1)* %out, i32 %in) {
5168entry:
5169  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") acquire
5170  ret void
5171}
5172
5173define amdgpu_kernel void @global_singlethread_one_as_release_atomicrmw(
5174; GFX6-LABEL: global_singlethread_one_as_release_atomicrmw:
5175; GFX6:       ; %bb.0: ; %entry
5176; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
5177; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5178; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5179; GFX6-NEXT:    s_mov_b32 s2, -1
5180; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5181; GFX6-NEXT:    v_mov_b32_e32 v0, s6
5182; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
5183; GFX6-NEXT:    s_endpgm
5184;
5185; GFX7-LABEL: global_singlethread_one_as_release_atomicrmw:
5186; GFX7:       ; %bb.0: ; %entry
5187; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5188; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
5189; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5190; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5191; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5192; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5193; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
5194; GFX7-NEXT:    s_endpgm
5195;
5196; GFX10-WGP-LABEL: global_singlethread_one_as_release_atomicrmw:
5197; GFX10-WGP:       ; %bb.0: ; %entry
5198; GFX10-WGP-NEXT:    s_clause 0x1
5199; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
5200; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5201; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
5202; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5203; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5204; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
5205; GFX10-WGP-NEXT:    s_endpgm
5206;
5207; GFX10-CU-LABEL: global_singlethread_one_as_release_atomicrmw:
5208; GFX10-CU:       ; %bb.0: ; %entry
5209; GFX10-CU-NEXT:    s_clause 0x1
5210; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
5211; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5212; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
5213; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5214; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5215; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
5216; GFX10-CU-NEXT:    s_endpgm
5217;
5218; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_atomicrmw:
5219; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5220; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
5221; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
5222; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
5223; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
5224; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5225; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5226; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
5227; SKIP-CACHE-INV-NEXT:    s_endpgm
5228;
5229; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_atomicrmw:
5230; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5231; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5232; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5233; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5234; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5235; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5236; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
5237; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5238;
5239; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_release_atomicrmw:
5240; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5241; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5242; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5243; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5244; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5245; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5246; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
5247; GFX90A-TGSPLIT-NEXT:    s_endpgm
5248    i32 addrspace(1)* %out, i32 %in) {
5249entry:
5250  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") release
5251  ret void
5252}
5253
5254define amdgpu_kernel void @global_singlethread_one_as_acq_rel_atomicrmw(
5255; GFX6-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
5256; GFX6:       ; %bb.0: ; %entry
5257; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
5258; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5259; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5260; GFX6-NEXT:    s_mov_b32 s2, -1
5261; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5262; GFX6-NEXT:    v_mov_b32_e32 v0, s6
5263; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
5264; GFX6-NEXT:    s_endpgm
5265;
5266; GFX7-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
5267; GFX7:       ; %bb.0: ; %entry
5268; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5269; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
5270; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5271; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5272; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5273; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5274; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
5275; GFX7-NEXT:    s_endpgm
5276;
5277; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
5278; GFX10-WGP:       ; %bb.0: ; %entry
5279; GFX10-WGP-NEXT:    s_clause 0x1
5280; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
5281; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5282; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
5283; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5284; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5285; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
5286; GFX10-WGP-NEXT:    s_endpgm
5287;
5288; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
5289; GFX10-CU:       ; %bb.0: ; %entry
5290; GFX10-CU-NEXT:    s_clause 0x1
5291; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
5292; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5293; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
5294; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5295; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5296; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
5297; GFX10-CU-NEXT:    s_endpgm
5298;
5299; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
5300; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5301; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
5302; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
5303; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
5304; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
5305; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5306; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5307; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
5308; SKIP-CACHE-INV-NEXT:    s_endpgm
5309;
5310; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
5311; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5312; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5313; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5314; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5315; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5316; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5317; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
5318; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5319;
5320; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
5321; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5322; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5323; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5324; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5325; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5326; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5327; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
5328; GFX90A-TGSPLIT-NEXT:    s_endpgm
5329    i32 addrspace(1)* %out, i32 %in) {
5330entry:
5331  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") acq_rel
5332  ret void
5333}
5334
5335define amdgpu_kernel void @global_singlethread_one_as_seq_cst_atomicrmw(
5336; GFX6-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
5337; GFX6:       ; %bb.0: ; %entry
5338; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
5339; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5340; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5341; GFX6-NEXT:    s_mov_b32 s2, -1
5342; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5343; GFX6-NEXT:    v_mov_b32_e32 v0, s6
5344; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
5345; GFX6-NEXT:    s_endpgm
5346;
5347; GFX7-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
5348; GFX7:       ; %bb.0: ; %entry
5349; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5350; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
5351; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5352; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5353; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5354; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5355; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
5356; GFX7-NEXT:    s_endpgm
5357;
5358; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
5359; GFX10-WGP:       ; %bb.0: ; %entry
5360; GFX10-WGP-NEXT:    s_clause 0x1
5361; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
5362; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5363; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
5364; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5365; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5366; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
5367; GFX10-WGP-NEXT:    s_endpgm
5368;
5369; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
5370; GFX10-CU:       ; %bb.0: ; %entry
5371; GFX10-CU-NEXT:    s_clause 0x1
5372; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
5373; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5374; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
5375; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5376; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5377; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
5378; GFX10-CU-NEXT:    s_endpgm
5379;
5380; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
5381; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5382; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
5383; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
5384; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
5385; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
5386; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5387; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5388; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
5389; SKIP-CACHE-INV-NEXT:    s_endpgm
5390;
5391; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
5392; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5393; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5394; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5395; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5396; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5397; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5398; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
5399; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5400;
5401; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
5402; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5403; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5404; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5405; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5406; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5407; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5408; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
5409; GFX90A-TGSPLIT-NEXT:    s_endpgm
5410    i32 addrspace(1)* %out, i32 %in) {
5411entry:
5412  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") seq_cst
5413  ret void
5414}
5415
5416define amdgpu_kernel void @global_singlethread_one_as_acquire_ret_atomicrmw(
5417; GFX6-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
5418; GFX6:       ; %bb.0: ; %entry
5419; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
5420; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5421; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5422; GFX6-NEXT:    s_mov_b32 s2, -1
5423; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5424; GFX6-NEXT:    v_mov_b32_e32 v0, s6
5425; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
5426; GFX6-NEXT:    s_waitcnt vmcnt(0)
5427; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5428; GFX6-NEXT:    s_endpgm
5429;
5430; GFX7-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
5431; GFX7:       ; %bb.0: ; %entry
5432; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5433; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
5434; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5435; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5436; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5437; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5438; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
5439; GFX7-NEXT:    s_waitcnt vmcnt(0)
5440; GFX7-NEXT:    flat_store_dword v[0:1], v2
5441; GFX7-NEXT:    s_endpgm
5442;
5443; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
5444; GFX10-WGP:       ; %bb.0: ; %entry
5445; GFX10-WGP-NEXT:    s_clause 0x1
5446; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
5447; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5448; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
5449; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5450; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5451; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
5452; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5453; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
5454; GFX10-WGP-NEXT:    s_endpgm
5455;
5456; GFX10-CU-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
5457; GFX10-CU:       ; %bb.0: ; %entry
5458; GFX10-CU-NEXT:    s_clause 0x1
5459; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
5460; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5461; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
5462; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5463; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5464; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
5465; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5466; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
5467; GFX10-CU-NEXT:    s_endpgm
5468;
5469; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
5470; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5471; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
5472; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
5473; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
5474; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
5475; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5476; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5477; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
5478; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5479; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5480; SKIP-CACHE-INV-NEXT:    s_endpgm
5481;
5482; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
5483; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5484; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5485; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5486; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5487; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5488; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5489; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
5490; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5491; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
5492; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5493;
5494; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
5495; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5496; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5497; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5498; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5499; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5500; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5501; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
5502; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5503; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
5504; GFX90A-TGSPLIT-NEXT:    s_endpgm
5505    i32 addrspace(1)* %out, i32 %in) {
5506entry:
5507  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") acquire
5508  store i32 %val, i32 addrspace(1)* %out, align 4
5509  ret void
5510}
5511
5512define amdgpu_kernel void @global_singlethread_one_as_acq_rel_ret_atomicrmw(
5513; GFX6-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
5514; GFX6:       ; %bb.0: ; %entry
5515; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
5516; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5517; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5518; GFX6-NEXT:    s_mov_b32 s2, -1
5519; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5520; GFX6-NEXT:    v_mov_b32_e32 v0, s6
5521; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
5522; GFX6-NEXT:    s_waitcnt vmcnt(0)
5523; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5524; GFX6-NEXT:    s_endpgm
5525;
5526; GFX7-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
5527; GFX7:       ; %bb.0: ; %entry
5528; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5529; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
5530; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5531; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5532; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5533; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5534; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
5535; GFX7-NEXT:    s_waitcnt vmcnt(0)
5536; GFX7-NEXT:    flat_store_dword v[0:1], v2
5537; GFX7-NEXT:    s_endpgm
5538;
5539; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
5540; GFX10-WGP:       ; %bb.0: ; %entry
5541; GFX10-WGP-NEXT:    s_clause 0x1
5542; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
5543; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5544; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
5545; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5546; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5547; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
5548; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5549; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
5550; GFX10-WGP-NEXT:    s_endpgm
5551;
5552; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
5553; GFX10-CU:       ; %bb.0: ; %entry
5554; GFX10-CU-NEXT:    s_clause 0x1
5555; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
5556; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5557; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
5558; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5559; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5560; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
5561; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5562; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
5563; GFX10-CU-NEXT:    s_endpgm
5564;
5565; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
5566; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5567; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
5568; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
5569; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
5570; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
5571; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5572; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5573; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
5574; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5575; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5576; SKIP-CACHE-INV-NEXT:    s_endpgm
5577;
5578; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
5579; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5580; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5581; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5582; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5583; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5584; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5585; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
5586; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5587; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
5588; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5589;
5590; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
5591; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5592; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5593; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5594; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5595; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5596; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5597; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
5598; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5599; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
5600; GFX90A-TGSPLIT-NEXT:    s_endpgm
5601    i32 addrspace(1)* %out, i32 %in) {
5602entry:
5603  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") acq_rel
5604  store i32 %val, i32 addrspace(1)* %out, align 4
5605  ret void
5606}
5607
5608define amdgpu_kernel void @global_singlethread_one_as_seq_cst_ret_atomicrmw(
5609; GFX6-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
5610; GFX6:       ; %bb.0: ; %entry
5611; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
5612; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5613; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5614; GFX6-NEXT:    s_mov_b32 s2, -1
5615; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5616; GFX6-NEXT:    v_mov_b32_e32 v0, s6
5617; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
5618; GFX6-NEXT:    s_waitcnt vmcnt(0)
5619; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5620; GFX6-NEXT:    s_endpgm
5621;
5622; GFX7-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
5623; GFX7:       ; %bb.0: ; %entry
5624; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5625; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
5626; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5627; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5628; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5629; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5630; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
5631; GFX7-NEXT:    s_waitcnt vmcnt(0)
5632; GFX7-NEXT:    flat_store_dword v[0:1], v2
5633; GFX7-NEXT:    s_endpgm
5634;
5635; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
5636; GFX10-WGP:       ; %bb.0: ; %entry
5637; GFX10-WGP-NEXT:    s_clause 0x1
5638; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
5639; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5640; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
5641; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5642; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5643; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
5644; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5645; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
5646; GFX10-WGP-NEXT:    s_endpgm
5647;
5648; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
5649; GFX10-CU:       ; %bb.0: ; %entry
5650; GFX10-CU-NEXT:    s_clause 0x1
5651; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
5652; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5653; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
5654; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5655; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5656; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
5657; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5658; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
5659; GFX10-CU-NEXT:    s_endpgm
5660;
5661; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
5662; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5663; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
5664; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
5665; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
5666; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
5667; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5668; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5669; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
5670; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5671; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5672; SKIP-CACHE-INV-NEXT:    s_endpgm
5673;
5674; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
5675; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5676; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5677; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5678; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5679; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5680; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5681; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
5682; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5683; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
5684; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5685;
5686; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
5687; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5688; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5689; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5690; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5691; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5692; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5693; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
5694; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5695; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
5696; GFX90A-TGSPLIT-NEXT:    s_endpgm
5697    i32 addrspace(1)* %out, i32 %in) {
5698entry:
5699  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") seq_cst
5700  store i32 %val, i32 addrspace(1)* %out, align 4
5701  ret void
5702}
5703
5704define amdgpu_kernel void @global_singlethread_one_as_monotonic_monotonic_cmpxchg(
5705; GFX6-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
5706; GFX6:       ; %bb.0: ; %entry
5707; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
5708; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5709; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5710; GFX6-NEXT:    s_mov_b32 s2, -1
5711; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5712; GFX6-NEXT:    v_mov_b32_e32 v0, s6
5713; GFX6-NEXT:    v_mov_b32_e32 v1, s7
5714; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
5715; GFX6-NEXT:    s_endpgm
5716;
5717; GFX7-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
5718; GFX7:       ; %bb.0: ; %entry
5719; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5720; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5721; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5722; GFX7-NEXT:    s_add_u32 s0, s0, 16
5723; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5724; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5725; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5726; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5727; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5728; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5729; GFX7-NEXT:    s_endpgm
5730;
5731; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
5732; GFX10-WGP:       ; %bb.0: ; %entry
5733; GFX10-WGP-NEXT:    s_clause 0x1
5734; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5735; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5736; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
5737; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5738; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5739; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5740; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
5741; GFX10-WGP-NEXT:    s_endpgm
5742;
5743; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
5744; GFX10-CU:       ; %bb.0: ; %entry
5745; GFX10-CU-NEXT:    s_clause 0x1
5746; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5747; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5748; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
5749; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5750; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5751; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5752; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
5753; GFX10-CU-NEXT:    s_endpgm
5754;
5755; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
5756; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5757; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
5758; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
5759; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
5760; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
5761; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5762; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5763; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5764; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
5765; SKIP-CACHE-INV-NEXT:    s_endpgm
5766;
5767; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
5768; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5769; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5770; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5771; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5772; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5773; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5774; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
5775; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5776;
5777; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
5778; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5779; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5780; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5781; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5782; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5783; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5784; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
5785; GFX90A-TGSPLIT-NEXT:    s_endpgm
5786    i32 addrspace(1)* %out, i32 %in, i32 %old) {
5787entry:
5788  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
5789  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic monotonic
5790  ret void
5791}
5792
5793define amdgpu_kernel void @global_singlethread_one_as_acquire_monotonic_cmpxchg(
5794; GFX6-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
5795; GFX6:       ; %bb.0: ; %entry
5796; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
5797; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5798; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5799; GFX6-NEXT:    s_mov_b32 s2, -1
5800; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5801; GFX6-NEXT:    v_mov_b32_e32 v0, s6
5802; GFX6-NEXT:    v_mov_b32_e32 v1, s7
5803; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
5804; GFX6-NEXT:    s_endpgm
5805;
5806; GFX7-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
5807; GFX7:       ; %bb.0: ; %entry
5808; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5809; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5810; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5811; GFX7-NEXT:    s_add_u32 s0, s0, 16
5812; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5813; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5814; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5815; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5816; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5817; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5818; GFX7-NEXT:    s_endpgm
5819;
5820; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
5821; GFX10-WGP:       ; %bb.0: ; %entry
5822; GFX10-WGP-NEXT:    s_clause 0x1
5823; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5824; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5825; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
5826; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5827; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5828; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5829; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
5830; GFX10-WGP-NEXT:    s_endpgm
5831;
5832; GFX10-CU-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
5833; GFX10-CU:       ; %bb.0: ; %entry
5834; GFX10-CU-NEXT:    s_clause 0x1
5835; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5836; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5837; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
5838; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5839; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5840; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5841; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
5842; GFX10-CU-NEXT:    s_endpgm
5843;
5844; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
5845; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5846; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
5847; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
5848; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
5849; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
5850; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5851; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5852; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5853; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
5854; SKIP-CACHE-INV-NEXT:    s_endpgm
5855;
5856; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
5857; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5858; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5859; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5860; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5861; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5862; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5863; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
5864; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5865;
5866; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
5867; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5868; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5869; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5870; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5871; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5872; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5873; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
5874; GFX90A-TGSPLIT-NEXT:    s_endpgm
5875    i32 addrspace(1)* %out, i32 %in, i32 %old) {
5876entry:
5877  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
5878  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire monotonic
5879  ret void
5880}
5881
5882define amdgpu_kernel void @global_singlethread_one_as_release_monotonic_cmpxchg(
5883; GFX6-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
5884; GFX6:       ; %bb.0: ; %entry
5885; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
5886; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5887; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5888; GFX6-NEXT:    s_mov_b32 s2, -1
5889; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5890; GFX6-NEXT:    v_mov_b32_e32 v0, s6
5891; GFX6-NEXT:    v_mov_b32_e32 v1, s7
5892; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
5893; GFX6-NEXT:    s_endpgm
5894;
5895; GFX7-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
5896; GFX7:       ; %bb.0: ; %entry
5897; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5898; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5899; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5900; GFX7-NEXT:    s_add_u32 s0, s0, 16
5901; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5902; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5903; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5904; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5905; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5906; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5907; GFX7-NEXT:    s_endpgm
5908;
5909; GFX10-WGP-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
5910; GFX10-WGP:       ; %bb.0: ; %entry
5911; GFX10-WGP-NEXT:    s_clause 0x1
5912; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5913; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5914; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
5915; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5916; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5917; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5918; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
5919; GFX10-WGP-NEXT:    s_endpgm
5920;
5921; GFX10-CU-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
5922; GFX10-CU:       ; %bb.0: ; %entry
5923; GFX10-CU-NEXT:    s_clause 0x1
5924; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5925; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5926; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
5927; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5928; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5929; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5930; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
5931; GFX10-CU-NEXT:    s_endpgm
5932;
5933; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
5934; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5935; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
5936; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
5937; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
5938; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
5939; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5940; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5941; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5942; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
5943; SKIP-CACHE-INV-NEXT:    s_endpgm
5944;
5945; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
5946; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5947; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5948; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5949; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5950; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5951; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5952; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
5953; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5954;
5955; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
5956; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5957; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5958; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5959; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5960; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5961; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5962; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
5963; GFX90A-TGSPLIT-NEXT:    s_endpgm
5964    i32 addrspace(1)* %out, i32 %in, i32 %old) {
5965entry:
5966  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
5967  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release monotonic
5968  ret void
5969}
5970
5971define amdgpu_kernel void @global_singlethread_one_as_acq_rel_monotonic_cmpxchg(
5972; GFX6-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
5973; GFX6:       ; %bb.0: ; %entry
5974; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
5975; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5976; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5977; GFX6-NEXT:    s_mov_b32 s2, -1
5978; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5979; GFX6-NEXT:    v_mov_b32_e32 v0, s6
5980; GFX6-NEXT:    v_mov_b32_e32 v1, s7
5981; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
5982; GFX6-NEXT:    s_endpgm
5983;
5984; GFX7-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
5985; GFX7:       ; %bb.0: ; %entry
5986; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5987; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5988; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5989; GFX7-NEXT:    s_add_u32 s0, s0, 16
5990; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5991; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5992; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5993; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5994; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5995; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5996; GFX7-NEXT:    s_endpgm
5997;
5998; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
5999; GFX10-WGP:       ; %bb.0: ; %entry
6000; GFX10-WGP-NEXT:    s_clause 0x1
6001; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6002; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6003; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
6004; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6005; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6006; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6007; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6008; GFX10-WGP-NEXT:    s_endpgm
6009;
6010; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
6011; GFX10-CU:       ; %bb.0: ; %entry
6012; GFX10-CU-NEXT:    s_clause 0x1
6013; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6014; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6015; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
6016; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6017; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6018; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6019; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6020; GFX10-CU-NEXT:    s_endpgm
6021;
6022; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
6023; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6024; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
6025; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6026; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
6027; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
6028; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6029; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6030; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6031; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
6032; SKIP-CACHE-INV-NEXT:    s_endpgm
6033;
6034; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
6035; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6036; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6037; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6038; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6039; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6040; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6041; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6042; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6043;
6044; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
6045; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6046; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6047; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6048; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6049; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6050; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6051; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6052; GFX90A-TGSPLIT-NEXT:    s_endpgm
6053    i32 addrspace(1)* %out, i32 %in, i32 %old) {
6054entry:
6055  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
6056  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel monotonic
6057  ret void
6058}
6059
6060define amdgpu_kernel void @global_singlethread_one_as_seq_cst_monotonic_cmpxchg(
6061; GFX6-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
6062; GFX6:       ; %bb.0: ; %entry
6063; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
6064; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6065; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
6066; GFX6-NEXT:    s_mov_b32 s2, -1
6067; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6068; GFX6-NEXT:    v_mov_b32_e32 v0, s6
6069; GFX6-NEXT:    v_mov_b32_e32 v1, s7
6070; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
6071; GFX6-NEXT:    s_endpgm
6072;
6073; GFX7-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
6074; GFX7:       ; %bb.0: ; %entry
6075; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6076; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6077; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6078; GFX7-NEXT:    s_add_u32 s0, s0, 16
6079; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6080; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6081; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6082; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6083; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6084; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6085; GFX7-NEXT:    s_endpgm
6086;
6087; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
6088; GFX10-WGP:       ; %bb.0: ; %entry
6089; GFX10-WGP-NEXT:    s_clause 0x1
6090; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6091; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6092; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
6093; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6094; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6095; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6096; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6097; GFX10-WGP-NEXT:    s_endpgm
6098;
6099; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
6100; GFX10-CU:       ; %bb.0: ; %entry
6101; GFX10-CU-NEXT:    s_clause 0x1
6102; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6103; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6104; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
6105; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6106; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6107; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6108; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6109; GFX10-CU-NEXT:    s_endpgm
6110;
6111; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
6112; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6113; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
6114; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6115; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
6116; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
6117; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6118; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6119; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6120; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
6121; SKIP-CACHE-INV-NEXT:    s_endpgm
6122;
6123; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
6124; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6125; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6126; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6127; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6128; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6129; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6130; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6131; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6132;
6133; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
6134; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6135; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6136; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6137; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6138; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6139; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6140; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6141; GFX90A-TGSPLIT-NEXT:    s_endpgm
6142    i32 addrspace(1)* %out, i32 %in, i32 %old) {
6143entry:
6144  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
6145  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst monotonic
6146  ret void
6147}
6148
6149define amdgpu_kernel void @global_singlethread_one_as_monotonic_acquire_cmpxchg(
6150; GFX6-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
6151; GFX6:       ; %bb.0: ; %entry
6152; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
6153; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6154; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
6155; GFX6-NEXT:    s_mov_b32 s2, -1
6156; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6157; GFX6-NEXT:    v_mov_b32_e32 v0, s6
6158; GFX6-NEXT:    v_mov_b32_e32 v1, s7
6159; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
6160; GFX6-NEXT:    s_endpgm
6161;
6162; GFX7-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
6163; GFX7:       ; %bb.0: ; %entry
6164; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6165; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6166; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6167; GFX7-NEXT:    s_add_u32 s0, s0, 16
6168; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6169; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6170; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6171; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6172; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6173; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6174; GFX7-NEXT:    s_endpgm
6175;
6176; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
6177; GFX10-WGP:       ; %bb.0: ; %entry
6178; GFX10-WGP-NEXT:    s_clause 0x1
6179; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6180; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6181; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
6182; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6183; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6184; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6185; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6186; GFX10-WGP-NEXT:    s_endpgm
6187;
6188; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
6189; GFX10-CU:       ; %bb.0: ; %entry
6190; GFX10-CU-NEXT:    s_clause 0x1
6191; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6192; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6193; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
6194; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6195; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6196; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6197; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6198; GFX10-CU-NEXT:    s_endpgm
6199;
6200; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
6201; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6202; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
6203; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6204; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
6205; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
6206; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6207; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6208; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6209; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
6210; SKIP-CACHE-INV-NEXT:    s_endpgm
6211;
6212; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
6213; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6214; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6215; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6216; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6217; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6218; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6219; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6220; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6221;
6222; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
6223; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6224; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6225; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6226; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6227; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6228; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6229; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6230; GFX90A-TGSPLIT-NEXT:    s_endpgm
6231    i32 addrspace(1)* %out, i32 %in, i32 %old) {
6232entry:
6233  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
6234  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic acquire
6235  ret void
6236}
6237
6238define amdgpu_kernel void @global_singlethread_one_as_acquire_acquire_cmpxchg(
6239; GFX6-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
6240; GFX6:       ; %bb.0: ; %entry
6241; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
6242; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6243; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
6244; GFX6-NEXT:    s_mov_b32 s2, -1
6245; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6246; GFX6-NEXT:    v_mov_b32_e32 v0, s6
6247; GFX6-NEXT:    v_mov_b32_e32 v1, s7
6248; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
6249; GFX6-NEXT:    s_endpgm
6250;
6251; GFX7-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
6252; GFX7:       ; %bb.0: ; %entry
6253; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6254; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6255; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6256; GFX7-NEXT:    s_add_u32 s0, s0, 16
6257; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6258; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6259; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6260; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6261; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6262; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6263; GFX7-NEXT:    s_endpgm
6264;
6265; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
6266; GFX10-WGP:       ; %bb.0: ; %entry
6267; GFX10-WGP-NEXT:    s_clause 0x1
6268; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6269; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6270; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
6271; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6272; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6273; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6274; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6275; GFX10-WGP-NEXT:    s_endpgm
6276;
6277; GFX10-CU-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
6278; GFX10-CU:       ; %bb.0: ; %entry
6279; GFX10-CU-NEXT:    s_clause 0x1
6280; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6281; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6282; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
6283; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6284; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6285; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6286; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6287; GFX10-CU-NEXT:    s_endpgm
6288;
6289; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
6290; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6291; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
6292; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6293; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
6294; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
6295; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6296; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6297; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6298; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
6299; SKIP-CACHE-INV-NEXT:    s_endpgm
6300;
6301; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
6302; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6303; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6304; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6305; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6306; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6307; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6308; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6309; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6310;
6311; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
6312; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6313; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6314; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6315; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6316; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6317; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6318; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6319; GFX90A-TGSPLIT-NEXT:    s_endpgm
6320    i32 addrspace(1)* %out, i32 %in, i32 %old) {
6321entry:
6322  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
6323  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire acquire
6324  ret void
6325}
6326
6327define amdgpu_kernel void @global_singlethread_one_as_release_acquire_cmpxchg(
6328; GFX6-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
6329; GFX6:       ; %bb.0: ; %entry
6330; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
6331; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6332; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
6333; GFX6-NEXT:    s_mov_b32 s2, -1
6334; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6335; GFX6-NEXT:    v_mov_b32_e32 v0, s6
6336; GFX6-NEXT:    v_mov_b32_e32 v1, s7
6337; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
6338; GFX6-NEXT:    s_endpgm
6339;
6340; GFX7-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
6341; GFX7:       ; %bb.0: ; %entry
6342; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6343; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6344; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6345; GFX7-NEXT:    s_add_u32 s0, s0, 16
6346; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6347; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6348; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6349; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6350; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6351; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6352; GFX7-NEXT:    s_endpgm
6353;
6354; GFX10-WGP-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
6355; GFX10-WGP:       ; %bb.0: ; %entry
6356; GFX10-WGP-NEXT:    s_clause 0x1
6357; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6358; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6359; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
6360; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6361; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6362; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6363; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6364; GFX10-WGP-NEXT:    s_endpgm
6365;
6366; GFX10-CU-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
6367; GFX10-CU:       ; %bb.0: ; %entry
6368; GFX10-CU-NEXT:    s_clause 0x1
6369; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6370; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6371; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
6372; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6373; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6374; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6375; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6376; GFX10-CU-NEXT:    s_endpgm
6377;
6378; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
6379; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6380; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
6381; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6382; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
6383; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
6384; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6385; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6386; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6387; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
6388; SKIP-CACHE-INV-NEXT:    s_endpgm
6389;
6390; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
6391; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6392; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6393; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6394; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6395; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6396; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6397; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6398; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6399;
6400; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
6401; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6402; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6403; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6404; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6405; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6406; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6407; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6408; GFX90A-TGSPLIT-NEXT:    s_endpgm
6409    i32 addrspace(1)* %out, i32 %in, i32 %old) {
6410entry:
6411  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
6412  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release acquire
6413  ret void
6414}
6415
6416define amdgpu_kernel void @global_singlethread_one_as_acq_rel_acquire_cmpxchg(
6417; GFX6-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
6418; GFX6:       ; %bb.0: ; %entry
6419; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
6420; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6421; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
6422; GFX6-NEXT:    s_mov_b32 s2, -1
6423; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6424; GFX6-NEXT:    v_mov_b32_e32 v0, s6
6425; GFX6-NEXT:    v_mov_b32_e32 v1, s7
6426; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
6427; GFX6-NEXT:    s_endpgm
6428;
6429; GFX7-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
6430; GFX7:       ; %bb.0: ; %entry
6431; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6432; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6433; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6434; GFX7-NEXT:    s_add_u32 s0, s0, 16
6435; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6436; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6437; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6438; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6439; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6440; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6441; GFX7-NEXT:    s_endpgm
6442;
6443; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
6444; GFX10-WGP:       ; %bb.0: ; %entry
6445; GFX10-WGP-NEXT:    s_clause 0x1
6446; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6447; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6448; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
6449; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6450; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6451; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6452; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6453; GFX10-WGP-NEXT:    s_endpgm
6454;
6455; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
6456; GFX10-CU:       ; %bb.0: ; %entry
6457; GFX10-CU-NEXT:    s_clause 0x1
6458; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6459; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6460; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
6461; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6462; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6463; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6464; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6465; GFX10-CU-NEXT:    s_endpgm
6466;
6467; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
6468; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6469; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
6470; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6471; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
6472; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
6473; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6474; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6475; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6476; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
6477; SKIP-CACHE-INV-NEXT:    s_endpgm
6478;
6479; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
6480; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6481; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6482; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6483; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6484; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6485; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6486; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6487; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6488;
6489; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
6490; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6491; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6492; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6493; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6494; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6495; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6496; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6497; GFX90A-TGSPLIT-NEXT:    s_endpgm
6498    i32 addrspace(1)* %out, i32 %in, i32 %old) {
6499entry:
6500  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
6501  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel acquire
6502  ret void
6503}
6504
6505define amdgpu_kernel void @global_singlethread_one_as_seq_cst_acquire_cmpxchg(
6506; GFX6-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
6507; GFX6:       ; %bb.0: ; %entry
6508; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
6509; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6510; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
6511; GFX6-NEXT:    s_mov_b32 s2, -1
6512; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6513; GFX6-NEXT:    v_mov_b32_e32 v0, s6
6514; GFX6-NEXT:    v_mov_b32_e32 v1, s7
6515; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
6516; GFX6-NEXT:    s_endpgm
6517;
6518; GFX7-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
6519; GFX7:       ; %bb.0: ; %entry
6520; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6521; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6522; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6523; GFX7-NEXT:    s_add_u32 s0, s0, 16
6524; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6525; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6526; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6527; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6528; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6529; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6530; GFX7-NEXT:    s_endpgm
6531;
6532; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
6533; GFX10-WGP:       ; %bb.0: ; %entry
6534; GFX10-WGP-NEXT:    s_clause 0x1
6535; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6536; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6537; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
6538; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6539; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6540; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6541; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6542; GFX10-WGP-NEXT:    s_endpgm
6543;
6544; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
6545; GFX10-CU:       ; %bb.0: ; %entry
6546; GFX10-CU-NEXT:    s_clause 0x1
6547; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6548; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6549; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
6550; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6551; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6552; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6553; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6554; GFX10-CU-NEXT:    s_endpgm
6555;
6556; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
6557; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6558; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
6559; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6560; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
6561; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
6562; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6563; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6564; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6565; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
6566; SKIP-CACHE-INV-NEXT:    s_endpgm
6567;
6568; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
6569; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6570; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6571; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6572; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6573; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6574; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6575; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6576; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6577;
6578; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
6579; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6580; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6581; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6582; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6583; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6584; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6585; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6586; GFX90A-TGSPLIT-NEXT:    s_endpgm
6587    i32 addrspace(1)* %out, i32 %in, i32 %old) {
6588entry:
6589  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
6590  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst acquire
6591  ret void
6592}
6593
6594define amdgpu_kernel void @global_singlethread_one_as_monotonic_seq_cst_cmpxchg(
6595; GFX6-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
6596; GFX6:       ; %bb.0: ; %entry
6597; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
6598; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6599; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
6600; GFX6-NEXT:    s_mov_b32 s2, -1
6601; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6602; GFX6-NEXT:    v_mov_b32_e32 v0, s6
6603; GFX6-NEXT:    v_mov_b32_e32 v1, s7
6604; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
6605; GFX6-NEXT:    s_endpgm
6606;
6607; GFX7-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
6608; GFX7:       ; %bb.0: ; %entry
6609; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6610; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6611; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6612; GFX7-NEXT:    s_add_u32 s0, s0, 16
6613; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6614; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6615; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6616; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6617; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6618; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6619; GFX7-NEXT:    s_endpgm
6620;
6621; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
6622; GFX10-WGP:       ; %bb.0: ; %entry
6623; GFX10-WGP-NEXT:    s_clause 0x1
6624; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6625; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6626; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
6627; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6628; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6629; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6630; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6631; GFX10-WGP-NEXT:    s_endpgm
6632;
6633; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
6634; GFX10-CU:       ; %bb.0: ; %entry
6635; GFX10-CU-NEXT:    s_clause 0x1
6636; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6637; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6638; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
6639; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6640; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6641; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6642; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6643; GFX10-CU-NEXT:    s_endpgm
6644;
6645; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
6646; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6647; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
6648; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6649; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
6650; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
6651; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6652; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6653; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6654; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
6655; SKIP-CACHE-INV-NEXT:    s_endpgm
6656;
6657; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
6658; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6659; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6660; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6661; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6662; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6663; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6664; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6665; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6666;
6667; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
6668; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6669; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6670; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6671; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6672; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6673; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6674; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6675; GFX90A-TGSPLIT-NEXT:    s_endpgm
6676    i32 addrspace(1)* %out, i32 %in, i32 %old) {
6677entry:
6678  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
6679  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic seq_cst
6680  ret void
6681}
6682
6683define amdgpu_kernel void @global_singlethread_one_as_acquire_seq_cst_cmpxchg(
6684; GFX6-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
6685; GFX6:       ; %bb.0: ; %entry
6686; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
6687; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6688; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
6689; GFX6-NEXT:    s_mov_b32 s2, -1
6690; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6691; GFX6-NEXT:    v_mov_b32_e32 v0, s6
6692; GFX6-NEXT:    v_mov_b32_e32 v1, s7
6693; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
6694; GFX6-NEXT:    s_endpgm
6695;
6696; GFX7-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
6697; GFX7:       ; %bb.0: ; %entry
6698; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6699; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6700; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6701; GFX7-NEXT:    s_add_u32 s0, s0, 16
6702; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6703; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6704; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6705; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6706; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6707; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6708; GFX7-NEXT:    s_endpgm
6709;
6710; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
6711; GFX10-WGP:       ; %bb.0: ; %entry
6712; GFX10-WGP-NEXT:    s_clause 0x1
6713; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6714; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6715; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
6716; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6717; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6718; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6719; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6720; GFX10-WGP-NEXT:    s_endpgm
6721;
6722; GFX10-CU-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
6723; GFX10-CU:       ; %bb.0: ; %entry
6724; GFX10-CU-NEXT:    s_clause 0x1
6725; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6726; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6727; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
6728; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6729; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6730; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6731; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6732; GFX10-CU-NEXT:    s_endpgm
6733;
6734; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
6735; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6736; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
6737; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6738; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
6739; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
6740; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6741; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6742; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6743; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
6744; SKIP-CACHE-INV-NEXT:    s_endpgm
6745;
6746; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
6747; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6748; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6749; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6750; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6751; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6752; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6753; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6754; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6755;
6756; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
6757; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6758; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6759; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6760; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6761; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6762; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6763; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6764; GFX90A-TGSPLIT-NEXT:    s_endpgm
6765    i32 addrspace(1)* %out, i32 %in, i32 %old) {
6766entry:
6767  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
6768  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire seq_cst
6769  ret void
6770}
6771
6772define amdgpu_kernel void @global_singlethread_one_as_release_seq_cst_cmpxchg(
6773; GFX6-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
6774; GFX6:       ; %bb.0: ; %entry
6775; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
6776; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6777; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
6778; GFX6-NEXT:    s_mov_b32 s2, -1
6779; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6780; GFX6-NEXT:    v_mov_b32_e32 v0, s6
6781; GFX6-NEXT:    v_mov_b32_e32 v1, s7
6782; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
6783; GFX6-NEXT:    s_endpgm
6784;
6785; GFX7-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
6786; GFX7:       ; %bb.0: ; %entry
6787; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6788; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6789; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6790; GFX7-NEXT:    s_add_u32 s0, s0, 16
6791; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6792; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6793; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6794; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6795; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6796; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6797; GFX7-NEXT:    s_endpgm
6798;
6799; GFX10-WGP-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
6800; GFX10-WGP:       ; %bb.0: ; %entry
6801; GFX10-WGP-NEXT:    s_clause 0x1
6802; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6803; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6804; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
6805; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6806; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6807; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6808; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6809; GFX10-WGP-NEXT:    s_endpgm
6810;
6811; GFX10-CU-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
6812; GFX10-CU:       ; %bb.0: ; %entry
6813; GFX10-CU-NEXT:    s_clause 0x1
6814; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6815; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6816; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
6817; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6818; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6819; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6820; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6821; GFX10-CU-NEXT:    s_endpgm
6822;
6823; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
6824; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6825; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
6826; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6827; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
6828; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
6829; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6830; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6831; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6832; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
6833; SKIP-CACHE-INV-NEXT:    s_endpgm
6834;
6835; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
6836; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6837; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6838; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6839; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6840; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6841; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6842; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6843; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6844;
6845; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
6846; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6847; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6848; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6849; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6850; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6851; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6852; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6853; GFX90A-TGSPLIT-NEXT:    s_endpgm
6854    i32 addrspace(1)* %out, i32 %in, i32 %old) {
6855entry:
6856  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
6857  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release seq_cst
6858  ret void
6859}
6860
6861define amdgpu_kernel void @global_singlethread_one_as_acq_rel_seq_cst_cmpxchg(
6862; GFX6-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
6863; GFX6:       ; %bb.0: ; %entry
6864; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
6865; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6866; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
6867; GFX6-NEXT:    s_mov_b32 s2, -1
6868; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6869; GFX6-NEXT:    v_mov_b32_e32 v0, s6
6870; GFX6-NEXT:    v_mov_b32_e32 v1, s7
6871; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
6872; GFX6-NEXT:    s_endpgm
6873;
6874; GFX7-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
6875; GFX7:       ; %bb.0: ; %entry
6876; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6877; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6878; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6879; GFX7-NEXT:    s_add_u32 s0, s0, 16
6880; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6881; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6882; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6883; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6884; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6885; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6886; GFX7-NEXT:    s_endpgm
6887;
6888; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
6889; GFX10-WGP:       ; %bb.0: ; %entry
6890; GFX10-WGP-NEXT:    s_clause 0x1
6891; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6892; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6893; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
6894; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6895; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6896; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6897; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6898; GFX10-WGP-NEXT:    s_endpgm
6899;
6900; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
6901; GFX10-CU:       ; %bb.0: ; %entry
6902; GFX10-CU-NEXT:    s_clause 0x1
6903; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6904; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6905; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
6906; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6907; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6908; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6909; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6910; GFX10-CU-NEXT:    s_endpgm
6911;
6912; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
6913; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6914; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
6915; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6916; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
6917; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
6918; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6919; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6920; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6921; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
6922; SKIP-CACHE-INV-NEXT:    s_endpgm
6923;
6924; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
6925; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6926; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6927; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6928; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6929; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6930; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6931; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6932; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6933;
6934; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
6935; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6936; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6937; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6938; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6939; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6940; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6941; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6942; GFX90A-TGSPLIT-NEXT:    s_endpgm
6943    i32 addrspace(1)* %out, i32 %in, i32 %old) {
6944entry:
6945  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
6946  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel seq_cst
6947  ret void
6948}
6949
6950define amdgpu_kernel void @global_singlethread_one_as_seq_cst_seq_cst_cmpxchg(
6951; GFX6-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
6952; GFX6:       ; %bb.0: ; %entry
6953; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
6954; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6955; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
6956; GFX6-NEXT:    s_mov_b32 s2, -1
6957; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6958; GFX6-NEXT:    v_mov_b32_e32 v0, s6
6959; GFX6-NEXT:    v_mov_b32_e32 v1, s7
6960; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
6961; GFX6-NEXT:    s_endpgm
6962;
6963; GFX7-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
6964; GFX7:       ; %bb.0: ; %entry
6965; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6966; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6967; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6968; GFX7-NEXT:    s_add_u32 s0, s0, 16
6969; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6970; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6971; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6972; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6973; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6974; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6975; GFX7-NEXT:    s_endpgm
6976;
6977; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
6978; GFX10-WGP:       ; %bb.0: ; %entry
6979; GFX10-WGP-NEXT:    s_clause 0x1
6980; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6981; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6982; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
6983; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6984; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6985; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6986; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6987; GFX10-WGP-NEXT:    s_endpgm
6988;
6989; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
6990; GFX10-CU:       ; %bb.0: ; %entry
6991; GFX10-CU-NEXT:    s_clause 0x1
6992; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6993; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6994; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
6995; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6996; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6997; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6998; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6999; GFX10-CU-NEXT:    s_endpgm
7000;
7001; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
7002; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7003; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
7004; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7005; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
7006; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
7007; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7008; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7009; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7010; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
7011; SKIP-CACHE-INV-NEXT:    s_endpgm
7012;
7013; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
7014; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7015; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7016; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7017; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7018; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7019; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7020; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7021; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7022;
7023; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
7024; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7025; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7026; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7027; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7028; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7029; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7030; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7031; GFX90A-TGSPLIT-NEXT:    s_endpgm
7032    i32 addrspace(1)* %out, i32 %in, i32 %old) {
7033entry:
7034  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
7035  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst seq_cst
7036  ret void
7037}
7038
7039define amdgpu_kernel void @global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg(
7040; GFX6-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
7041; GFX6:       ; %bb.0: ; %entry
7042; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
7043; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7044; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7045; GFX6-NEXT:    s_mov_b32 s2, -1
7046; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7047; GFX6-NEXT:    v_mov_b32_e32 v0, s6
7048; GFX6-NEXT:    v_mov_b32_e32 v1, s7
7049; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
7050; GFX6-NEXT:    s_waitcnt vmcnt(0)
7051; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
7052; GFX6-NEXT:    s_endpgm
7053;
7054; GFX7-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
7055; GFX7:       ; %bb.0: ; %entry
7056; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7057; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7058; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7059; GFX7-NEXT:    s_add_u32 s4, s0, 16
7060; GFX7-NEXT:    s_addc_u32 s5, s1, 0
7061; GFX7-NEXT:    v_mov_b32_e32 v0, s4
7062; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7063; GFX7-NEXT:    v_mov_b32_e32 v1, s5
7064; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7065; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7066; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7067; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7068; GFX7-NEXT:    s_waitcnt vmcnt(0)
7069; GFX7-NEXT:    flat_store_dword v[0:1], v2
7070; GFX7-NEXT:    s_endpgm
7071;
7072; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
7073; GFX10-WGP:       ; %bb.0: ; %entry
7074; GFX10-WGP-NEXT:    s_clause 0x1
7075; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7076; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7077; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
7078; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7079; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7080; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7081; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7082; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7083; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
7084; GFX10-WGP-NEXT:    s_endpgm
7085;
7086; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
7087; GFX10-CU:       ; %bb.0: ; %entry
7088; GFX10-CU-NEXT:    s_clause 0x1
7089; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7090; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7091; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
7092; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7093; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7094; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7095; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7096; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7097; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
7098; GFX10-CU-NEXT:    s_endpgm
7099;
7100; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
7101; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7102; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
7103; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7104; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
7105; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
7106; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7107; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7108; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7109; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
7110; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7111; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
7112; SKIP-CACHE-INV-NEXT:    s_endpgm
7113;
7114; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
7115; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7116; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7117; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7118; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7119; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7120; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7121; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7122; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7123; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
7124; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7125;
7126; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
7127; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7128; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7129; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7130; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7131; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7132; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7133; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7134; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7135; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
7136; GFX90A-TGSPLIT-NEXT:    s_endpgm
7137    i32 addrspace(1)* %out, i32 %in, i32 %old) {
7138entry:
7139  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
7140  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic monotonic
7141  %val0 = extractvalue { i32, i1 } %val, 0
7142  store i32 %val0, i32 addrspace(1)* %out, align 4
7143  ret void
7144}
7145
7146define amdgpu_kernel void @global_singlethread_one_as_acquire_monotonic_ret_cmpxchg(
7147; GFX6-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
7148; GFX6:       ; %bb.0: ; %entry
7149; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
7150; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7151; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7152; GFX6-NEXT:    s_mov_b32 s2, -1
7153; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7154; GFX6-NEXT:    v_mov_b32_e32 v0, s6
7155; GFX6-NEXT:    v_mov_b32_e32 v1, s7
7156; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
7157; GFX6-NEXT:    s_waitcnt vmcnt(0)
7158; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
7159; GFX6-NEXT:    s_endpgm
7160;
7161; GFX7-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
7162; GFX7:       ; %bb.0: ; %entry
7163; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7164; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7165; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7166; GFX7-NEXT:    s_add_u32 s4, s0, 16
7167; GFX7-NEXT:    s_addc_u32 s5, s1, 0
7168; GFX7-NEXT:    v_mov_b32_e32 v0, s4
7169; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7170; GFX7-NEXT:    v_mov_b32_e32 v1, s5
7171; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7172; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7173; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7174; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7175; GFX7-NEXT:    s_waitcnt vmcnt(0)
7176; GFX7-NEXT:    flat_store_dword v[0:1], v2
7177; GFX7-NEXT:    s_endpgm
7178;
7179; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
7180; GFX10-WGP:       ; %bb.0: ; %entry
7181; GFX10-WGP-NEXT:    s_clause 0x1
7182; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7183; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7184; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
7185; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7186; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7187; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7188; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7189; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7190; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
7191; GFX10-WGP-NEXT:    s_endpgm
7192;
7193; GFX10-CU-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
7194; GFX10-CU:       ; %bb.0: ; %entry
7195; GFX10-CU-NEXT:    s_clause 0x1
7196; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7197; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7198; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
7199; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7200; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7201; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7202; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7203; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7204; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
7205; GFX10-CU-NEXT:    s_endpgm
7206;
7207; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
7208; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7209; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
7210; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7211; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
7212; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
7213; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7214; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7215; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7216; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
7217; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7218; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
7219; SKIP-CACHE-INV-NEXT:    s_endpgm
7220;
7221; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
7222; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7223; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7224; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7225; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7226; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7227; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7228; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7229; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7230; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
7231; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7232;
7233; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
7234; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7235; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7236; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7237; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7238; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7239; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7240; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7241; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7242; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
7243; GFX90A-TGSPLIT-NEXT:    s_endpgm
7244    i32 addrspace(1)* %out, i32 %in, i32 %old) {
7245entry:
7246  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
7247  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire monotonic
7248  %val0 = extractvalue { i32, i1 } %val, 0
7249  store i32 %val0, i32 addrspace(1)* %out, align 4
7250  ret void
7251}
7252
7253define amdgpu_kernel void @global_singlethread_one_as_release_monotonic_ret_cmpxchg(
7254; GFX6-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
7255; GFX6:       ; %bb.0: ; %entry
7256; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
7257; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7258; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7259; GFX6-NEXT:    s_mov_b32 s2, -1
7260; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7261; GFX6-NEXT:    v_mov_b32_e32 v0, s6
7262; GFX6-NEXT:    v_mov_b32_e32 v1, s7
7263; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
7264; GFX6-NEXT:    s_waitcnt vmcnt(0)
7265; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
7266; GFX6-NEXT:    s_endpgm
7267;
7268; GFX7-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
7269; GFX7:       ; %bb.0: ; %entry
7270; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7271; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7272; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7273; GFX7-NEXT:    s_add_u32 s4, s0, 16
7274; GFX7-NEXT:    s_addc_u32 s5, s1, 0
7275; GFX7-NEXT:    v_mov_b32_e32 v0, s4
7276; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7277; GFX7-NEXT:    v_mov_b32_e32 v1, s5
7278; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7279; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7280; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7281; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7282; GFX7-NEXT:    s_waitcnt vmcnt(0)
7283; GFX7-NEXT:    flat_store_dword v[0:1], v2
7284; GFX7-NEXT:    s_endpgm
7285;
7286; GFX10-WGP-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
7287; GFX10-WGP:       ; %bb.0: ; %entry
7288; GFX10-WGP-NEXT:    s_clause 0x1
7289; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7290; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7291; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
7292; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7293; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7294; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7295; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7296; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7297; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
7298; GFX10-WGP-NEXT:    s_endpgm
7299;
7300; GFX10-CU-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
7301; GFX10-CU:       ; %bb.0: ; %entry
7302; GFX10-CU-NEXT:    s_clause 0x1
7303; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7304; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7305; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
7306; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7307; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7308; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7309; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7310; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7311; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
7312; GFX10-CU-NEXT:    s_endpgm
7313;
7314; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
7315; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7316; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
7317; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7318; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
7319; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
7320; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7321; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7322; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7323; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
7324; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7325; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
7326; SKIP-CACHE-INV-NEXT:    s_endpgm
7327;
7328; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
7329; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7330; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7331; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7332; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7333; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7334; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7335; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7336; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7337; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
7338; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7339;
7340; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
7341; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7342; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7343; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7344; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7345; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7346; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7347; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7348; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7349; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
7350; GFX90A-TGSPLIT-NEXT:    s_endpgm
7351    i32 addrspace(1)* %out, i32 %in, i32 %old) {
7352entry:
7353  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
7354  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release monotonic
7355  %val0 = extractvalue { i32, i1 } %val, 0
7356  store i32 %val0, i32 addrspace(1)* %out, align 4
7357  ret void
7358}
7359
7360define amdgpu_kernel void @global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg(
7361; GFX6-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
7362; GFX6:       ; %bb.0: ; %entry
7363; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
7364; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7365; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7366; GFX6-NEXT:    s_mov_b32 s2, -1
7367; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7368; GFX6-NEXT:    v_mov_b32_e32 v0, s6
7369; GFX6-NEXT:    v_mov_b32_e32 v1, s7
7370; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
7371; GFX6-NEXT:    s_waitcnt vmcnt(0)
7372; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
7373; GFX6-NEXT:    s_endpgm
7374;
7375; GFX7-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
7376; GFX7:       ; %bb.0: ; %entry
7377; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7378; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7379; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7380; GFX7-NEXT:    s_add_u32 s4, s0, 16
7381; GFX7-NEXT:    s_addc_u32 s5, s1, 0
7382; GFX7-NEXT:    v_mov_b32_e32 v0, s4
7383; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7384; GFX7-NEXT:    v_mov_b32_e32 v1, s5
7385; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7386; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7387; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7388; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7389; GFX7-NEXT:    s_waitcnt vmcnt(0)
7390; GFX7-NEXT:    flat_store_dword v[0:1], v2
7391; GFX7-NEXT:    s_endpgm
7392;
7393; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
7394; GFX10-WGP:       ; %bb.0: ; %entry
7395; GFX10-WGP-NEXT:    s_clause 0x1
7396; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7397; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7398; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
7399; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7400; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7401; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7402; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7403; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7404; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
7405; GFX10-WGP-NEXT:    s_endpgm
7406;
7407; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
7408; GFX10-CU:       ; %bb.0: ; %entry
7409; GFX10-CU-NEXT:    s_clause 0x1
7410; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7411; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7412; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
7413; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7414; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7415; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7416; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7417; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7418; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
7419; GFX10-CU-NEXT:    s_endpgm
7420;
7421; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
7422; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7423; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
7424; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7425; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
7426; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
7427; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7428; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7429; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7430; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
7431; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7432; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
7433; SKIP-CACHE-INV-NEXT:    s_endpgm
7434;
7435; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
7436; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7437; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7438; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7439; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7440; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7441; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7442; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7443; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7444; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
7445; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7446;
7447; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
7448; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7449; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7450; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7451; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7452; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7453; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7454; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7455; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7456; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
7457; GFX90A-TGSPLIT-NEXT:    s_endpgm
7458    i32 addrspace(1)* %out, i32 %in, i32 %old) {
7459entry:
7460  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
7461  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel monotonic
7462  %val0 = extractvalue { i32, i1 } %val, 0
7463  store i32 %val0, i32 addrspace(1)* %out, align 4
7464  ret void
7465}
7466
7467define amdgpu_kernel void @global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg(
7468; GFX6-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
7469; GFX6:       ; %bb.0: ; %entry
7470; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
7471; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7472; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7473; GFX6-NEXT:    s_mov_b32 s2, -1
7474; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7475; GFX6-NEXT:    v_mov_b32_e32 v0, s6
7476; GFX6-NEXT:    v_mov_b32_e32 v1, s7
7477; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
7478; GFX6-NEXT:    s_waitcnt vmcnt(0)
7479; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
7480; GFX6-NEXT:    s_endpgm
7481;
7482; GFX7-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
7483; GFX7:       ; %bb.0: ; %entry
7484; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7485; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7486; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7487; GFX7-NEXT:    s_add_u32 s4, s0, 16
7488; GFX7-NEXT:    s_addc_u32 s5, s1, 0
7489; GFX7-NEXT:    v_mov_b32_e32 v0, s4
7490; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7491; GFX7-NEXT:    v_mov_b32_e32 v1, s5
7492; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7493; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7494; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7495; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7496; GFX7-NEXT:    s_waitcnt vmcnt(0)
7497; GFX7-NEXT:    flat_store_dword v[0:1], v2
7498; GFX7-NEXT:    s_endpgm
7499;
7500; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
7501; GFX10-WGP:       ; %bb.0: ; %entry
7502; GFX10-WGP-NEXT:    s_clause 0x1
7503; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7504; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7505; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
7506; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7507; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7508; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7509; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7510; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7511; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
7512; GFX10-WGP-NEXT:    s_endpgm
7513;
7514; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
7515; GFX10-CU:       ; %bb.0: ; %entry
7516; GFX10-CU-NEXT:    s_clause 0x1
7517; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7518; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7519; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
7520; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7521; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7522; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7523; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7524; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7525; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
7526; GFX10-CU-NEXT:    s_endpgm
7527;
7528; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
7529; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7530; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
7531; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7532; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
7533; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
7534; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7535; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7536; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7537; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
7538; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7539; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
7540; SKIP-CACHE-INV-NEXT:    s_endpgm
7541;
7542; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
7543; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7544; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7545; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7546; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7547; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7548; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7549; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7550; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7551; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
7552; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7553;
7554; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
7555; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7556; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7557; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7558; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7559; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7560; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7561; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7562; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7563; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
7564; GFX90A-TGSPLIT-NEXT:    s_endpgm
7565    i32 addrspace(1)* %out, i32 %in, i32 %old) {
7566entry:
7567  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
7568  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst monotonic
7569  %val0 = extractvalue { i32, i1 } %val, 0
7570  store i32 %val0, i32 addrspace(1)* %out, align 4
7571  ret void
7572}
7573
7574define amdgpu_kernel void @global_singlethread_one_as_monotonic_acquire_ret_cmpxchg(
7575; GFX6-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
7576; GFX6:       ; %bb.0: ; %entry
7577; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
7578; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7579; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7580; GFX6-NEXT:    s_mov_b32 s2, -1
7581; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7582; GFX6-NEXT:    v_mov_b32_e32 v0, s6
7583; GFX6-NEXT:    v_mov_b32_e32 v1, s7
7584; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
7585; GFX6-NEXT:    s_waitcnt vmcnt(0)
7586; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
7587; GFX6-NEXT:    s_endpgm
7588;
7589; GFX7-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
7590; GFX7:       ; %bb.0: ; %entry
7591; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7592; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7593; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7594; GFX7-NEXT:    s_add_u32 s4, s0, 16
7595; GFX7-NEXT:    s_addc_u32 s5, s1, 0
7596; GFX7-NEXT:    v_mov_b32_e32 v0, s4
7597; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7598; GFX7-NEXT:    v_mov_b32_e32 v1, s5
7599; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7600; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7601; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7602; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7603; GFX7-NEXT:    s_waitcnt vmcnt(0)
7604; GFX7-NEXT:    flat_store_dword v[0:1], v2
7605; GFX7-NEXT:    s_endpgm
7606;
7607; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
7608; GFX10-WGP:       ; %bb.0: ; %entry
7609; GFX10-WGP-NEXT:    s_clause 0x1
7610; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7611; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7612; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
7613; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7614; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7615; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7616; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7617; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7618; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
7619; GFX10-WGP-NEXT:    s_endpgm
7620;
7621; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
7622; GFX10-CU:       ; %bb.0: ; %entry
7623; GFX10-CU-NEXT:    s_clause 0x1
7624; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7625; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7626; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
7627; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7628; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7629; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7630; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7631; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7632; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
7633; GFX10-CU-NEXT:    s_endpgm
7634;
7635; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
7636; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7637; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
7638; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7639; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
7640; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
7641; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7642; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7643; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7644; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
7645; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7646; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
7647; SKIP-CACHE-INV-NEXT:    s_endpgm
7648;
7649; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
7650; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7651; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7652; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7653; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7654; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7655; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7656; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7657; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7658; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
7659; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7660;
7661; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
7662; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7663; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7664; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7665; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7666; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7667; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7668; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7669; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7670; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
7671; GFX90A-TGSPLIT-NEXT:    s_endpgm
7672    i32 addrspace(1)* %out, i32 %in, i32 %old) {
7673entry:
7674  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
7675  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic acquire
7676  %val0 = extractvalue { i32, i1 } %val, 0
7677  store i32 %val0, i32 addrspace(1)* %out, align 4
7678  ret void
7679}
7680
7681define amdgpu_kernel void @global_singlethread_one_as_acquire_acquire_ret_cmpxchg(
7682; GFX6-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
7683; GFX6:       ; %bb.0: ; %entry
7684; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
7685; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7686; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7687; GFX6-NEXT:    s_mov_b32 s2, -1
7688; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7689; GFX6-NEXT:    v_mov_b32_e32 v0, s6
7690; GFX6-NEXT:    v_mov_b32_e32 v1, s7
7691; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
7692; GFX6-NEXT:    s_waitcnt vmcnt(0)
7693; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
7694; GFX6-NEXT:    s_endpgm
7695;
7696; GFX7-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
7697; GFX7:       ; %bb.0: ; %entry
7698; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7699; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7700; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7701; GFX7-NEXT:    s_add_u32 s4, s0, 16
7702; GFX7-NEXT:    s_addc_u32 s5, s1, 0
7703; GFX7-NEXT:    v_mov_b32_e32 v0, s4
7704; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7705; GFX7-NEXT:    v_mov_b32_e32 v1, s5
7706; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7707; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7708; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7709; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7710; GFX7-NEXT:    s_waitcnt vmcnt(0)
7711; GFX7-NEXT:    flat_store_dword v[0:1], v2
7712; GFX7-NEXT:    s_endpgm
7713;
7714; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
7715; GFX10-WGP:       ; %bb.0: ; %entry
7716; GFX10-WGP-NEXT:    s_clause 0x1
7717; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7718; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7719; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
7720; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7721; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7722; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7723; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7724; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7725; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
7726; GFX10-WGP-NEXT:    s_endpgm
7727;
7728; GFX10-CU-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
7729; GFX10-CU:       ; %bb.0: ; %entry
7730; GFX10-CU-NEXT:    s_clause 0x1
7731; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7732; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7733; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
7734; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7735; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7736; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7737; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7738; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7739; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
7740; GFX10-CU-NEXT:    s_endpgm
7741;
7742; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
7743; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7744; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
7745; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7746; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
7747; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
7748; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7749; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7750; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7751; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
7752; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7753; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
7754; SKIP-CACHE-INV-NEXT:    s_endpgm
7755;
7756; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
7757; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7758; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7759; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7760; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7761; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7762; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7763; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7764; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7765; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
7766; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7767;
7768; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
7769; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7770; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7771; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7772; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7773; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7774; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7775; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7776; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7777; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
7778; GFX90A-TGSPLIT-NEXT:    s_endpgm
7779    i32 addrspace(1)* %out, i32 %in, i32 %old) {
7780entry:
7781  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
7782  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire acquire
7783  %val0 = extractvalue { i32, i1 } %val, 0
7784  store i32 %val0, i32 addrspace(1)* %out, align 4
7785  ret void
7786}
7787
7788define amdgpu_kernel void @global_singlethread_one_as_release_acquire_ret_cmpxchg(
7789; GFX6-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
7790; GFX6:       ; %bb.0: ; %entry
7791; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
7792; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7793; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7794; GFX6-NEXT:    s_mov_b32 s2, -1
7795; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7796; GFX6-NEXT:    v_mov_b32_e32 v0, s6
7797; GFX6-NEXT:    v_mov_b32_e32 v1, s7
7798; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
7799; GFX6-NEXT:    s_waitcnt vmcnt(0)
7800; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
7801; GFX6-NEXT:    s_endpgm
7802;
7803; GFX7-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
7804; GFX7:       ; %bb.0: ; %entry
7805; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7806; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7807; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7808; GFX7-NEXT:    s_add_u32 s4, s0, 16
7809; GFX7-NEXT:    s_addc_u32 s5, s1, 0
7810; GFX7-NEXT:    v_mov_b32_e32 v0, s4
7811; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7812; GFX7-NEXT:    v_mov_b32_e32 v1, s5
7813; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7814; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7815; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7816; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7817; GFX7-NEXT:    s_waitcnt vmcnt(0)
7818; GFX7-NEXT:    flat_store_dword v[0:1], v2
7819; GFX7-NEXT:    s_endpgm
7820;
7821; GFX10-WGP-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
7822; GFX10-WGP:       ; %bb.0: ; %entry
7823; GFX10-WGP-NEXT:    s_clause 0x1
7824; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7825; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7826; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
7827; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7828; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7829; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7830; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7831; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7832; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
7833; GFX10-WGP-NEXT:    s_endpgm
7834;
7835; GFX10-CU-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
7836; GFX10-CU:       ; %bb.0: ; %entry
7837; GFX10-CU-NEXT:    s_clause 0x1
7838; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7839; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7840; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
7841; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7842; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7843; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7844; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7845; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7846; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
7847; GFX10-CU-NEXT:    s_endpgm
7848;
7849; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
7850; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7851; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
7852; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7853; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
7854; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
7855; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7856; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7857; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7858; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
7859; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7860; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
7861; SKIP-CACHE-INV-NEXT:    s_endpgm
7862;
7863; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
7864; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7865; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7866; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7867; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7868; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7869; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7870; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7871; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7872; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
7873; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7874;
7875; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
7876; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7877; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7878; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7879; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7880; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7881; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7882; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7883; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7884; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
7885; GFX90A-TGSPLIT-NEXT:    s_endpgm
7886    i32 addrspace(1)* %out, i32 %in, i32 %old) {
7887entry:
7888  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
7889  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release acquire
7890  %val0 = extractvalue { i32, i1 } %val, 0
7891  store i32 %val0, i32 addrspace(1)* %out, align 4
7892  ret void
7893}
7894
7895define amdgpu_kernel void @global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg(
7896; GFX6-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
7897; GFX6:       ; %bb.0: ; %entry
7898; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
7899; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7900; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7901; GFX6-NEXT:    s_mov_b32 s2, -1
7902; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7903; GFX6-NEXT:    v_mov_b32_e32 v0, s6
7904; GFX6-NEXT:    v_mov_b32_e32 v1, s7
7905; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
7906; GFX6-NEXT:    s_waitcnt vmcnt(0)
7907; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
7908; GFX6-NEXT:    s_endpgm
7909;
7910; GFX7-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
7911; GFX7:       ; %bb.0: ; %entry
7912; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7913; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7914; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7915; GFX7-NEXT:    s_add_u32 s4, s0, 16
7916; GFX7-NEXT:    s_addc_u32 s5, s1, 0
7917; GFX7-NEXT:    v_mov_b32_e32 v0, s4
7918; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7919; GFX7-NEXT:    v_mov_b32_e32 v1, s5
7920; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7921; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7922; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7923; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7924; GFX7-NEXT:    s_waitcnt vmcnt(0)
7925; GFX7-NEXT:    flat_store_dword v[0:1], v2
7926; GFX7-NEXT:    s_endpgm
7927;
7928; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
7929; GFX10-WGP:       ; %bb.0: ; %entry
7930; GFX10-WGP-NEXT:    s_clause 0x1
7931; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7932; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7933; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
7934; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7935; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7936; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7937; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7938; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7939; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
7940; GFX10-WGP-NEXT:    s_endpgm
7941;
7942; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
7943; GFX10-CU:       ; %bb.0: ; %entry
7944; GFX10-CU-NEXT:    s_clause 0x1
7945; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7946; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7947; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
7948; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7949; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7950; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7951; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7952; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7953; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
7954; GFX10-CU-NEXT:    s_endpgm
7955;
7956; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
7957; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7958; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
7959; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7960; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
7961; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
7962; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7963; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7964; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7965; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
7966; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7967; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
7968; SKIP-CACHE-INV-NEXT:    s_endpgm
7969;
7970; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
7971; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7972; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7973; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7974; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7975; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7976; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7977; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7978; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7979; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
7980; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7981;
7982; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
7983; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7984; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7985; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7986; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7987; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7988; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7989; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7990; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7991; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
7992; GFX90A-TGSPLIT-NEXT:    s_endpgm
7993    i32 addrspace(1)* %out, i32 %in, i32 %old) {
7994entry:
7995  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
7996  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel acquire
7997  %val0 = extractvalue { i32, i1 } %val, 0
7998  store i32 %val0, i32 addrspace(1)* %out, align 4
7999  ret void
8000}
8001
8002define amdgpu_kernel void @global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg(
8003; GFX6-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
8004; GFX6:       ; %bb.0: ; %entry
8005; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
8006; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8007; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
8008; GFX6-NEXT:    s_mov_b32 s2, -1
8009; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8010; GFX6-NEXT:    v_mov_b32_e32 v0, s6
8011; GFX6-NEXT:    v_mov_b32_e32 v1, s7
8012; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
8013; GFX6-NEXT:    s_waitcnt vmcnt(0)
8014; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
8015; GFX6-NEXT:    s_endpgm
8016;
8017; GFX7-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
8018; GFX7:       ; %bb.0: ; %entry
8019; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8020; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8021; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8022; GFX7-NEXT:    s_add_u32 s4, s0, 16
8023; GFX7-NEXT:    s_addc_u32 s5, s1, 0
8024; GFX7-NEXT:    v_mov_b32_e32 v0, s4
8025; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8026; GFX7-NEXT:    v_mov_b32_e32 v1, s5
8027; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8028; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8029; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8030; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8031; GFX7-NEXT:    s_waitcnt vmcnt(0)
8032; GFX7-NEXT:    flat_store_dword v[0:1], v2
8033; GFX7-NEXT:    s_endpgm
8034;
8035; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
8036; GFX10-WGP:       ; %bb.0: ; %entry
8037; GFX10-WGP-NEXT:    s_clause 0x1
8038; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8039; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8040; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
8041; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8042; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8043; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8044; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8045; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8046; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
8047; GFX10-WGP-NEXT:    s_endpgm
8048;
8049; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
8050; GFX10-CU:       ; %bb.0: ; %entry
8051; GFX10-CU-NEXT:    s_clause 0x1
8052; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8053; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8054; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
8055; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8056; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8057; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8058; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8059; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8060; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
8061; GFX10-CU-NEXT:    s_endpgm
8062;
8063; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
8064; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8065; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
8066; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
8067; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
8068; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
8069; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8070; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8071; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8072; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
8073; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8074; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
8075; SKIP-CACHE-INV-NEXT:    s_endpgm
8076;
8077; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
8078; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8079; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8080; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8081; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8082; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8083; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8084; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8085; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8086; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
8087; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8088;
8089; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
8090; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8091; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8092; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8093; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8094; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8095; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8096; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8097; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8098; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
8099; GFX90A-TGSPLIT-NEXT:    s_endpgm
8100    i32 addrspace(1)* %out, i32 %in, i32 %old) {
8101entry:
8102  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
8103  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst acquire
8104  %val0 = extractvalue { i32, i1 } %val, 0
8105  store i32 %val0, i32 addrspace(1)* %out, align 4
8106  ret void
8107}
8108
8109define amdgpu_kernel void @global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg(
8110; GFX6-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
8111; GFX6:       ; %bb.0: ; %entry
8112; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
8113; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8114; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
8115; GFX6-NEXT:    s_mov_b32 s2, -1
8116; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8117; GFX6-NEXT:    v_mov_b32_e32 v0, s6
8118; GFX6-NEXT:    v_mov_b32_e32 v1, s7
8119; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
8120; GFX6-NEXT:    s_waitcnt vmcnt(0)
8121; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
8122; GFX6-NEXT:    s_endpgm
8123;
8124; GFX7-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
8125; GFX7:       ; %bb.0: ; %entry
8126; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8127; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8128; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8129; GFX7-NEXT:    s_add_u32 s4, s0, 16
8130; GFX7-NEXT:    s_addc_u32 s5, s1, 0
8131; GFX7-NEXT:    v_mov_b32_e32 v0, s4
8132; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8133; GFX7-NEXT:    v_mov_b32_e32 v1, s5
8134; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8135; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8136; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8137; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8138; GFX7-NEXT:    s_waitcnt vmcnt(0)
8139; GFX7-NEXT:    flat_store_dword v[0:1], v2
8140; GFX7-NEXT:    s_endpgm
8141;
8142; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
8143; GFX10-WGP:       ; %bb.0: ; %entry
8144; GFX10-WGP-NEXT:    s_clause 0x1
8145; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8146; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8147; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
8148; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8149; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8150; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8151; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8152; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8153; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
8154; GFX10-WGP-NEXT:    s_endpgm
8155;
8156; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
8157; GFX10-CU:       ; %bb.0: ; %entry
8158; GFX10-CU-NEXT:    s_clause 0x1
8159; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8160; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8161; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
8162; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8163; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8164; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8165; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8166; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8167; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
8168; GFX10-CU-NEXT:    s_endpgm
8169;
8170; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
8171; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8172; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
8173; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
8174; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
8175; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
8176; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8177; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8178; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8179; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
8180; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8181; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
8182; SKIP-CACHE-INV-NEXT:    s_endpgm
8183;
8184; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
8185; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8186; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8187; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8188; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8189; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8190; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8191; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8192; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8193; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
8194; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8195;
8196; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
8197; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8198; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8199; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8200; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8201; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8202; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8203; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8204; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8205; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
8206; GFX90A-TGSPLIT-NEXT:    s_endpgm
8207    i32 addrspace(1)* %out, i32 %in, i32 %old) {
8208entry:
8209  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
8210  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic seq_cst
8211  %val0 = extractvalue { i32, i1 } %val, 0
8212  store i32 %val0, i32 addrspace(1)* %out, align 4
8213  ret void
8214}
8215
8216define amdgpu_kernel void @global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg(
8217; GFX6-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
8218; GFX6:       ; %bb.0: ; %entry
8219; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
8220; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8221; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
8222; GFX6-NEXT:    s_mov_b32 s2, -1
8223; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8224; GFX6-NEXT:    v_mov_b32_e32 v0, s6
8225; GFX6-NEXT:    v_mov_b32_e32 v1, s7
8226; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
8227; GFX6-NEXT:    s_waitcnt vmcnt(0)
8228; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
8229; GFX6-NEXT:    s_endpgm
8230;
8231; GFX7-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
8232; GFX7:       ; %bb.0: ; %entry
8233; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8234; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8235; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8236; GFX7-NEXT:    s_add_u32 s4, s0, 16
8237; GFX7-NEXT:    s_addc_u32 s5, s1, 0
8238; GFX7-NEXT:    v_mov_b32_e32 v0, s4
8239; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8240; GFX7-NEXT:    v_mov_b32_e32 v1, s5
8241; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8242; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8243; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8244; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8245; GFX7-NEXT:    s_waitcnt vmcnt(0)
8246; GFX7-NEXT:    flat_store_dword v[0:1], v2
8247; GFX7-NEXT:    s_endpgm
8248;
8249; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
8250; GFX10-WGP:       ; %bb.0: ; %entry
8251; GFX10-WGP-NEXT:    s_clause 0x1
8252; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8253; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8254; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
8255; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8256; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8257; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8258; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8259; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8260; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
8261; GFX10-WGP-NEXT:    s_endpgm
8262;
8263; GFX10-CU-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
8264; GFX10-CU:       ; %bb.0: ; %entry
8265; GFX10-CU-NEXT:    s_clause 0x1
8266; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8267; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8268; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
8269; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8270; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8271; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8272; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8273; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8274; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
8275; GFX10-CU-NEXT:    s_endpgm
8276;
8277; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
8278; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8279; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
8280; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
8281; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
8282; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
8283; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8284; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8285; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8286; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
8287; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8288; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
8289; SKIP-CACHE-INV-NEXT:    s_endpgm
8290;
8291; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
8292; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8293; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8294; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8295; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8296; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8297; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8298; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8299; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8300; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
8301; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8302;
8303; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
8304; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8305; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8306; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8307; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8308; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8309; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8310; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8311; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8312; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
8313; GFX90A-TGSPLIT-NEXT:    s_endpgm
8314    i32 addrspace(1)* %out, i32 %in, i32 %old) {
8315entry:
8316  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
8317  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire seq_cst
8318  %val0 = extractvalue { i32, i1 } %val, 0
8319  store i32 %val0, i32 addrspace(1)* %out, align 4
8320  ret void
8321}
8322
8323define amdgpu_kernel void @global_singlethread_one_as_release_seq_cst_ret_cmpxchg(
8324; GFX6-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
8325; GFX6:       ; %bb.0: ; %entry
8326; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
8327; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8328; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
8329; GFX6-NEXT:    s_mov_b32 s2, -1
8330; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8331; GFX6-NEXT:    v_mov_b32_e32 v0, s6
8332; GFX6-NEXT:    v_mov_b32_e32 v1, s7
8333; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
8334; GFX6-NEXT:    s_waitcnt vmcnt(0)
8335; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
8336; GFX6-NEXT:    s_endpgm
8337;
8338; GFX7-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
8339; GFX7:       ; %bb.0: ; %entry
8340; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8341; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8342; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8343; GFX7-NEXT:    s_add_u32 s4, s0, 16
8344; GFX7-NEXT:    s_addc_u32 s5, s1, 0
8345; GFX7-NEXT:    v_mov_b32_e32 v0, s4
8346; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8347; GFX7-NEXT:    v_mov_b32_e32 v1, s5
8348; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8349; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8350; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8351; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8352; GFX7-NEXT:    s_waitcnt vmcnt(0)
8353; GFX7-NEXT:    flat_store_dword v[0:1], v2
8354; GFX7-NEXT:    s_endpgm
8355;
8356; GFX10-WGP-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
8357; GFX10-WGP:       ; %bb.0: ; %entry
8358; GFX10-WGP-NEXT:    s_clause 0x1
8359; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8360; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8361; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
8362; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8363; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8364; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8365; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8366; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8367; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
8368; GFX10-WGP-NEXT:    s_endpgm
8369;
8370; GFX10-CU-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
8371; GFX10-CU:       ; %bb.0: ; %entry
8372; GFX10-CU-NEXT:    s_clause 0x1
8373; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8374; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8375; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
8376; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8377; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8378; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8379; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8380; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8381; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
8382; GFX10-CU-NEXT:    s_endpgm
8383;
8384; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
8385; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8386; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
8387; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
8388; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
8389; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
8390; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8391; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8392; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8393; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
8394; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8395; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
8396; SKIP-CACHE-INV-NEXT:    s_endpgm
8397;
8398; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
8399; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8400; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8401; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8402; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8403; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8404; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8405; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8406; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8407; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
8408; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8409;
8410; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
8411; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8412; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8413; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8414; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8415; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8416; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8417; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8418; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8419; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
8420; GFX90A-TGSPLIT-NEXT:    s_endpgm
8421    i32 addrspace(1)* %out, i32 %in, i32 %old) {
8422entry:
8423  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
8424  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release seq_cst
8425  %val0 = extractvalue { i32, i1 } %val, 0
8426  store i32 %val0, i32 addrspace(1)* %out, align 4
8427  ret void
8428}
8429
8430define amdgpu_kernel void @global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg(
8431; GFX6-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
8432; GFX6:       ; %bb.0: ; %entry
8433; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
8434; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8435; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
8436; GFX6-NEXT:    s_mov_b32 s2, -1
8437; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8438; GFX6-NEXT:    v_mov_b32_e32 v0, s6
8439; GFX6-NEXT:    v_mov_b32_e32 v1, s7
8440; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
8441; GFX6-NEXT:    s_waitcnt vmcnt(0)
8442; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
8443; GFX6-NEXT:    s_endpgm
8444;
8445; GFX7-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
8446; GFX7:       ; %bb.0: ; %entry
8447; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8448; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8449; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8450; GFX7-NEXT:    s_add_u32 s4, s0, 16
8451; GFX7-NEXT:    s_addc_u32 s5, s1, 0
8452; GFX7-NEXT:    v_mov_b32_e32 v0, s4
8453; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8454; GFX7-NEXT:    v_mov_b32_e32 v1, s5
8455; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8456; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8457; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8458; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8459; GFX7-NEXT:    s_waitcnt vmcnt(0)
8460; GFX7-NEXT:    flat_store_dword v[0:1], v2
8461; GFX7-NEXT:    s_endpgm
8462;
8463; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
8464; GFX10-WGP:       ; %bb.0: ; %entry
8465; GFX10-WGP-NEXT:    s_clause 0x1
8466; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8467; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8468; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
8469; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8470; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8471; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8472; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8473; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8474; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
8475; GFX10-WGP-NEXT:    s_endpgm
8476;
8477; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
8478; GFX10-CU:       ; %bb.0: ; %entry
8479; GFX10-CU-NEXT:    s_clause 0x1
8480; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8481; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8482; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
8483; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8484; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8485; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8486; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8487; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8488; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
8489; GFX10-CU-NEXT:    s_endpgm
8490;
8491; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
8492; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8493; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
8494; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
8495; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
8496; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
8497; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8498; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8499; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8500; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
8501; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8502; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
8503; SKIP-CACHE-INV-NEXT:    s_endpgm
8504;
8505; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
8506; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8507; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8508; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8509; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8510; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8511; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8512; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8513; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8514; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
8515; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8516;
8517; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
8518; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8519; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8520; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8521; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8522; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8523; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8524; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8525; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8526; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
8527; GFX90A-TGSPLIT-NEXT:    s_endpgm
8528    i32 addrspace(1)* %out, i32 %in, i32 %old) {
8529entry:
8530  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
8531  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel seq_cst
8532  %val0 = extractvalue { i32, i1 } %val, 0
8533  store i32 %val0, i32 addrspace(1)* %out, align 4
8534  ret void
8535}
8536
8537define amdgpu_kernel void @global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg(
8538; GFX6-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
8539; GFX6:       ; %bb.0: ; %entry
8540; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
8541; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8542; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
8543; GFX6-NEXT:    s_mov_b32 s2, -1
8544; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8545; GFX6-NEXT:    v_mov_b32_e32 v0, s6
8546; GFX6-NEXT:    v_mov_b32_e32 v1, s7
8547; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
8548; GFX6-NEXT:    s_waitcnt vmcnt(0)
8549; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
8550; GFX6-NEXT:    s_endpgm
8551;
8552; GFX7-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
8553; GFX7:       ; %bb.0: ; %entry
8554; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8555; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8556; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8557; GFX7-NEXT:    s_add_u32 s4, s0, 16
8558; GFX7-NEXT:    s_addc_u32 s5, s1, 0
8559; GFX7-NEXT:    v_mov_b32_e32 v0, s4
8560; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8561; GFX7-NEXT:    v_mov_b32_e32 v1, s5
8562; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8563; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8564; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8565; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8566; GFX7-NEXT:    s_waitcnt vmcnt(0)
8567; GFX7-NEXT:    flat_store_dword v[0:1], v2
8568; GFX7-NEXT:    s_endpgm
8569;
8570; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
8571; GFX10-WGP:       ; %bb.0: ; %entry
8572; GFX10-WGP-NEXT:    s_clause 0x1
8573; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8574; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8575; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
8576; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8577; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8578; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8579; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8580; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8581; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
8582; GFX10-WGP-NEXT:    s_endpgm
8583;
8584; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
8585; GFX10-CU:       ; %bb.0: ; %entry
8586; GFX10-CU-NEXT:    s_clause 0x1
8587; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8588; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8589; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
8590; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8591; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8592; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8593; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8594; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8595; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
8596; GFX10-CU-NEXT:    s_endpgm
8597;
8598; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
8599; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8600; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
8601; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
8602; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
8603; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
8604; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8605; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8606; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8607; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
8608; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8609; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
8610; SKIP-CACHE-INV-NEXT:    s_endpgm
8611;
8612; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
8613; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8614; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8615; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8616; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8617; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8618; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8619; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8620; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8621; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
8622; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8623;
8624; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
8625; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8626; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8627; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8628; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8629; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8630; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8631; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8632; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8633; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
8634; GFX90A-TGSPLIT-NEXT:    s_endpgm
8635    i32 addrspace(1)* %out, i32 %in, i32 %old) {
8636entry:
8637  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
8638  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst seq_cst
8639  %val0 = extractvalue { i32, i1 } %val, 0
8640  store i32 %val0, i32 addrspace(1)* %out, align 4
8641  ret void
8642}
8643
8644