1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx600 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX6 %s
3; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
4; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
5; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
6; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
7; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-NOTTGSPLIT %s
8; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-TGSPLIT %s
9
10define amdgpu_kernel void @global_singlethread_unordered_load(
11; GFX6-LABEL: global_singlethread_unordered_load:
12; GFX6:       ; %bb.0: ; %entry
13; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x0
14; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
15; GFX6-NEXT:    s_mov_b32 s2, -1
16; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
17; GFX6-NEXT:    s_mov_b32 s0, s4
18; GFX6-NEXT:    s_mov_b32 s1, s5
19; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
20; GFX6-NEXT:    s_mov_b32 s4, s6
21; GFX6-NEXT:    s_mov_b32 s5, s7
22; GFX6-NEXT:    s_mov_b32 s6, s2
23; GFX6-NEXT:    s_mov_b32 s7, s3
24; GFX6-NEXT:    s_waitcnt vmcnt(0)
25; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
26; GFX6-NEXT:    s_endpgm
27;
28; GFX7-LABEL: global_singlethread_unordered_load:
29; GFX7:       ; %bb.0: ; %entry
30; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
31; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
32; GFX7-NEXT:    v_mov_b32_e32 v0, s0
33; GFX7-NEXT:    v_mov_b32_e32 v1, s1
34; GFX7-NEXT:    flat_load_dword v0, v[0:1]
35; GFX7-NEXT:    v_mov_b32_e32 v2, s2
36; GFX7-NEXT:    v_mov_b32_e32 v3, s3
37; GFX7-NEXT:    s_waitcnt vmcnt(0)
38; GFX7-NEXT:    flat_store_dword v[2:3], v0
39; GFX7-NEXT:    s_endpgm
40;
41; GFX10-WGP-LABEL: global_singlethread_unordered_load:
42; GFX10-WGP:       ; %bb.0: ; %entry
43; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
44; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
45; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
46; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
47; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
48; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
49; GFX10-WGP-NEXT:    s_endpgm
50;
51; GFX10-CU-LABEL: global_singlethread_unordered_load:
52; GFX10-CU:       ; %bb.0: ; %entry
53; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
54; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
55; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
56; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
57; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
58; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
59; GFX10-CU-NEXT:    s_endpgm
60;
61; SKIP-CACHE-INV-LABEL: global_singlethread_unordered_load:
62; SKIP-CACHE-INV:       ; %bb.0: ; %entry
63; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
64; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
65; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
66; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
67; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
68; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
69; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
70; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
71; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
72; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
73; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
74; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
75; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
76; SKIP-CACHE-INV-NEXT:    s_endpgm
77;
78; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_unordered_load:
79; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
80; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
81; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
82; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
83; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
84; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
85; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
86; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
87;
88; GFX90A-TGSPLIT-LABEL: global_singlethread_unordered_load:
89; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
90; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
91; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
92; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
93; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
94; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
95; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
96; GFX90A-TGSPLIT-NEXT:    s_endpgm
97;
98;
99    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
100entry:
101  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread") unordered, align 4
102  store i32 %val, i32 addrspace(1)* %out
103  ret void
104}
105
106define amdgpu_kernel void @global_singlethread_monotonic_load(
107; GFX6-LABEL: global_singlethread_monotonic_load:
108; GFX6:       ; %bb.0: ; %entry
109; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x0
110; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
111; GFX6-NEXT:    s_mov_b32 s2, -1
112; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
113; GFX6-NEXT:    s_mov_b32 s0, s4
114; GFX6-NEXT:    s_mov_b32 s1, s5
115; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
116; GFX6-NEXT:    s_mov_b32 s4, s6
117; GFX6-NEXT:    s_mov_b32 s5, s7
118; GFX6-NEXT:    s_mov_b32 s6, s2
119; GFX6-NEXT:    s_mov_b32 s7, s3
120; GFX6-NEXT:    s_waitcnt vmcnt(0)
121; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
122; GFX6-NEXT:    s_endpgm
123;
124; GFX7-LABEL: global_singlethread_monotonic_load:
125; GFX7:       ; %bb.0: ; %entry
126; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
127; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
128; GFX7-NEXT:    v_mov_b32_e32 v0, s0
129; GFX7-NEXT:    v_mov_b32_e32 v1, s1
130; GFX7-NEXT:    flat_load_dword v0, v[0:1]
131; GFX7-NEXT:    v_mov_b32_e32 v2, s2
132; GFX7-NEXT:    v_mov_b32_e32 v3, s3
133; GFX7-NEXT:    s_waitcnt vmcnt(0)
134; GFX7-NEXT:    flat_store_dword v[2:3], v0
135; GFX7-NEXT:    s_endpgm
136;
137; GFX10-WGP-LABEL: global_singlethread_monotonic_load:
138; GFX10-WGP:       ; %bb.0: ; %entry
139; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
140; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
141; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
142; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
143; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
144; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
145; GFX10-WGP-NEXT:    s_endpgm
146;
147; GFX10-CU-LABEL: global_singlethread_monotonic_load:
148; GFX10-CU:       ; %bb.0: ; %entry
149; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
150; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
151; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
152; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
153; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
154; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
155; GFX10-CU-NEXT:    s_endpgm
156;
157; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_load:
158; SKIP-CACHE-INV:       ; %bb.0: ; %entry
159; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
160; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
161; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
162; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
163; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
164; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
165; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
166; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
167; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
168; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
169; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
170; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
171; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
172; SKIP-CACHE-INV-NEXT:    s_endpgm
173;
174; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_load:
175; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
176; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
177; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
178; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
179; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
180; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
181; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
182; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
183;
184; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_load:
185; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
186; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
187; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
188; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
189; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
190; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
191; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
192; GFX90A-TGSPLIT-NEXT:    s_endpgm
193;
194;
195    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
196entry:
197  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread") monotonic, align 4
198  store i32 %val, i32 addrspace(1)* %out
199  ret void
200}
201
202define amdgpu_kernel void @global_singlethread_acquire_load(
203; GFX6-LABEL: global_singlethread_acquire_load:
204; GFX6:       ; %bb.0: ; %entry
205; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x0
206; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
207; GFX6-NEXT:    s_mov_b32 s2, -1
208; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
209; GFX6-NEXT:    s_mov_b32 s0, s4
210; GFX6-NEXT:    s_mov_b32 s1, s5
211; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
212; GFX6-NEXT:    s_mov_b32 s4, s6
213; GFX6-NEXT:    s_mov_b32 s5, s7
214; GFX6-NEXT:    s_mov_b32 s6, s2
215; GFX6-NEXT:    s_mov_b32 s7, s3
216; GFX6-NEXT:    s_waitcnt vmcnt(0)
217; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
218; GFX6-NEXT:    s_endpgm
219;
220; GFX7-LABEL: global_singlethread_acquire_load:
221; GFX7:       ; %bb.0: ; %entry
222; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
223; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
224; GFX7-NEXT:    v_mov_b32_e32 v0, s0
225; GFX7-NEXT:    v_mov_b32_e32 v1, s1
226; GFX7-NEXT:    flat_load_dword v0, v[0:1]
227; GFX7-NEXT:    v_mov_b32_e32 v2, s2
228; GFX7-NEXT:    v_mov_b32_e32 v3, s3
229; GFX7-NEXT:    s_waitcnt vmcnt(0)
230; GFX7-NEXT:    flat_store_dword v[2:3], v0
231; GFX7-NEXT:    s_endpgm
232;
233; GFX10-WGP-LABEL: global_singlethread_acquire_load:
234; GFX10-WGP:       ; %bb.0: ; %entry
235; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
236; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
237; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
238; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
239; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
240; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
241; GFX10-WGP-NEXT:    s_endpgm
242;
243; GFX10-CU-LABEL: global_singlethread_acquire_load:
244; GFX10-CU:       ; %bb.0: ; %entry
245; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
246; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
247; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
248; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
249; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
250; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
251; GFX10-CU-NEXT:    s_endpgm
252;
253; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_load:
254; SKIP-CACHE-INV:       ; %bb.0: ; %entry
255; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
256; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
257; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
258; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
259; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
260; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
261; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
262; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
263; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
264; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
265; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
266; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
267; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
268; SKIP-CACHE-INV-NEXT:    s_endpgm
269;
270; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_load:
271; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
272; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
273; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
274; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
275; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
276; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
277; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
278; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
279;
280; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_load:
281; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
282; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
283; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
284; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
285; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
286; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
287; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
288; GFX90A-TGSPLIT-NEXT:    s_endpgm
289;
290;
291    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
292entry:
293  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread") acquire, align 4
294  store i32 %val, i32 addrspace(1)* %out
295  ret void
296}
297
298define amdgpu_kernel void @global_singlethread_seq_cst_load(
299; GFX6-LABEL: global_singlethread_seq_cst_load:
300; GFX6:       ; %bb.0: ; %entry
301; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x0
302; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
303; GFX6-NEXT:    s_mov_b32 s2, -1
304; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
305; GFX6-NEXT:    s_mov_b32 s0, s4
306; GFX6-NEXT:    s_mov_b32 s1, s5
307; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
308; GFX6-NEXT:    s_mov_b32 s4, s6
309; GFX6-NEXT:    s_mov_b32 s5, s7
310; GFX6-NEXT:    s_mov_b32 s6, s2
311; GFX6-NEXT:    s_mov_b32 s7, s3
312; GFX6-NEXT:    s_waitcnt vmcnt(0)
313; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
314; GFX6-NEXT:    s_endpgm
315;
316; GFX7-LABEL: global_singlethread_seq_cst_load:
317; GFX7:       ; %bb.0: ; %entry
318; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
319; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
320; GFX7-NEXT:    v_mov_b32_e32 v0, s0
321; GFX7-NEXT:    v_mov_b32_e32 v1, s1
322; GFX7-NEXT:    flat_load_dword v0, v[0:1]
323; GFX7-NEXT:    v_mov_b32_e32 v2, s2
324; GFX7-NEXT:    v_mov_b32_e32 v3, s3
325; GFX7-NEXT:    s_waitcnt vmcnt(0)
326; GFX7-NEXT:    flat_store_dword v[2:3], v0
327; GFX7-NEXT:    s_endpgm
328;
329; GFX10-WGP-LABEL: global_singlethread_seq_cst_load:
330; GFX10-WGP:       ; %bb.0: ; %entry
331; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
332; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
333; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
334; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
335; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
336; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
337; GFX10-WGP-NEXT:    s_endpgm
338;
339; GFX10-CU-LABEL: global_singlethread_seq_cst_load:
340; GFX10-CU:       ; %bb.0: ; %entry
341; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
342; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
343; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
344; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
345; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
346; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
347; GFX10-CU-NEXT:    s_endpgm
348;
349; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_load:
350; SKIP-CACHE-INV:       ; %bb.0: ; %entry
351; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
352; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
353; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
354; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
355; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
356; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
357; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
358; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
359; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
360; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
361; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
362; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
363; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
364; SKIP-CACHE-INV-NEXT:    s_endpgm
365;
366; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_load:
367; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
368; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
369; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
370; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
371; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
372; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
373; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
374; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
375;
376; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_load:
377; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
378; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
379; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
380; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
381; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
382; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
383; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
384; GFX90A-TGSPLIT-NEXT:    s_endpgm
385;
386;
387    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
388entry:
389  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread") seq_cst, align 4
390  store i32 %val, i32 addrspace(1)* %out
391  ret void
392}
393
394define amdgpu_kernel void @global_singlethread_unordered_store(
395; GFX6-LABEL: global_singlethread_unordered_store:
396; GFX6:       ; %bb.0: ; %entry
397; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
398; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
399; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
400; GFX6-NEXT:    s_mov_b32 s2, -1
401; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
402; GFX6-NEXT:    v_mov_b32_e32 v0, s6
403; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
404; GFX6-NEXT:    s_endpgm
405;
406; GFX7-LABEL: global_singlethread_unordered_store:
407; GFX7:       ; %bb.0: ; %entry
408; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
409; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
410; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
411; GFX7-NEXT:    v_mov_b32_e32 v2, s2
412; GFX7-NEXT:    v_mov_b32_e32 v0, s0
413; GFX7-NEXT:    v_mov_b32_e32 v1, s1
414; GFX7-NEXT:    flat_store_dword v[0:1], v2
415; GFX7-NEXT:    s_endpgm
416;
417; GFX10-WGP-LABEL: global_singlethread_unordered_store:
418; GFX10-WGP:       ; %bb.0: ; %entry
419; GFX10-WGP-NEXT:    s_clause 0x1
420; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
421; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
422; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
423; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
424; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
425; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
426; GFX10-WGP-NEXT:    s_endpgm
427;
428; GFX10-CU-LABEL: global_singlethread_unordered_store:
429; GFX10-CU:       ; %bb.0: ; %entry
430; GFX10-CU-NEXT:    s_clause 0x1
431; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
432; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
433; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
434; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
435; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
436; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
437; GFX10-CU-NEXT:    s_endpgm
438;
439; SKIP-CACHE-INV-LABEL: global_singlethread_unordered_store:
440; SKIP-CACHE-INV:       ; %bb.0: ; %entry
441; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
442; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
443; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
444; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
445; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
446; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
447; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
448; SKIP-CACHE-INV-NEXT:    s_endpgm
449;
450; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_unordered_store:
451; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
452; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
453; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
454; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
455; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
456; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
457; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
458; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
459;
460; GFX90A-TGSPLIT-LABEL: global_singlethread_unordered_store:
461; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
462; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
463; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
464; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
465; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
466; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
467; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
468; GFX90A-TGSPLIT-NEXT:    s_endpgm
469;
470;
471    i32 %in, i32 addrspace(1)* %out) {
472entry:
473  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread") unordered, align 4
474  ret void
475}
476
477define amdgpu_kernel void @global_singlethread_monotonic_store(
478; GFX6-LABEL: global_singlethread_monotonic_store:
479; GFX6:       ; %bb.0: ; %entry
480; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
481; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
482; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
483; GFX6-NEXT:    s_mov_b32 s2, -1
484; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
485; GFX6-NEXT:    v_mov_b32_e32 v0, s6
486; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
487; GFX6-NEXT:    s_endpgm
488;
489; GFX7-LABEL: global_singlethread_monotonic_store:
490; GFX7:       ; %bb.0: ; %entry
491; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
492; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
493; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
494; GFX7-NEXT:    v_mov_b32_e32 v2, s2
495; GFX7-NEXT:    v_mov_b32_e32 v0, s0
496; GFX7-NEXT:    v_mov_b32_e32 v1, s1
497; GFX7-NEXT:    flat_store_dword v[0:1], v2
498; GFX7-NEXT:    s_endpgm
499;
500; GFX10-WGP-LABEL: global_singlethread_monotonic_store:
501; GFX10-WGP:       ; %bb.0: ; %entry
502; GFX10-WGP-NEXT:    s_clause 0x1
503; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
504; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
505; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
506; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
507; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
508; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
509; GFX10-WGP-NEXT:    s_endpgm
510;
511; GFX10-CU-LABEL: global_singlethread_monotonic_store:
512; GFX10-CU:       ; %bb.0: ; %entry
513; GFX10-CU-NEXT:    s_clause 0x1
514; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
515; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
516; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
517; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
518; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
519; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
520; GFX10-CU-NEXT:    s_endpgm
521;
522; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_store:
523; SKIP-CACHE-INV:       ; %bb.0: ; %entry
524; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
525; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
526; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
527; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
528; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
529; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
530; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
531; SKIP-CACHE-INV-NEXT:    s_endpgm
532;
533; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_store:
534; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
535; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
536; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
537; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
538; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
539; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
540; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
541; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
542;
543; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_store:
544; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
545; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
546; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
547; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
548; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
549; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
550; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
551; GFX90A-TGSPLIT-NEXT:    s_endpgm
552;
553;
554    i32 %in, i32 addrspace(1)* %out) {
555entry:
556  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread") monotonic, align 4
557  ret void
558}
559
560define amdgpu_kernel void @global_singlethread_release_store(
561; GFX6-LABEL: global_singlethread_release_store:
562; GFX6:       ; %bb.0: ; %entry
563; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
564; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
565; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
566; GFX6-NEXT:    s_mov_b32 s2, -1
567; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
568; GFX6-NEXT:    v_mov_b32_e32 v0, s6
569; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
570; GFX6-NEXT:    s_endpgm
571;
572; GFX7-LABEL: global_singlethread_release_store:
573; GFX7:       ; %bb.0: ; %entry
574; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
575; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
576; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
577; GFX7-NEXT:    v_mov_b32_e32 v2, s2
578; GFX7-NEXT:    v_mov_b32_e32 v0, s0
579; GFX7-NEXT:    v_mov_b32_e32 v1, s1
580; GFX7-NEXT:    flat_store_dword v[0:1], v2
581; GFX7-NEXT:    s_endpgm
582;
583; GFX10-WGP-LABEL: global_singlethread_release_store:
584; GFX10-WGP:       ; %bb.0: ; %entry
585; GFX10-WGP-NEXT:    s_clause 0x1
586; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
587; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
588; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
589; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
590; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
591; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
592; GFX10-WGP-NEXT:    s_endpgm
593;
594; GFX10-CU-LABEL: global_singlethread_release_store:
595; GFX10-CU:       ; %bb.0: ; %entry
596; GFX10-CU-NEXT:    s_clause 0x1
597; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
598; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
599; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
600; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
601; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
602; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
603; GFX10-CU-NEXT:    s_endpgm
604;
605; SKIP-CACHE-INV-LABEL: global_singlethread_release_store:
606; SKIP-CACHE-INV:       ; %bb.0: ; %entry
607; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
608; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
609; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
610; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
611; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
612; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
613; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
614; SKIP-CACHE-INV-NEXT:    s_endpgm
615;
616; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_release_store:
617; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
618; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
619; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
620; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
621; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
622; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
623; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
624; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
625;
626; GFX90A-TGSPLIT-LABEL: global_singlethread_release_store:
627; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
628; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
629; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
630; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
631; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
632; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
633; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
634; GFX90A-TGSPLIT-NEXT:    s_endpgm
635;
636;
637    i32 %in, i32 addrspace(1)* %out) {
638entry:
639  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread") release, align 4
640  ret void
641}
642
643define amdgpu_kernel void @global_singlethread_seq_cst_store(
644; GFX6-LABEL: global_singlethread_seq_cst_store:
645; GFX6:       ; %bb.0: ; %entry
646; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
647; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
648; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
649; GFX6-NEXT:    s_mov_b32 s2, -1
650; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
651; GFX6-NEXT:    v_mov_b32_e32 v0, s6
652; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
653; GFX6-NEXT:    s_endpgm
654;
655; GFX7-LABEL: global_singlethread_seq_cst_store:
656; GFX7:       ; %bb.0: ; %entry
657; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
658; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
659; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
660; GFX7-NEXT:    v_mov_b32_e32 v2, s2
661; GFX7-NEXT:    v_mov_b32_e32 v0, s0
662; GFX7-NEXT:    v_mov_b32_e32 v1, s1
663; GFX7-NEXT:    flat_store_dword v[0:1], v2
664; GFX7-NEXT:    s_endpgm
665;
666; GFX10-WGP-LABEL: global_singlethread_seq_cst_store:
667; GFX10-WGP:       ; %bb.0: ; %entry
668; GFX10-WGP-NEXT:    s_clause 0x1
669; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
670; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
671; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
672; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
673; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
674; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
675; GFX10-WGP-NEXT:    s_endpgm
676;
677; GFX10-CU-LABEL: global_singlethread_seq_cst_store:
678; GFX10-CU:       ; %bb.0: ; %entry
679; GFX10-CU-NEXT:    s_clause 0x1
680; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
681; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
682; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
683; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
684; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
685; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
686; GFX10-CU-NEXT:    s_endpgm
687;
688; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_store:
689; SKIP-CACHE-INV:       ; %bb.0: ; %entry
690; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
691; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
692; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
693; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
694; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
695; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
696; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
697; SKIP-CACHE-INV-NEXT:    s_endpgm
698;
699; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_store:
700; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
701; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
702; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
703; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
704; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
705; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
706; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
707; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
708;
709; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_store:
710; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
711; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
712; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
713; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
714; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
715; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
716; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
717; GFX90A-TGSPLIT-NEXT:    s_endpgm
718;
719;
720    i32 %in, i32 addrspace(1)* %out) {
721entry:
722  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread") seq_cst, align 4
723  ret void
724}
725
726define amdgpu_kernel void @global_singlethread_monotonic_atomicrmw(
727; GFX6-LABEL: global_singlethread_monotonic_atomicrmw:
728; GFX6:       ; %bb.0: ; %entry
729; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
730; GFX6-NEXT:    s_load_dword s4, s[4:5], 0x2
731; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
732; GFX6-NEXT:    s_mov_b32 s2, -1
733; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
734; GFX6-NEXT:    v_mov_b32_e32 v0, s4
735; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
736; GFX6-NEXT:    s_endpgm
737;
738; GFX7-LABEL: global_singlethread_monotonic_atomicrmw:
739; GFX7:       ; %bb.0: ; %entry
740; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
741; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
742; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
743; GFX7-NEXT:    v_mov_b32_e32 v0, s0
744; GFX7-NEXT:    v_mov_b32_e32 v1, s1
745; GFX7-NEXT:    v_mov_b32_e32 v2, s2
746; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
747; GFX7-NEXT:    s_endpgm
748;
749; GFX10-WGP-LABEL: global_singlethread_monotonic_atomicrmw:
750; GFX10-WGP:       ; %bb.0: ; %entry
751; GFX10-WGP-NEXT:    s_clause 0x1
752; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
753; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
754; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
755; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
756; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
757; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
758; GFX10-WGP-NEXT:    s_endpgm
759;
760; GFX10-CU-LABEL: global_singlethread_monotonic_atomicrmw:
761; GFX10-CU:       ; %bb.0: ; %entry
762; GFX10-CU-NEXT:    s_clause 0x1
763; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
764; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
765; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
766; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
767; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
768; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
769; GFX10-CU-NEXT:    s_endpgm
770;
771; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_atomicrmw:
772; SKIP-CACHE-INV:       ; %bb.0: ; %entry
773; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
774; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
775; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
776; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
777; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
778; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
779; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
780; SKIP-CACHE-INV-NEXT:    s_endpgm
781;
782; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_atomicrmw:
783; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
784; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
785; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
786; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
787; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
788; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
789; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
790; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
791;
792; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_atomicrmw:
793; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
794; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
795; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
796; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
797; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
798; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
799; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
800; GFX90A-TGSPLIT-NEXT:    s_endpgm
801;
802;
803    i32 addrspace(1)* %out, i32 %in) {
804entry:
805  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") monotonic
806  ret void
807}
808
809define amdgpu_kernel void @global_singlethread_acquire_atomicrmw(
810; GFX6-LABEL: global_singlethread_acquire_atomicrmw:
811; GFX6:       ; %bb.0: ; %entry
812; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
813; GFX6-NEXT:    s_load_dword s4, s[4:5], 0x2
814; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
815; GFX6-NEXT:    s_mov_b32 s2, -1
816; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
817; GFX6-NEXT:    v_mov_b32_e32 v0, s4
818; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
819; GFX6-NEXT:    s_endpgm
820;
821; GFX7-LABEL: global_singlethread_acquire_atomicrmw:
822; GFX7:       ; %bb.0: ; %entry
823; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
824; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
825; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
826; GFX7-NEXT:    v_mov_b32_e32 v0, s0
827; GFX7-NEXT:    v_mov_b32_e32 v1, s1
828; GFX7-NEXT:    v_mov_b32_e32 v2, s2
829; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
830; GFX7-NEXT:    s_endpgm
831;
832; GFX10-WGP-LABEL: global_singlethread_acquire_atomicrmw:
833; GFX10-WGP:       ; %bb.0: ; %entry
834; GFX10-WGP-NEXT:    s_clause 0x1
835; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
836; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
837; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
838; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
839; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
840; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
841; GFX10-WGP-NEXT:    s_endpgm
842;
843; GFX10-CU-LABEL: global_singlethread_acquire_atomicrmw:
844; GFX10-CU:       ; %bb.0: ; %entry
845; GFX10-CU-NEXT:    s_clause 0x1
846; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
847; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
848; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
849; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
850; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
851; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
852; GFX10-CU-NEXT:    s_endpgm
853;
854; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_atomicrmw:
855; SKIP-CACHE-INV:       ; %bb.0: ; %entry
856; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
857; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
858; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
859; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
860; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
861; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
862; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
863; SKIP-CACHE-INV-NEXT:    s_endpgm
864;
865; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_atomicrmw:
866; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
867; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
868; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
869; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
870; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
871; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
872; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
873; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
874;
875; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_atomicrmw:
876; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
877; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
878; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
879; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
880; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
881; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
882; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
883; GFX90A-TGSPLIT-NEXT:    s_endpgm
884;
885;
886    i32 addrspace(1)* %out, i32 %in) {
887entry:
888  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") acquire
889  ret void
890}
891
892define amdgpu_kernel void @global_singlethread_release_atomicrmw(
893; GFX6-LABEL: global_singlethread_release_atomicrmw:
894; GFX6:       ; %bb.0: ; %entry
895; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
896; GFX6-NEXT:    s_load_dword s4, s[4:5], 0x2
897; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
898; GFX6-NEXT:    s_mov_b32 s2, -1
899; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
900; GFX6-NEXT:    v_mov_b32_e32 v0, s4
901; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
902; GFX6-NEXT:    s_endpgm
903;
904; GFX7-LABEL: global_singlethread_release_atomicrmw:
905; GFX7:       ; %bb.0: ; %entry
906; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
907; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
908; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
909; GFX7-NEXT:    v_mov_b32_e32 v0, s0
910; GFX7-NEXT:    v_mov_b32_e32 v1, s1
911; GFX7-NEXT:    v_mov_b32_e32 v2, s2
912; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
913; GFX7-NEXT:    s_endpgm
914;
915; GFX10-WGP-LABEL: global_singlethread_release_atomicrmw:
916; GFX10-WGP:       ; %bb.0: ; %entry
917; GFX10-WGP-NEXT:    s_clause 0x1
918; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
919; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
920; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
921; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
922; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
923; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
924; GFX10-WGP-NEXT:    s_endpgm
925;
926; GFX10-CU-LABEL: global_singlethread_release_atomicrmw:
927; GFX10-CU:       ; %bb.0: ; %entry
928; GFX10-CU-NEXT:    s_clause 0x1
929; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
930; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
931; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
932; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
933; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
934; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
935; GFX10-CU-NEXT:    s_endpgm
936;
937; SKIP-CACHE-INV-LABEL: global_singlethread_release_atomicrmw:
938; SKIP-CACHE-INV:       ; %bb.0: ; %entry
939; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
940; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
941; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
942; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
943; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
944; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
945; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
946; SKIP-CACHE-INV-NEXT:    s_endpgm
947;
948; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_release_atomicrmw:
949; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
950; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
951; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
952; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
953; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
954; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
955; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
956; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
957;
958; GFX90A-TGSPLIT-LABEL: global_singlethread_release_atomicrmw:
959; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
960; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
961; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
962; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
963; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
964; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
965; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
966; GFX90A-TGSPLIT-NEXT:    s_endpgm
967;
968;
969    i32 addrspace(1)* %out, i32 %in) {
970entry:
971  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") release
972  ret void
973}
974
975define amdgpu_kernel void @global_singlethread_acq_rel_atomicrmw(
976; GFX6-LABEL: global_singlethread_acq_rel_atomicrmw:
977; GFX6:       ; %bb.0: ; %entry
978; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
979; GFX6-NEXT:    s_load_dword s4, s[4:5], 0x2
980; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
981; GFX6-NEXT:    s_mov_b32 s2, -1
982; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
983; GFX6-NEXT:    v_mov_b32_e32 v0, s4
984; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
985; GFX6-NEXT:    s_endpgm
986;
987; GFX7-LABEL: global_singlethread_acq_rel_atomicrmw:
988; GFX7:       ; %bb.0: ; %entry
989; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
990; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
991; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
992; GFX7-NEXT:    v_mov_b32_e32 v0, s0
993; GFX7-NEXT:    v_mov_b32_e32 v1, s1
994; GFX7-NEXT:    v_mov_b32_e32 v2, s2
995; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
996; GFX7-NEXT:    s_endpgm
997;
998; GFX10-WGP-LABEL: global_singlethread_acq_rel_atomicrmw:
999; GFX10-WGP:       ; %bb.0: ; %entry
1000; GFX10-WGP-NEXT:    s_clause 0x1
1001; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1002; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1003; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
1004; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1005; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1006; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
1007; GFX10-WGP-NEXT:    s_endpgm
1008;
1009; GFX10-CU-LABEL: global_singlethread_acq_rel_atomicrmw:
1010; GFX10-CU:       ; %bb.0: ; %entry
1011; GFX10-CU-NEXT:    s_clause 0x1
1012; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1013; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1014; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
1015; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1016; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1017; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
1018; GFX10-CU-NEXT:    s_endpgm
1019;
1020; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_atomicrmw:
1021; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1022; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
1023; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
1024; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
1025; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
1026; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1027; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1028; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
1029; SKIP-CACHE-INV-NEXT:    s_endpgm
1030;
1031; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_atomicrmw:
1032; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1033; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1034; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1035; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1036; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1037; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1038; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1039; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1040;
1041; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_atomicrmw:
1042; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1043; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1044; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1045; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1046; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1047; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1048; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1049; GFX90A-TGSPLIT-NEXT:    s_endpgm
1050;
1051;
1052    i32 addrspace(1)* %out, i32 %in) {
1053entry:
1054  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") acq_rel
1055  ret void
1056}
1057
1058define amdgpu_kernel void @global_singlethread_seq_cst_atomicrmw(
1059; GFX6-LABEL: global_singlethread_seq_cst_atomicrmw:
1060; GFX6:       ; %bb.0: ; %entry
1061; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1062; GFX6-NEXT:    s_load_dword s4, s[4:5], 0x2
1063; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1064; GFX6-NEXT:    s_mov_b32 s2, -1
1065; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1066; GFX6-NEXT:    v_mov_b32_e32 v0, s4
1067; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
1068; GFX6-NEXT:    s_endpgm
1069;
1070; GFX7-LABEL: global_singlethread_seq_cst_atomicrmw:
1071; GFX7:       ; %bb.0: ; %entry
1072; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1073; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1074; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1075; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1076; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1077; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1078; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1079; GFX7-NEXT:    s_endpgm
1080;
1081; GFX10-WGP-LABEL: global_singlethread_seq_cst_atomicrmw:
1082; GFX10-WGP:       ; %bb.0: ; %entry
1083; GFX10-WGP-NEXT:    s_clause 0x1
1084; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1085; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1086; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
1087; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1088; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1089; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
1090; GFX10-WGP-NEXT:    s_endpgm
1091;
1092; GFX10-CU-LABEL: global_singlethread_seq_cst_atomicrmw:
1093; GFX10-CU:       ; %bb.0: ; %entry
1094; GFX10-CU-NEXT:    s_clause 0x1
1095; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1096; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1097; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
1098; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1099; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1100; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
1101; GFX10-CU-NEXT:    s_endpgm
1102;
1103; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_atomicrmw:
1104; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1105; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
1106; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
1107; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
1108; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
1109; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1110; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1111; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
1112; SKIP-CACHE-INV-NEXT:    s_endpgm
1113;
1114; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_atomicrmw:
1115; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1116; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1117; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1118; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1119; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1120; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1121; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1122; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1123;
1124; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_atomicrmw:
1125; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1126; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1127; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1128; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1129; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1130; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1131; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1132; GFX90A-TGSPLIT-NEXT:    s_endpgm
1133;
1134;
1135    i32 addrspace(1)* %out, i32 %in) {
1136entry:
1137  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") seq_cst
1138  ret void
1139}
1140
1141define amdgpu_kernel void @global_singlethread_acquire_ret_atomicrmw(
1142; GFX6-LABEL: global_singlethread_acquire_ret_atomicrmw:
1143; GFX6:       ; %bb.0: ; %entry
1144; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1145; GFX6-NEXT:    s_load_dword s4, s[4:5], 0x2
1146; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1147; GFX6-NEXT:    s_mov_b32 s2, -1
1148; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1149; GFX6-NEXT:    v_mov_b32_e32 v0, s4
1150; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
1151; GFX6-NEXT:    s_waitcnt vmcnt(0)
1152; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1153; GFX6-NEXT:    s_endpgm
1154;
1155; GFX7-LABEL: global_singlethread_acquire_ret_atomicrmw:
1156; GFX7:       ; %bb.0: ; %entry
1157; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1158; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1159; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1160; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1161; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1162; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1163; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1164; GFX7-NEXT:    s_waitcnt vmcnt(0)
1165; GFX7-NEXT:    flat_store_dword v[0:1], v2
1166; GFX7-NEXT:    s_endpgm
1167;
1168; GFX10-WGP-LABEL: global_singlethread_acquire_ret_atomicrmw:
1169; GFX10-WGP:       ; %bb.0: ; %entry
1170; GFX10-WGP-NEXT:    s_clause 0x1
1171; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1172; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1173; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
1174; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1175; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1176; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1177; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
1178; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
1179; GFX10-WGP-NEXT:    s_endpgm
1180;
1181; GFX10-CU-LABEL: global_singlethread_acquire_ret_atomicrmw:
1182; GFX10-CU:       ; %bb.0: ; %entry
1183; GFX10-CU-NEXT:    s_clause 0x1
1184; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1185; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1186; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
1187; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1188; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1189; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1190; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
1191; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
1192; GFX10-CU-NEXT:    s_endpgm
1193;
1194; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_ret_atomicrmw:
1195; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1196; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
1197; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
1198; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
1199; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
1200; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1201; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1202; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
1203; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
1204; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
1205; SKIP-CACHE-INV-NEXT:    s_endpgm
1206;
1207; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_ret_atomicrmw:
1208; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1209; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1210; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1211; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1212; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1213; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1214; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1215; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1216; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
1217; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1218;
1219; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_ret_atomicrmw:
1220; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1221; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1222; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1223; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1224; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1225; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1226; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1227; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1228; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
1229; GFX90A-TGSPLIT-NEXT:    s_endpgm
1230;
1231;
1232    i32 addrspace(1)* %out, i32 %in) {
1233entry:
1234  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") acquire
1235  store i32 %val, i32 addrspace(1)* %out, align 4
1236  ret void
1237}
1238
1239define amdgpu_kernel void @global_singlethread_acq_rel_ret_atomicrmw(
1240; GFX6-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1241; GFX6:       ; %bb.0: ; %entry
1242; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1243; GFX6-NEXT:    s_load_dword s4, s[4:5], 0x2
1244; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1245; GFX6-NEXT:    s_mov_b32 s2, -1
1246; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1247; GFX6-NEXT:    v_mov_b32_e32 v0, s4
1248; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
1249; GFX6-NEXT:    s_waitcnt vmcnt(0)
1250; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1251; GFX6-NEXT:    s_endpgm
1252;
1253; GFX7-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1254; GFX7:       ; %bb.0: ; %entry
1255; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1256; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1257; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1258; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1259; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1260; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1261; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1262; GFX7-NEXT:    s_waitcnt vmcnt(0)
1263; GFX7-NEXT:    flat_store_dword v[0:1], v2
1264; GFX7-NEXT:    s_endpgm
1265;
1266; GFX10-WGP-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1267; GFX10-WGP:       ; %bb.0: ; %entry
1268; GFX10-WGP-NEXT:    s_clause 0x1
1269; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1270; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1271; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
1272; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1273; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1274; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1275; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
1276; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
1277; GFX10-WGP-NEXT:    s_endpgm
1278;
1279; GFX10-CU-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1280; GFX10-CU:       ; %bb.0: ; %entry
1281; GFX10-CU-NEXT:    s_clause 0x1
1282; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1283; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1284; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
1285; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1286; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1287; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1288; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
1289; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
1290; GFX10-CU-NEXT:    s_endpgm
1291;
1292; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1293; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1294; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
1295; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
1296; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
1297; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
1298; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1299; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1300; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
1301; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
1302; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
1303; SKIP-CACHE-INV-NEXT:    s_endpgm
1304;
1305; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1306; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1307; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1308; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1309; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1310; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1311; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1312; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1313; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1314; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
1315; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1316;
1317; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1318; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1319; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1320; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1321; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1322; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1323; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1324; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1325; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1326; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
1327; GFX90A-TGSPLIT-NEXT:    s_endpgm
1328;
1329;
1330    i32 addrspace(1)* %out, i32 %in) {
1331entry:
1332  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") acq_rel
1333  store i32 %val, i32 addrspace(1)* %out, align 4
1334  ret void
1335}
1336
1337define amdgpu_kernel void @global_singlethread_seq_cst_ret_atomicrmw(
1338; GFX6-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
1339; GFX6:       ; %bb.0: ; %entry
1340; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1341; GFX6-NEXT:    s_load_dword s4, s[4:5], 0x2
1342; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1343; GFX6-NEXT:    s_mov_b32 s2, -1
1344; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1345; GFX6-NEXT:    v_mov_b32_e32 v0, s4
1346; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
1347; GFX6-NEXT:    s_waitcnt vmcnt(0)
1348; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1349; GFX6-NEXT:    s_endpgm
1350;
1351; GFX7-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
1352; GFX7:       ; %bb.0: ; %entry
1353; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1354; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1355; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1356; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1357; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1358; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1359; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1360; GFX7-NEXT:    s_waitcnt vmcnt(0)
1361; GFX7-NEXT:    flat_store_dword v[0:1], v2
1362; GFX7-NEXT:    s_endpgm
1363;
1364; GFX10-WGP-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
1365; GFX10-WGP:       ; %bb.0: ; %entry
1366; GFX10-WGP-NEXT:    s_clause 0x1
1367; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1368; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1369; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
1370; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1371; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1372; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1373; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
1374; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
1375; GFX10-WGP-NEXT:    s_endpgm
1376;
1377; GFX10-CU-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
1378; GFX10-CU:       ; %bb.0: ; %entry
1379; GFX10-CU-NEXT:    s_clause 0x1
1380; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1381; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1382; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
1383; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1384; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1385; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1386; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
1387; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
1388; GFX10-CU-NEXT:    s_endpgm
1389;
1390; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
1391; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1392; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
1393; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
1394; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
1395; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
1396; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1397; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1398; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
1399; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
1400; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
1401; SKIP-CACHE-INV-NEXT:    s_endpgm
1402;
1403; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
1404; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1405; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1406; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1407; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1408; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1409; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1410; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1411; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1412; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
1413; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1414;
1415; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
1416; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1417; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1418; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1419; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1420; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1421; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1422; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1423; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1424; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
1425; GFX90A-TGSPLIT-NEXT:    s_endpgm
1426;
1427;
1428    i32 addrspace(1)* %out, i32 %in) {
1429entry:
1430  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") seq_cst
1431  store i32 %val, i32 addrspace(1)* %out, align 4
1432  ret void
1433}
1434
1435define amdgpu_kernel void @global_singlethread_monotonic_monotonic_cmpxchg(
1436; GFX6-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
1437; GFX6:       ; %bb.0: ; %entry
1438; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1439; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
1440; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1441; GFX6-NEXT:    s_mov_b32 s2, -1
1442; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1443; GFX6-NEXT:    v_mov_b32_e32 v0, s4
1444; GFX6-NEXT:    v_mov_b32_e32 v1, s5
1445; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
1446; GFX6-NEXT:    s_endpgm
1447;
1448; GFX7-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
1449; GFX7:       ; %bb.0: ; %entry
1450; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1451; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1452; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1453; GFX7-NEXT:    s_add_u32 s0, s0, 16
1454; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1455; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1456; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1457; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1458; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1459; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1460; GFX7-NEXT:    s_endpgm
1461;
1462; GFX10-WGP-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
1463; GFX10-WGP:       ; %bb.0: ; %entry
1464; GFX10-WGP-NEXT:    s_clause 0x1
1465; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1466; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1467; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
1468; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1469; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
1470; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
1471; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
1472; GFX10-WGP-NEXT:    s_endpgm
1473;
1474; GFX10-CU-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
1475; GFX10-CU:       ; %bb.0: ; %entry
1476; GFX10-CU-NEXT:    s_clause 0x1
1477; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1478; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1479; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
1480; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1481; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
1482; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
1483; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
1484; GFX10-CU-NEXT:    s_endpgm
1485;
1486; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
1487; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1488; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
1489; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
1490; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
1491; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
1492; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1493; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1494; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
1495; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
1496; SKIP-CACHE-INV-NEXT:    s_endpgm
1497;
1498; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
1499; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1500; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1501; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1502; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1503; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1504; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
1505; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
1506; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1507;
1508; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
1509; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1510; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1511; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1512; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1513; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1514; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
1515; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
1516; GFX90A-TGSPLIT-NEXT:    s_endpgm
1517;
1518;
1519    i32 addrspace(1)* %out, i32 %in, i32 %old) {
1520entry:
1521  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
1522  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic monotonic
1523  ret void
1524}
1525
1526define amdgpu_kernel void @global_singlethread_acquire_monotonic_cmpxchg(
1527; GFX6-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
1528; GFX6:       ; %bb.0: ; %entry
1529; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1530; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
1531; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1532; GFX6-NEXT:    s_mov_b32 s2, -1
1533; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1534; GFX6-NEXT:    v_mov_b32_e32 v0, s4
1535; GFX6-NEXT:    v_mov_b32_e32 v1, s5
1536; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
1537; GFX6-NEXT:    s_endpgm
1538;
1539; GFX7-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
1540; GFX7:       ; %bb.0: ; %entry
1541; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1542; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1543; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1544; GFX7-NEXT:    s_add_u32 s0, s0, 16
1545; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1546; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1547; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1548; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1549; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1550; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1551; GFX7-NEXT:    s_endpgm
1552;
1553; GFX10-WGP-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
1554; GFX10-WGP:       ; %bb.0: ; %entry
1555; GFX10-WGP-NEXT:    s_clause 0x1
1556; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1557; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1558; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
1559; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1560; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
1561; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
1562; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
1563; GFX10-WGP-NEXT:    s_endpgm
1564;
1565; GFX10-CU-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
1566; GFX10-CU:       ; %bb.0: ; %entry
1567; GFX10-CU-NEXT:    s_clause 0x1
1568; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1569; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1570; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
1571; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1572; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
1573; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
1574; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
1575; GFX10-CU-NEXT:    s_endpgm
1576;
1577; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
1578; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1579; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
1580; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
1581; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
1582; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
1583; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1584; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1585; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
1586; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
1587; SKIP-CACHE-INV-NEXT:    s_endpgm
1588;
1589; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
1590; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1591; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1592; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1593; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1594; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1595; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
1596; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
1597; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1598;
1599; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
1600; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1601; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1602; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1603; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1604; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1605; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
1606; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
1607; GFX90A-TGSPLIT-NEXT:    s_endpgm
1608;
1609;
1610    i32 addrspace(1)* %out, i32 %in, i32 %old) {
1611entry:
1612  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
1613  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire monotonic
1614  ret void
1615}
1616
1617define amdgpu_kernel void @global_singlethread_release_monotonic_cmpxchg(
1618; GFX6-LABEL: global_singlethread_release_monotonic_cmpxchg:
1619; GFX6:       ; %bb.0: ; %entry
1620; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1621; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
1622; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1623; GFX6-NEXT:    s_mov_b32 s2, -1
1624; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1625; GFX6-NEXT:    v_mov_b32_e32 v0, s4
1626; GFX6-NEXT:    v_mov_b32_e32 v1, s5
1627; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
1628; GFX6-NEXT:    s_endpgm
1629;
1630; GFX7-LABEL: global_singlethread_release_monotonic_cmpxchg:
1631; GFX7:       ; %bb.0: ; %entry
1632; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1633; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1634; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1635; GFX7-NEXT:    s_add_u32 s0, s0, 16
1636; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1637; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1638; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1639; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1640; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1641; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1642; GFX7-NEXT:    s_endpgm
1643;
1644; GFX10-WGP-LABEL: global_singlethread_release_monotonic_cmpxchg:
1645; GFX10-WGP:       ; %bb.0: ; %entry
1646; GFX10-WGP-NEXT:    s_clause 0x1
1647; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1648; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1649; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
1650; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1651; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
1652; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
1653; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
1654; GFX10-WGP-NEXT:    s_endpgm
1655;
1656; GFX10-CU-LABEL: global_singlethread_release_monotonic_cmpxchg:
1657; GFX10-CU:       ; %bb.0: ; %entry
1658; GFX10-CU-NEXT:    s_clause 0x1
1659; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1660; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1661; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
1662; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1663; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
1664; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
1665; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
1666; GFX10-CU-NEXT:    s_endpgm
1667;
1668; SKIP-CACHE-INV-LABEL: global_singlethread_release_monotonic_cmpxchg:
1669; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1670; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
1671; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
1672; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
1673; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
1674; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1675; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1676; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
1677; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
1678; SKIP-CACHE-INV-NEXT:    s_endpgm
1679;
1680; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_release_monotonic_cmpxchg:
1681; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1682; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1683; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1684; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1685; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1686; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
1687; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
1688; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1689;
1690; GFX90A-TGSPLIT-LABEL: global_singlethread_release_monotonic_cmpxchg:
1691; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1692; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1693; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1694; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1695; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1696; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
1697; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
1698; GFX90A-TGSPLIT-NEXT:    s_endpgm
1699;
1700;
1701    i32 addrspace(1)* %out, i32 %in, i32 %old) {
1702entry:
1703  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
1704  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") release monotonic
1705  ret void
1706}
1707
1708define amdgpu_kernel void @global_singlethread_acq_rel_monotonic_cmpxchg(
1709; GFX6-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
1710; GFX6:       ; %bb.0: ; %entry
1711; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1712; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
1713; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1714; GFX6-NEXT:    s_mov_b32 s2, -1
1715; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1716; GFX6-NEXT:    v_mov_b32_e32 v0, s4
1717; GFX6-NEXT:    v_mov_b32_e32 v1, s5
1718; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
1719; GFX6-NEXT:    s_endpgm
1720;
1721; GFX7-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
1722; GFX7:       ; %bb.0: ; %entry
1723; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1724; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1725; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1726; GFX7-NEXT:    s_add_u32 s0, s0, 16
1727; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1728; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1729; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1730; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1731; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1732; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1733; GFX7-NEXT:    s_endpgm
1734;
1735; GFX10-WGP-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
1736; GFX10-WGP:       ; %bb.0: ; %entry
1737; GFX10-WGP-NEXT:    s_clause 0x1
1738; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1739; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1740; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
1741; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1742; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
1743; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
1744; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
1745; GFX10-WGP-NEXT:    s_endpgm
1746;
1747; GFX10-CU-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
1748; GFX10-CU:       ; %bb.0: ; %entry
1749; GFX10-CU-NEXT:    s_clause 0x1
1750; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1751; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1752; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
1753; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1754; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
1755; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
1756; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
1757; GFX10-CU-NEXT:    s_endpgm
1758;
1759; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
1760; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1761; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
1762; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
1763; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
1764; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
1765; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1766; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1767; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
1768; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
1769; SKIP-CACHE-INV-NEXT:    s_endpgm
1770;
1771; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
1772; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1773; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1774; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1775; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1776; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1777; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
1778; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
1779; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1780;
1781; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
1782; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1783; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1784; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1785; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1786; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1787; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
1788; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
1789; GFX90A-TGSPLIT-NEXT:    s_endpgm
1790;
1791;
1792    i32 addrspace(1)* %out, i32 %in, i32 %old) {
1793entry:
1794  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
1795  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel monotonic
1796  ret void
1797}
1798
1799define amdgpu_kernel void @global_singlethread_seq_cst_monotonic_cmpxchg(
1800; GFX6-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
1801; GFX6:       ; %bb.0: ; %entry
1802; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1803; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
1804; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1805; GFX6-NEXT:    s_mov_b32 s2, -1
1806; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1807; GFX6-NEXT:    v_mov_b32_e32 v0, s4
1808; GFX6-NEXT:    v_mov_b32_e32 v1, s5
1809; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
1810; GFX6-NEXT:    s_endpgm
1811;
1812; GFX7-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
1813; GFX7:       ; %bb.0: ; %entry
1814; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1815; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1816; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1817; GFX7-NEXT:    s_add_u32 s0, s0, 16
1818; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1819; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1820; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1821; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1822; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1823; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1824; GFX7-NEXT:    s_endpgm
1825;
1826; GFX10-WGP-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
1827; GFX10-WGP:       ; %bb.0: ; %entry
1828; GFX10-WGP-NEXT:    s_clause 0x1
1829; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1830; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1831; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
1832; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1833; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
1834; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
1835; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
1836; GFX10-WGP-NEXT:    s_endpgm
1837;
1838; GFX10-CU-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
1839; GFX10-CU:       ; %bb.0: ; %entry
1840; GFX10-CU-NEXT:    s_clause 0x1
1841; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1842; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1843; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
1844; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1845; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
1846; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
1847; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
1848; GFX10-CU-NEXT:    s_endpgm
1849;
1850; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
1851; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1852; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
1853; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
1854; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
1855; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
1856; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1857; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1858; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
1859; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
1860; SKIP-CACHE-INV-NEXT:    s_endpgm
1861;
1862; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
1863; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1864; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1865; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1866; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1867; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1868; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
1869; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
1870; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1871;
1872; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
1873; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1874; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1875; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1876; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1877; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1878; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
1879; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
1880; GFX90A-TGSPLIT-NEXT:    s_endpgm
1881;
1882;
1883    i32 addrspace(1)* %out, i32 %in, i32 %old) {
1884entry:
1885  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
1886  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst monotonic
1887  ret void
1888}
1889
1890define amdgpu_kernel void @global_singlethread_acquire_acquire_cmpxchg(
1891; GFX6-LABEL: global_singlethread_acquire_acquire_cmpxchg:
1892; GFX6:       ; %bb.0: ; %entry
1893; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1894; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
1895; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1896; GFX6-NEXT:    s_mov_b32 s2, -1
1897; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1898; GFX6-NEXT:    v_mov_b32_e32 v0, s4
1899; GFX6-NEXT:    v_mov_b32_e32 v1, s5
1900; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
1901; GFX6-NEXT:    s_endpgm
1902;
1903; GFX7-LABEL: global_singlethread_acquire_acquire_cmpxchg:
1904; GFX7:       ; %bb.0: ; %entry
1905; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1906; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1907; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1908; GFX7-NEXT:    s_add_u32 s0, s0, 16
1909; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1910; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1911; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1912; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1913; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1914; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1915; GFX7-NEXT:    s_endpgm
1916;
1917; GFX10-WGP-LABEL: global_singlethread_acquire_acquire_cmpxchg:
1918; GFX10-WGP:       ; %bb.0: ; %entry
1919; GFX10-WGP-NEXT:    s_clause 0x1
1920; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1921; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1922; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
1923; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1924; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
1925; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
1926; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
1927; GFX10-WGP-NEXT:    s_endpgm
1928;
1929; GFX10-CU-LABEL: global_singlethread_acquire_acquire_cmpxchg:
1930; GFX10-CU:       ; %bb.0: ; %entry
1931; GFX10-CU-NEXT:    s_clause 0x1
1932; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1933; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1934; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
1935; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1936; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
1937; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
1938; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
1939; GFX10-CU-NEXT:    s_endpgm
1940;
1941; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_acquire_cmpxchg:
1942; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1943; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
1944; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
1945; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
1946; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
1947; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1948; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1949; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
1950; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
1951; SKIP-CACHE-INV-NEXT:    s_endpgm
1952;
1953; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_acquire_cmpxchg:
1954; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1955; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1956; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1957; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1958; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1959; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
1960; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
1961; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1962;
1963; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_acquire_cmpxchg:
1964; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1965; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1966; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1967; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1968; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1969; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
1970; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
1971; GFX90A-TGSPLIT-NEXT:    s_endpgm
1972;
1973;
1974    i32 addrspace(1)* %out, i32 %in, i32 %old) {
1975entry:
1976  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
1977  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire acquire
1978  ret void
1979}
1980
1981define amdgpu_kernel void @global_singlethread_release_acquire_cmpxchg(
1982; GFX6-LABEL: global_singlethread_release_acquire_cmpxchg:
1983; GFX6:       ; %bb.0: ; %entry
1984; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1985; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
1986; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1987; GFX6-NEXT:    s_mov_b32 s2, -1
1988; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1989; GFX6-NEXT:    v_mov_b32_e32 v0, s4
1990; GFX6-NEXT:    v_mov_b32_e32 v1, s5
1991; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
1992; GFX6-NEXT:    s_endpgm
1993;
1994; GFX7-LABEL: global_singlethread_release_acquire_cmpxchg:
1995; GFX7:       ; %bb.0: ; %entry
1996; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1997; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1998; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1999; GFX7-NEXT:    s_add_u32 s0, s0, 16
2000; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2001; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2002; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2003; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2004; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2005; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2006; GFX7-NEXT:    s_endpgm
2007;
2008; GFX10-WGP-LABEL: global_singlethread_release_acquire_cmpxchg:
2009; GFX10-WGP:       ; %bb.0: ; %entry
2010; GFX10-WGP-NEXT:    s_clause 0x1
2011; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2012; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2013; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2014; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2015; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
2016; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
2017; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
2018; GFX10-WGP-NEXT:    s_endpgm
2019;
2020; GFX10-CU-LABEL: global_singlethread_release_acquire_cmpxchg:
2021; GFX10-CU:       ; %bb.0: ; %entry
2022; GFX10-CU-NEXT:    s_clause 0x1
2023; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2024; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2025; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2026; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2027; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
2028; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
2029; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
2030; GFX10-CU-NEXT:    s_endpgm
2031;
2032; SKIP-CACHE-INV-LABEL: global_singlethread_release_acquire_cmpxchg:
2033; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2034; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
2035; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2036; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
2037; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
2038; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2039; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2040; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
2041; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
2042; SKIP-CACHE-INV-NEXT:    s_endpgm
2043;
2044; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_release_acquire_cmpxchg:
2045; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2046; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2047; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2048; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2049; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2050; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
2051; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
2052; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2053;
2054; GFX90A-TGSPLIT-LABEL: global_singlethread_release_acquire_cmpxchg:
2055; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2056; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2057; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2058; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2059; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2060; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
2061; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
2062; GFX90A-TGSPLIT-NEXT:    s_endpgm
2063;
2064;
2065    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2066entry:
2067  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2068  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") release acquire
2069  ret void
2070}
2071
2072define amdgpu_kernel void @global_singlethread_acq_rel_acquire_cmpxchg(
2073; GFX6-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
2074; GFX6:       ; %bb.0: ; %entry
2075; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2076; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
2077; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2078; GFX6-NEXT:    s_mov_b32 s2, -1
2079; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2080; GFX6-NEXT:    v_mov_b32_e32 v0, s4
2081; GFX6-NEXT:    v_mov_b32_e32 v1, s5
2082; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2083; GFX6-NEXT:    s_endpgm
2084;
2085; GFX7-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
2086; GFX7:       ; %bb.0: ; %entry
2087; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2088; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2089; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2090; GFX7-NEXT:    s_add_u32 s0, s0, 16
2091; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2092; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2093; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2094; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2095; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2096; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2097; GFX7-NEXT:    s_endpgm
2098;
2099; GFX10-WGP-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
2100; GFX10-WGP:       ; %bb.0: ; %entry
2101; GFX10-WGP-NEXT:    s_clause 0x1
2102; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2103; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2104; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2105; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2106; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
2107; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
2108; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
2109; GFX10-WGP-NEXT:    s_endpgm
2110;
2111; GFX10-CU-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
2112; GFX10-CU:       ; %bb.0: ; %entry
2113; GFX10-CU-NEXT:    s_clause 0x1
2114; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2115; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2116; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2117; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2118; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
2119; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
2120; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
2121; GFX10-CU-NEXT:    s_endpgm
2122;
2123; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
2124; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2125; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
2126; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2127; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
2128; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
2129; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2130; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2131; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
2132; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
2133; SKIP-CACHE-INV-NEXT:    s_endpgm
2134;
2135; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
2136; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2137; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2138; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2139; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2140; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2141; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
2142; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
2143; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2144;
2145; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
2146; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2147; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2148; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2149; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2150; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2151; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
2152; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
2153; GFX90A-TGSPLIT-NEXT:    s_endpgm
2154;
2155;
2156    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2157entry:
2158  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2159  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel acquire
2160  ret void
2161}
2162
2163define amdgpu_kernel void @global_singlethread_seq_cst_acquire_cmpxchg(
2164; GFX6-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
2165; GFX6:       ; %bb.0: ; %entry
2166; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2167; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
2168; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2169; GFX6-NEXT:    s_mov_b32 s2, -1
2170; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2171; GFX6-NEXT:    v_mov_b32_e32 v0, s4
2172; GFX6-NEXT:    v_mov_b32_e32 v1, s5
2173; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2174; GFX6-NEXT:    s_endpgm
2175;
2176; GFX7-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
2177; GFX7:       ; %bb.0: ; %entry
2178; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2179; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2180; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2181; GFX7-NEXT:    s_add_u32 s0, s0, 16
2182; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2183; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2184; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2185; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2186; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2187; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2188; GFX7-NEXT:    s_endpgm
2189;
2190; GFX10-WGP-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
2191; GFX10-WGP:       ; %bb.0: ; %entry
2192; GFX10-WGP-NEXT:    s_clause 0x1
2193; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2194; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2195; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2196; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2197; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
2198; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
2199; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
2200; GFX10-WGP-NEXT:    s_endpgm
2201;
2202; GFX10-CU-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
2203; GFX10-CU:       ; %bb.0: ; %entry
2204; GFX10-CU-NEXT:    s_clause 0x1
2205; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2206; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2207; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2208; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2209; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
2210; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
2211; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
2212; GFX10-CU-NEXT:    s_endpgm
2213;
2214; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
2215; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2216; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
2217; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2218; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
2219; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
2220; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2221; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2222; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
2223; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
2224; SKIP-CACHE-INV-NEXT:    s_endpgm
2225;
2226; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
2227; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2228; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2229; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2230; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2231; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2232; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
2233; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
2234; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2235;
2236; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
2237; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2238; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2239; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2240; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2241; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2242; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
2243; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
2244; GFX90A-TGSPLIT-NEXT:    s_endpgm
2245;
2246;
2247    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2248entry:
2249  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2250  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst acquire
2251  ret void
2252}
2253
2254define amdgpu_kernel void @global_singlethread_seq_cst_seq_cst_cmpxchg(
2255; GFX6-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
2256; GFX6:       ; %bb.0: ; %entry
2257; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2258; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
2259; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2260; GFX6-NEXT:    s_mov_b32 s2, -1
2261; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2262; GFX6-NEXT:    v_mov_b32_e32 v0, s4
2263; GFX6-NEXT:    v_mov_b32_e32 v1, s5
2264; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2265; GFX6-NEXT:    s_endpgm
2266;
2267; GFX7-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
2268; GFX7:       ; %bb.0: ; %entry
2269; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2270; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2271; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2272; GFX7-NEXT:    s_add_u32 s0, s0, 16
2273; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2274; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2275; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2276; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2277; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2278; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2279; GFX7-NEXT:    s_endpgm
2280;
2281; GFX10-WGP-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
2282; GFX10-WGP:       ; %bb.0: ; %entry
2283; GFX10-WGP-NEXT:    s_clause 0x1
2284; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2285; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2286; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2287; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2288; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
2289; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
2290; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
2291; GFX10-WGP-NEXT:    s_endpgm
2292;
2293; GFX10-CU-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
2294; GFX10-CU:       ; %bb.0: ; %entry
2295; GFX10-CU-NEXT:    s_clause 0x1
2296; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2297; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2298; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2299; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2300; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
2301; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
2302; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
2303; GFX10-CU-NEXT:    s_endpgm
2304;
2305; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
2306; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2307; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
2308; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2309; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
2310; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
2311; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2312; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2313; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
2314; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
2315; SKIP-CACHE-INV-NEXT:    s_endpgm
2316;
2317; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
2318; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2319; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2320; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2321; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2322; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2323; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
2324; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
2325; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2326;
2327; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
2328; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2329; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2330; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2331; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2332; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2333; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
2334; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
2335; GFX90A-TGSPLIT-NEXT:    s_endpgm
2336;
2337;
2338    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2339entry:
2340  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2341  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst seq_cst
2342  ret void
2343}
2344
2345define amdgpu_kernel void @global_singlethread_acquire_monotonic_ret_cmpxchg(
2346; GFX6-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
2347; GFX6:       ; %bb.0: ; %entry
2348; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2349; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
2350; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2351; GFX6-NEXT:    s_mov_b32 s2, -1
2352; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2353; GFX6-NEXT:    v_mov_b32_e32 v0, s4
2354; GFX6-NEXT:    v_mov_b32_e32 v1, s5
2355; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
2356; GFX6-NEXT:    s_waitcnt vmcnt(0)
2357; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
2358; GFX6-NEXT:    s_endpgm
2359;
2360; GFX7-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
2361; GFX7:       ; %bb.0: ; %entry
2362; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2363; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2364; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2365; GFX7-NEXT:    s_add_u32 s4, s0, 16
2366; GFX7-NEXT:    s_addc_u32 s5, s1, 0
2367; GFX7-NEXT:    v_mov_b32_e32 v0, s4
2368; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2369; GFX7-NEXT:    v_mov_b32_e32 v1, s5
2370; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2371; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2372; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2373; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2374; GFX7-NEXT:    s_waitcnt vmcnt(0)
2375; GFX7-NEXT:    flat_store_dword v[0:1], v2
2376; GFX7-NEXT:    s_endpgm
2377;
2378; GFX10-WGP-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
2379; GFX10-WGP:       ; %bb.0: ; %entry
2380; GFX10-WGP-NEXT:    s_clause 0x1
2381; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2382; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2383; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2384; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2385; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
2386; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
2387; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
2388; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
2389; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[0:1]
2390; GFX10-WGP-NEXT:    s_endpgm
2391;
2392; GFX10-CU-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
2393; GFX10-CU:       ; %bb.0: ; %entry
2394; GFX10-CU-NEXT:    s_clause 0x1
2395; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2396; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2397; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2398; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2399; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
2400; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
2401; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
2402; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
2403; GFX10-CU-NEXT:    global_store_dword v2, v0, s[0:1]
2404; GFX10-CU-NEXT:    s_endpgm
2405;
2406; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
2407; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2408; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
2409; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2410; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
2411; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
2412; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2413; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2414; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
2415; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
2416; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
2417; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
2418; SKIP-CACHE-INV-NEXT:    s_endpgm
2419;
2420; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
2421; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2422; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2423; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2424; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2425; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2426; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
2427; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
2428; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2429; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
2430; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2431;
2432; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
2433; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2434; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2435; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2436; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2437; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2438; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
2439; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
2440; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2441; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
2442; GFX90A-TGSPLIT-NEXT:    s_endpgm
2443;
2444;
2445    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2446entry:
2447  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2448  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire monotonic
2449  %val0 = extractvalue { i32, i1 } %val, 0
2450  store i32 %val0, i32 addrspace(1)* %out, align 4
2451  ret void
2452}
2453
2454define amdgpu_kernel void @global_singlethread_acq_rel_monotonic_ret_cmpxchg(
2455; GFX6-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
2456; GFX6:       ; %bb.0: ; %entry
2457; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2458; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
2459; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2460; GFX6-NEXT:    s_mov_b32 s2, -1
2461; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2462; GFX6-NEXT:    v_mov_b32_e32 v0, s4
2463; GFX6-NEXT:    v_mov_b32_e32 v1, s5
2464; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
2465; GFX6-NEXT:    s_waitcnt vmcnt(0)
2466; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
2467; GFX6-NEXT:    s_endpgm
2468;
2469; GFX7-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
2470; GFX7:       ; %bb.0: ; %entry
2471; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2472; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2473; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2474; GFX7-NEXT:    s_add_u32 s4, s0, 16
2475; GFX7-NEXT:    s_addc_u32 s5, s1, 0
2476; GFX7-NEXT:    v_mov_b32_e32 v0, s4
2477; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2478; GFX7-NEXT:    v_mov_b32_e32 v1, s5
2479; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2480; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2481; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2482; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2483; GFX7-NEXT:    s_waitcnt vmcnt(0)
2484; GFX7-NEXT:    flat_store_dword v[0:1], v2
2485; GFX7-NEXT:    s_endpgm
2486;
2487; GFX10-WGP-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
2488; GFX10-WGP:       ; %bb.0: ; %entry
2489; GFX10-WGP-NEXT:    s_clause 0x1
2490; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2491; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2492; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2493; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2494; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
2495; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
2496; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
2497; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
2498; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[0:1]
2499; GFX10-WGP-NEXT:    s_endpgm
2500;
2501; GFX10-CU-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
2502; GFX10-CU:       ; %bb.0: ; %entry
2503; GFX10-CU-NEXT:    s_clause 0x1
2504; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2505; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2506; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2507; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2508; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
2509; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
2510; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
2511; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
2512; GFX10-CU-NEXT:    global_store_dword v2, v0, s[0:1]
2513; GFX10-CU-NEXT:    s_endpgm
2514;
2515; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
2516; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2517; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
2518; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2519; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
2520; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
2521; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2522; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2523; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
2524; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
2525; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
2526; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
2527; SKIP-CACHE-INV-NEXT:    s_endpgm
2528;
2529; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
2530; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2531; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2532; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2533; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2534; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2535; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
2536; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
2537; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2538; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
2539; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2540;
2541; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
2542; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2543; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2544; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2545; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2546; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2547; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
2548; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
2549; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2550; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
2551; GFX90A-TGSPLIT-NEXT:    s_endpgm
2552;
2553;
2554    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2555entry:
2556  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2557  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel monotonic
2558  %val0 = extractvalue { i32, i1 } %val, 0
2559  store i32 %val0, i32 addrspace(1)* %out, align 4
2560  ret void
2561}
2562
2563define amdgpu_kernel void @global_singlethread_seq_cst_monotonic_ret_cmpxchg(
2564; GFX6-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
2565; GFX6:       ; %bb.0: ; %entry
2566; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2567; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
2568; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2569; GFX6-NEXT:    s_mov_b32 s2, -1
2570; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2571; GFX6-NEXT:    v_mov_b32_e32 v0, s4
2572; GFX6-NEXT:    v_mov_b32_e32 v1, s5
2573; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
2574; GFX6-NEXT:    s_waitcnt vmcnt(0)
2575; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
2576; GFX6-NEXT:    s_endpgm
2577;
2578; GFX7-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
2579; GFX7:       ; %bb.0: ; %entry
2580; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2581; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2582; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2583; GFX7-NEXT:    s_add_u32 s4, s0, 16
2584; GFX7-NEXT:    s_addc_u32 s5, s1, 0
2585; GFX7-NEXT:    v_mov_b32_e32 v0, s4
2586; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2587; GFX7-NEXT:    v_mov_b32_e32 v1, s5
2588; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2589; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2590; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2591; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2592; GFX7-NEXT:    s_waitcnt vmcnt(0)
2593; GFX7-NEXT:    flat_store_dword v[0:1], v2
2594; GFX7-NEXT:    s_endpgm
2595;
2596; GFX10-WGP-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
2597; GFX10-WGP:       ; %bb.0: ; %entry
2598; GFX10-WGP-NEXT:    s_clause 0x1
2599; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2600; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2601; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2602; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2603; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
2604; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
2605; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
2606; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
2607; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[0:1]
2608; GFX10-WGP-NEXT:    s_endpgm
2609;
2610; GFX10-CU-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
2611; GFX10-CU:       ; %bb.0: ; %entry
2612; GFX10-CU-NEXT:    s_clause 0x1
2613; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2614; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2615; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2616; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2617; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
2618; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
2619; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
2620; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
2621; GFX10-CU-NEXT:    global_store_dword v2, v0, s[0:1]
2622; GFX10-CU-NEXT:    s_endpgm
2623;
2624; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
2625; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2626; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
2627; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2628; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
2629; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
2630; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2631; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2632; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
2633; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
2634; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
2635; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
2636; SKIP-CACHE-INV-NEXT:    s_endpgm
2637;
2638; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
2639; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2640; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2641; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2642; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2643; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2644; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
2645; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
2646; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2647; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
2648; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2649;
2650; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
2651; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2652; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2653; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2654; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2655; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2656; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
2657; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
2658; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2659; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
2660; GFX90A-TGSPLIT-NEXT:    s_endpgm
2661;
2662;
2663    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2664entry:
2665  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2666  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst monotonic
2667  %val0 = extractvalue { i32, i1 } %val, 0
2668  store i32 %val0, i32 addrspace(1)* %out, align 4
2669  ret void
2670}
2671
2672define amdgpu_kernel void @global_singlethread_acquire_acquire_ret_cmpxchg(
2673; GFX6-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
2674; GFX6:       ; %bb.0: ; %entry
2675; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2676; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
2677; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2678; GFX6-NEXT:    s_mov_b32 s2, -1
2679; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2680; GFX6-NEXT:    v_mov_b32_e32 v0, s4
2681; GFX6-NEXT:    v_mov_b32_e32 v1, s5
2682; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
2683; GFX6-NEXT:    s_waitcnt vmcnt(0)
2684; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
2685; GFX6-NEXT:    s_endpgm
2686;
2687; GFX7-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
2688; GFX7:       ; %bb.0: ; %entry
2689; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2690; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2691; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2692; GFX7-NEXT:    s_add_u32 s4, s0, 16
2693; GFX7-NEXT:    s_addc_u32 s5, s1, 0
2694; GFX7-NEXT:    v_mov_b32_e32 v0, s4
2695; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2696; GFX7-NEXT:    v_mov_b32_e32 v1, s5
2697; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2698; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2699; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2700; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2701; GFX7-NEXT:    s_waitcnt vmcnt(0)
2702; GFX7-NEXT:    flat_store_dword v[0:1], v2
2703; GFX7-NEXT:    s_endpgm
2704;
2705; GFX10-WGP-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
2706; GFX10-WGP:       ; %bb.0: ; %entry
2707; GFX10-WGP-NEXT:    s_clause 0x1
2708; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2709; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2710; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2711; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2712; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
2713; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
2714; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
2715; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
2716; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[0:1]
2717; GFX10-WGP-NEXT:    s_endpgm
2718;
2719; GFX10-CU-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
2720; GFX10-CU:       ; %bb.0: ; %entry
2721; GFX10-CU-NEXT:    s_clause 0x1
2722; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2723; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2724; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2725; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2726; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
2727; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
2728; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
2729; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
2730; GFX10-CU-NEXT:    global_store_dword v2, v0, s[0:1]
2731; GFX10-CU-NEXT:    s_endpgm
2732;
2733; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
2734; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2735; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
2736; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2737; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
2738; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
2739; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2740; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2741; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
2742; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
2743; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
2744; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
2745; SKIP-CACHE-INV-NEXT:    s_endpgm
2746;
2747; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
2748; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2749; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2750; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2751; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2752; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2753; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
2754; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
2755; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2756; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
2757; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2758;
2759; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
2760; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2761; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2762; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2763; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2764; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2765; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
2766; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
2767; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2768; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
2769; GFX90A-TGSPLIT-NEXT:    s_endpgm
2770;
2771;
2772    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2773entry:
2774  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2775  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire acquire
2776  %val0 = extractvalue { i32, i1 } %val, 0
2777  store i32 %val0, i32 addrspace(1)* %out, align 4
2778  ret void
2779}
2780
2781define amdgpu_kernel void @global_singlethread_release_acquire_ret_cmpxchg(
2782; GFX6-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
2783; GFX6:       ; %bb.0: ; %entry
2784; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2785; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
2786; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2787; GFX6-NEXT:    s_mov_b32 s2, -1
2788; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2789; GFX6-NEXT:    v_mov_b32_e32 v0, s4
2790; GFX6-NEXT:    v_mov_b32_e32 v1, s5
2791; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
2792; GFX6-NEXT:    s_waitcnt vmcnt(0)
2793; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
2794; GFX6-NEXT:    s_endpgm
2795;
2796; GFX7-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
2797; GFX7:       ; %bb.0: ; %entry
2798; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2799; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2800; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2801; GFX7-NEXT:    s_add_u32 s4, s0, 16
2802; GFX7-NEXT:    s_addc_u32 s5, s1, 0
2803; GFX7-NEXT:    v_mov_b32_e32 v0, s4
2804; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2805; GFX7-NEXT:    v_mov_b32_e32 v1, s5
2806; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2807; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2808; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2809; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2810; GFX7-NEXT:    s_waitcnt vmcnt(0)
2811; GFX7-NEXT:    flat_store_dword v[0:1], v2
2812; GFX7-NEXT:    s_endpgm
2813;
2814; GFX10-WGP-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
2815; GFX10-WGP:       ; %bb.0: ; %entry
2816; GFX10-WGP-NEXT:    s_clause 0x1
2817; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2818; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2819; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2820; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2821; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
2822; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
2823; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
2824; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
2825; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[0:1]
2826; GFX10-WGP-NEXT:    s_endpgm
2827;
2828; GFX10-CU-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
2829; GFX10-CU:       ; %bb.0: ; %entry
2830; GFX10-CU-NEXT:    s_clause 0x1
2831; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2832; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2833; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2834; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2835; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
2836; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
2837; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
2838; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
2839; GFX10-CU-NEXT:    global_store_dword v2, v0, s[0:1]
2840; GFX10-CU-NEXT:    s_endpgm
2841;
2842; SKIP-CACHE-INV-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
2843; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2844; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
2845; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2846; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
2847; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
2848; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2849; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2850; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
2851; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
2852; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
2853; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
2854; SKIP-CACHE-INV-NEXT:    s_endpgm
2855;
2856; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
2857; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2858; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2859; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2860; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2861; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2862; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
2863; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
2864; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2865; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
2866; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2867;
2868; GFX90A-TGSPLIT-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
2869; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2870; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2871; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2872; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2873; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2874; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
2875; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
2876; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2877; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
2878; GFX90A-TGSPLIT-NEXT:    s_endpgm
2879;
2880;
2881    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2882entry:
2883  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2884  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") release acquire
2885  %val0 = extractvalue { i32, i1 } %val, 0
2886  store i32 %val0, i32 addrspace(1)* %out, align 4
2887  ret void
2888}
2889
2890define amdgpu_kernel void @global_singlethread_acq_rel_acquire_ret_cmpxchg(
2891; GFX6-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
2892; GFX6:       ; %bb.0: ; %entry
2893; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2894; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
2895; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2896; GFX6-NEXT:    s_mov_b32 s2, -1
2897; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2898; GFX6-NEXT:    v_mov_b32_e32 v0, s4
2899; GFX6-NEXT:    v_mov_b32_e32 v1, s5
2900; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
2901; GFX6-NEXT:    s_waitcnt vmcnt(0)
2902; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
2903; GFX6-NEXT:    s_endpgm
2904;
2905; GFX7-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
2906; GFX7:       ; %bb.0: ; %entry
2907; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2908; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2909; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2910; GFX7-NEXT:    s_add_u32 s4, s0, 16
2911; GFX7-NEXT:    s_addc_u32 s5, s1, 0
2912; GFX7-NEXT:    v_mov_b32_e32 v0, s4
2913; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2914; GFX7-NEXT:    v_mov_b32_e32 v1, s5
2915; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2916; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2917; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2918; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2919; GFX7-NEXT:    s_waitcnt vmcnt(0)
2920; GFX7-NEXT:    flat_store_dword v[0:1], v2
2921; GFX7-NEXT:    s_endpgm
2922;
2923; GFX10-WGP-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
2924; GFX10-WGP:       ; %bb.0: ; %entry
2925; GFX10-WGP-NEXT:    s_clause 0x1
2926; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2927; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2928; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2929; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2930; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
2931; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
2932; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
2933; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
2934; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[0:1]
2935; GFX10-WGP-NEXT:    s_endpgm
2936;
2937; GFX10-CU-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
2938; GFX10-CU:       ; %bb.0: ; %entry
2939; GFX10-CU-NEXT:    s_clause 0x1
2940; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2941; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2942; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2943; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2944; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
2945; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
2946; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
2947; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
2948; GFX10-CU-NEXT:    global_store_dword v2, v0, s[0:1]
2949; GFX10-CU-NEXT:    s_endpgm
2950;
2951; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
2952; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2953; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
2954; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2955; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
2956; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
2957; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2958; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2959; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
2960; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
2961; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
2962; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
2963; SKIP-CACHE-INV-NEXT:    s_endpgm
2964;
2965; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
2966; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2967; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2968; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2969; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2970; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2971; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
2972; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
2973; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2974; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
2975; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2976;
2977; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
2978; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2979; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2980; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2981; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2982; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2983; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
2984; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
2985; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2986; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
2987; GFX90A-TGSPLIT-NEXT:    s_endpgm
2988;
2989;
2990    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2991entry:
2992  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2993  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel acquire
2994  %val0 = extractvalue { i32, i1 } %val, 0
2995  store i32 %val0, i32 addrspace(1)* %out, align 4
2996  ret void
2997}
2998
2999define amdgpu_kernel void @global_singlethread_seq_cst_acquire_ret_cmpxchg(
3000; GFX6-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
3001; GFX6:       ; %bb.0: ; %entry
3002; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3003; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
3004; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3005; GFX6-NEXT:    s_mov_b32 s2, -1
3006; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3007; GFX6-NEXT:    v_mov_b32_e32 v0, s4
3008; GFX6-NEXT:    v_mov_b32_e32 v1, s5
3009; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3010; GFX6-NEXT:    s_waitcnt vmcnt(0)
3011; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3012; GFX6-NEXT:    s_endpgm
3013;
3014; GFX7-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
3015; GFX7:       ; %bb.0: ; %entry
3016; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3017; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3018; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3019; GFX7-NEXT:    s_add_u32 s4, s0, 16
3020; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3021; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3022; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3023; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3024; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3025; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3026; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3027; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3028; GFX7-NEXT:    s_waitcnt vmcnt(0)
3029; GFX7-NEXT:    flat_store_dword v[0:1], v2
3030; GFX7-NEXT:    s_endpgm
3031;
3032; GFX10-WGP-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
3033; GFX10-WGP:       ; %bb.0: ; %entry
3034; GFX10-WGP-NEXT:    s_clause 0x1
3035; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3036; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3037; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3038; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3039; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
3040; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
3041; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
3042; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3043; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[0:1]
3044; GFX10-WGP-NEXT:    s_endpgm
3045;
3046; GFX10-CU-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
3047; GFX10-CU:       ; %bb.0: ; %entry
3048; GFX10-CU-NEXT:    s_clause 0x1
3049; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3050; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3051; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3052; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3053; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
3054; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
3055; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
3056; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3057; GFX10-CU-NEXT:    global_store_dword v2, v0, s[0:1]
3058; GFX10-CU-NEXT:    s_endpgm
3059;
3060; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
3061; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3062; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
3063; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
3064; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
3065; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
3066; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3067; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3068; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
3069; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
3070; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3071; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
3072; SKIP-CACHE-INV-NEXT:    s_endpgm
3073;
3074; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
3075; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3076; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3077; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3078; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3079; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3080; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
3081; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
3082; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3083; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
3084; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3085;
3086; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
3087; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3088; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3089; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3090; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3091; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3092; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
3093; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
3094; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3095; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
3096; GFX90A-TGSPLIT-NEXT:    s_endpgm
3097;
3098;
3099    i32 addrspace(1)* %out, i32 %in, i32 %old) {
3100entry:
3101  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3102  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst acquire
3103  %val0 = extractvalue { i32, i1 } %val, 0
3104  store i32 %val0, i32 addrspace(1)* %out, align 4
3105  ret void
3106}
3107
3108define amdgpu_kernel void @global_singlethread_seq_cst_seq_cst_ret_cmpxchg(
3109; GFX6-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
3110; GFX6:       ; %bb.0: ; %entry
3111; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3112; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
3113; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3114; GFX6-NEXT:    s_mov_b32 s2, -1
3115; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3116; GFX6-NEXT:    v_mov_b32_e32 v0, s4
3117; GFX6-NEXT:    v_mov_b32_e32 v1, s5
3118; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3119; GFX6-NEXT:    s_waitcnt vmcnt(0)
3120; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3121; GFX6-NEXT:    s_endpgm
3122;
3123; GFX7-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
3124; GFX7:       ; %bb.0: ; %entry
3125; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3126; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3127; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3128; GFX7-NEXT:    s_add_u32 s4, s0, 16
3129; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3130; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3131; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3132; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3133; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3134; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3135; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3136; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3137; GFX7-NEXT:    s_waitcnt vmcnt(0)
3138; GFX7-NEXT:    flat_store_dword v[0:1], v2
3139; GFX7-NEXT:    s_endpgm
3140;
3141; GFX10-WGP-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
3142; GFX10-WGP:       ; %bb.0: ; %entry
3143; GFX10-WGP-NEXT:    s_clause 0x1
3144; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3145; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3146; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3147; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3148; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
3149; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
3150; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
3151; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3152; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[0:1]
3153; GFX10-WGP-NEXT:    s_endpgm
3154;
3155; GFX10-CU-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
3156; GFX10-CU:       ; %bb.0: ; %entry
3157; GFX10-CU-NEXT:    s_clause 0x1
3158; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3159; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3160; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3161; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3162; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
3163; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
3164; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
3165; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3166; GFX10-CU-NEXT:    global_store_dword v2, v0, s[0:1]
3167; GFX10-CU-NEXT:    s_endpgm
3168;
3169; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
3170; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3171; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
3172; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
3173; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
3174; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
3175; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3176; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3177; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
3178; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
3179; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3180; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
3181; SKIP-CACHE-INV-NEXT:    s_endpgm
3182;
3183; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
3184; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3185; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3186; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3187; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3188; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3189; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
3190; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
3191; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3192; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
3193; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3194;
3195; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
3196; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3197; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3198; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3199; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3200; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3201; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
3202; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
3203; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3204; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
3205; GFX90A-TGSPLIT-NEXT:    s_endpgm
3206;
3207;
3208    i32 addrspace(1)* %out, i32 %in, i32 %old) {
3209entry:
3210  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3211  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst seq_cst
3212  %val0 = extractvalue { i32, i1 } %val, 0
3213  store i32 %val0, i32 addrspace(1)* %out, align 4
3214  ret void
3215}
3216
3217define amdgpu_kernel void @global_singlethread_one_as_unordered_load(
3218; GFX6-LABEL: global_singlethread_one_as_unordered_load:
3219; GFX6:       ; %bb.0: ; %entry
3220; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x0
3221; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3222; GFX6-NEXT:    s_mov_b32 s2, -1
3223; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3224; GFX6-NEXT:    s_mov_b32 s0, s4
3225; GFX6-NEXT:    s_mov_b32 s1, s5
3226; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
3227; GFX6-NEXT:    s_mov_b32 s4, s6
3228; GFX6-NEXT:    s_mov_b32 s5, s7
3229; GFX6-NEXT:    s_mov_b32 s6, s2
3230; GFX6-NEXT:    s_mov_b32 s7, s3
3231; GFX6-NEXT:    s_waitcnt vmcnt(0)
3232; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
3233; GFX6-NEXT:    s_endpgm
3234;
3235; GFX7-LABEL: global_singlethread_one_as_unordered_load:
3236; GFX7:       ; %bb.0: ; %entry
3237; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3238; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3239; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3240; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3241; GFX7-NEXT:    flat_load_dword v0, v[0:1]
3242; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3243; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3244; GFX7-NEXT:    s_waitcnt vmcnt(0)
3245; GFX7-NEXT:    flat_store_dword v[2:3], v0
3246; GFX7-NEXT:    s_endpgm
3247;
3248; GFX10-WGP-LABEL: global_singlethread_one_as_unordered_load:
3249; GFX10-WGP:       ; %bb.0: ; %entry
3250; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3251; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
3252; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3253; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
3254; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3255; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
3256; GFX10-WGP-NEXT:    s_endpgm
3257;
3258; GFX10-CU-LABEL: global_singlethread_one_as_unordered_load:
3259; GFX10-CU:       ; %bb.0: ; %entry
3260; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3261; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
3262; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3263; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
3264; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3265; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
3266; GFX10-CU-NEXT:    s_endpgm
3267;
3268; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_unordered_load:
3269; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3270; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
3271; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3272; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3273; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3274; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
3275; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
3276; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
3277; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
3278; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
3279; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
3280; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
3281; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3282; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
3283; SKIP-CACHE-INV-NEXT:    s_endpgm
3284;
3285; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_unordered_load:
3286; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3287; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3288; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
3289; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3290; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
3291; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3292; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
3293; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3294;
3295; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_unordered_load:
3296; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3297; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3298; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
3299; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3300; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
3301; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3302; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
3303; GFX90A-TGSPLIT-NEXT:    s_endpgm
3304;
3305;
3306    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
3307entry:
3308  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread-one-as") unordered, align 4
3309  store i32 %val, i32 addrspace(1)* %out
3310  ret void
3311}
3312
3313define amdgpu_kernel void @global_singlethread_one_as_monotonic_load(
3314; GFX6-LABEL: global_singlethread_one_as_monotonic_load:
3315; GFX6:       ; %bb.0: ; %entry
3316; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x0
3317; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3318; GFX6-NEXT:    s_mov_b32 s2, -1
3319; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3320; GFX6-NEXT:    s_mov_b32 s0, s4
3321; GFX6-NEXT:    s_mov_b32 s1, s5
3322; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
3323; GFX6-NEXT:    s_mov_b32 s4, s6
3324; GFX6-NEXT:    s_mov_b32 s5, s7
3325; GFX6-NEXT:    s_mov_b32 s6, s2
3326; GFX6-NEXT:    s_mov_b32 s7, s3
3327; GFX6-NEXT:    s_waitcnt vmcnt(0)
3328; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
3329; GFX6-NEXT:    s_endpgm
3330;
3331; GFX7-LABEL: global_singlethread_one_as_monotonic_load:
3332; GFX7:       ; %bb.0: ; %entry
3333; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3334; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3335; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3336; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3337; GFX7-NEXT:    flat_load_dword v0, v[0:1]
3338; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3339; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3340; GFX7-NEXT:    s_waitcnt vmcnt(0)
3341; GFX7-NEXT:    flat_store_dword v[2:3], v0
3342; GFX7-NEXT:    s_endpgm
3343;
3344; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_load:
3345; GFX10-WGP:       ; %bb.0: ; %entry
3346; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3347; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
3348; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3349; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
3350; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3351; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
3352; GFX10-WGP-NEXT:    s_endpgm
3353;
3354; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_load:
3355; GFX10-CU:       ; %bb.0: ; %entry
3356; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3357; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
3358; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3359; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
3360; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3361; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
3362; GFX10-CU-NEXT:    s_endpgm
3363;
3364; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_load:
3365; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3366; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
3367; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3368; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3369; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3370; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
3371; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
3372; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
3373; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
3374; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
3375; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
3376; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
3377; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3378; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
3379; SKIP-CACHE-INV-NEXT:    s_endpgm
3380;
3381; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_load:
3382; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3383; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3384; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
3385; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3386; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
3387; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3388; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
3389; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3390;
3391; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_load:
3392; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3393; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3394; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
3395; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3396; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
3397; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3398; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
3399; GFX90A-TGSPLIT-NEXT:    s_endpgm
3400;
3401;
3402    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
3403entry:
3404  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread-one-as") monotonic, align 4
3405  store i32 %val, i32 addrspace(1)* %out
3406  ret void
3407}
3408
3409define amdgpu_kernel void @global_singlethread_one_as_acquire_load(
3410; GFX6-LABEL: global_singlethread_one_as_acquire_load:
3411; GFX6:       ; %bb.0: ; %entry
3412; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x0
3413; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3414; GFX6-NEXT:    s_mov_b32 s2, -1
3415; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3416; GFX6-NEXT:    s_mov_b32 s0, s4
3417; GFX6-NEXT:    s_mov_b32 s1, s5
3418; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
3419; GFX6-NEXT:    s_mov_b32 s4, s6
3420; GFX6-NEXT:    s_mov_b32 s5, s7
3421; GFX6-NEXT:    s_mov_b32 s6, s2
3422; GFX6-NEXT:    s_mov_b32 s7, s3
3423; GFX6-NEXT:    s_waitcnt vmcnt(0)
3424; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
3425; GFX6-NEXT:    s_endpgm
3426;
3427; GFX7-LABEL: global_singlethread_one_as_acquire_load:
3428; GFX7:       ; %bb.0: ; %entry
3429; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3430; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3431; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3432; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3433; GFX7-NEXT:    flat_load_dword v0, v[0:1]
3434; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3435; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3436; GFX7-NEXT:    s_waitcnt vmcnt(0)
3437; GFX7-NEXT:    flat_store_dword v[2:3], v0
3438; GFX7-NEXT:    s_endpgm
3439;
3440; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_load:
3441; GFX10-WGP:       ; %bb.0: ; %entry
3442; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3443; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
3444; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3445; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
3446; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3447; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
3448; GFX10-WGP-NEXT:    s_endpgm
3449;
3450; GFX10-CU-LABEL: global_singlethread_one_as_acquire_load:
3451; GFX10-CU:       ; %bb.0: ; %entry
3452; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3453; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
3454; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3455; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
3456; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3457; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
3458; GFX10-CU-NEXT:    s_endpgm
3459;
3460; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_load:
3461; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3462; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
3463; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3464; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3465; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3466; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
3467; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
3468; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
3469; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
3470; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
3471; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
3472; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
3473; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3474; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
3475; SKIP-CACHE-INV-NEXT:    s_endpgm
3476;
3477; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_load:
3478; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3479; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3480; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
3481; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3482; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
3483; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3484; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
3485; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3486;
3487; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_load:
3488; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3489; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3490; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
3491; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3492; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
3493; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3494; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
3495; GFX90A-TGSPLIT-NEXT:    s_endpgm
3496;
3497;
3498    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
3499entry:
3500  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread-one-as") acquire, align 4
3501  store i32 %val, i32 addrspace(1)* %out
3502  ret void
3503}
3504
3505define amdgpu_kernel void @global_singlethread_one_as_seq_cst_load(
3506; GFX6-LABEL: global_singlethread_one_as_seq_cst_load:
3507; GFX6:       ; %bb.0: ; %entry
3508; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x0
3509; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3510; GFX6-NEXT:    s_mov_b32 s2, -1
3511; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3512; GFX6-NEXT:    s_mov_b32 s0, s4
3513; GFX6-NEXT:    s_mov_b32 s1, s5
3514; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
3515; GFX6-NEXT:    s_mov_b32 s4, s6
3516; GFX6-NEXT:    s_mov_b32 s5, s7
3517; GFX6-NEXT:    s_mov_b32 s6, s2
3518; GFX6-NEXT:    s_mov_b32 s7, s3
3519; GFX6-NEXT:    s_waitcnt vmcnt(0)
3520; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
3521; GFX6-NEXT:    s_endpgm
3522;
3523; GFX7-LABEL: global_singlethread_one_as_seq_cst_load:
3524; GFX7:       ; %bb.0: ; %entry
3525; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3526; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3527; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3528; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3529; GFX7-NEXT:    flat_load_dword v0, v[0:1]
3530; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3531; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3532; GFX7-NEXT:    s_waitcnt vmcnt(0)
3533; GFX7-NEXT:    flat_store_dword v[2:3], v0
3534; GFX7-NEXT:    s_endpgm
3535;
3536; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_load:
3537; GFX10-WGP:       ; %bb.0: ; %entry
3538; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3539; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
3540; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3541; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
3542; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3543; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
3544; GFX10-WGP-NEXT:    s_endpgm
3545;
3546; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_load:
3547; GFX10-CU:       ; %bb.0: ; %entry
3548; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3549; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
3550; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3551; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
3552; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3553; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
3554; GFX10-CU-NEXT:    s_endpgm
3555;
3556; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_load:
3557; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3558; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
3559; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3560; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3561; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3562; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
3563; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
3564; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
3565; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
3566; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
3567; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
3568; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
3569; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3570; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
3571; SKIP-CACHE-INV-NEXT:    s_endpgm
3572;
3573; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_load:
3574; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3575; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3576; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
3577; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3578; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
3579; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3580; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
3581; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3582;
3583; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_load:
3584; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3585; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3586; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
3587; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3588; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
3589; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3590; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
3591; GFX90A-TGSPLIT-NEXT:    s_endpgm
3592;
3593;
3594    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
3595entry:
3596  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread-one-as") seq_cst, align 4
3597  store i32 %val, i32 addrspace(1)* %out
3598  ret void
3599}
3600
3601define amdgpu_kernel void @global_singlethread_one_as_unordered_store(
3602; GFX6-LABEL: global_singlethread_one_as_unordered_store:
3603; GFX6:       ; %bb.0: ; %entry
3604; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
3605; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
3606; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3607; GFX6-NEXT:    s_mov_b32 s2, -1
3608; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3609; GFX6-NEXT:    v_mov_b32_e32 v0, s6
3610; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3611; GFX6-NEXT:    s_endpgm
3612;
3613; GFX7-LABEL: global_singlethread_one_as_unordered_store:
3614; GFX7:       ; %bb.0: ; %entry
3615; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
3616; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
3617; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3618; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3619; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3620; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3621; GFX7-NEXT:    flat_store_dword v[0:1], v2
3622; GFX7-NEXT:    s_endpgm
3623;
3624; GFX10-WGP-LABEL: global_singlethread_one_as_unordered_store:
3625; GFX10-WGP:       ; %bb.0: ; %entry
3626; GFX10-WGP-NEXT:    s_clause 0x1
3627; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
3628; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3629; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
3630; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3631; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
3632; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
3633; GFX10-WGP-NEXT:    s_endpgm
3634;
3635; GFX10-CU-LABEL: global_singlethread_one_as_unordered_store:
3636; GFX10-CU:       ; %bb.0: ; %entry
3637; GFX10-CU-NEXT:    s_clause 0x1
3638; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
3639; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3640; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
3641; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3642; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
3643; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
3644; GFX10-CU-NEXT:    s_endpgm
3645;
3646; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_unordered_store:
3647; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3648; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
3649; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
3650; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3651; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3652; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3653; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3654; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3655; SKIP-CACHE-INV-NEXT:    s_endpgm
3656;
3657; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_unordered_store:
3658; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3659; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
3660; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3661; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
3662; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3663; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3664; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
3665; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3666;
3667; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_unordered_store:
3668; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3669; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
3670; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3671; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
3672; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3673; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3674; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
3675; GFX90A-TGSPLIT-NEXT:    s_endpgm
3676;
3677;
3678    i32 %in, i32 addrspace(1)* %out) {
3679entry:
3680  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread-one-as") unordered, align 4
3681  ret void
3682}
3683
3684define amdgpu_kernel void @global_singlethread_one_as_monotonic_store(
3685; GFX6-LABEL: global_singlethread_one_as_monotonic_store:
3686; GFX6:       ; %bb.0: ; %entry
3687; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
3688; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
3689; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3690; GFX6-NEXT:    s_mov_b32 s2, -1
3691; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3692; GFX6-NEXT:    v_mov_b32_e32 v0, s6
3693; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3694; GFX6-NEXT:    s_endpgm
3695;
3696; GFX7-LABEL: global_singlethread_one_as_monotonic_store:
3697; GFX7:       ; %bb.0: ; %entry
3698; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
3699; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
3700; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3701; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3702; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3703; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3704; GFX7-NEXT:    flat_store_dword v[0:1], v2
3705; GFX7-NEXT:    s_endpgm
3706;
3707; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_store:
3708; GFX10-WGP:       ; %bb.0: ; %entry
3709; GFX10-WGP-NEXT:    s_clause 0x1
3710; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
3711; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3712; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
3713; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3714; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
3715; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
3716; GFX10-WGP-NEXT:    s_endpgm
3717;
3718; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_store:
3719; GFX10-CU:       ; %bb.0: ; %entry
3720; GFX10-CU-NEXT:    s_clause 0x1
3721; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
3722; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3723; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
3724; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3725; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
3726; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
3727; GFX10-CU-NEXT:    s_endpgm
3728;
3729; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_store:
3730; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3731; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
3732; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
3733; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3734; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3735; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3736; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3737; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3738; SKIP-CACHE-INV-NEXT:    s_endpgm
3739;
3740; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_store:
3741; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3742; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
3743; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3744; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
3745; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3746; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3747; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
3748; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3749;
3750; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_store:
3751; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3752; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
3753; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3754; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
3755; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3756; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3757; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
3758; GFX90A-TGSPLIT-NEXT:    s_endpgm
3759;
3760;
3761    i32 %in, i32 addrspace(1)* %out) {
3762entry:
3763  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread-one-as") monotonic, align 4
3764  ret void
3765}
3766
3767define amdgpu_kernel void @global_singlethread_one_as_release_store(
3768; GFX6-LABEL: global_singlethread_one_as_release_store:
3769; GFX6:       ; %bb.0: ; %entry
3770; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
3771; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
3772; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3773; GFX6-NEXT:    s_mov_b32 s2, -1
3774; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3775; GFX6-NEXT:    v_mov_b32_e32 v0, s6
3776; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3777; GFX6-NEXT:    s_endpgm
3778;
3779; GFX7-LABEL: global_singlethread_one_as_release_store:
3780; GFX7:       ; %bb.0: ; %entry
3781; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
3782; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
3783; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3784; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3785; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3786; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3787; GFX7-NEXT:    flat_store_dword v[0:1], v2
3788; GFX7-NEXT:    s_endpgm
3789;
3790; GFX10-WGP-LABEL: global_singlethread_one_as_release_store:
3791; GFX10-WGP:       ; %bb.0: ; %entry
3792; GFX10-WGP-NEXT:    s_clause 0x1
3793; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
3794; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3795; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
3796; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3797; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
3798; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
3799; GFX10-WGP-NEXT:    s_endpgm
3800;
3801; GFX10-CU-LABEL: global_singlethread_one_as_release_store:
3802; GFX10-CU:       ; %bb.0: ; %entry
3803; GFX10-CU-NEXT:    s_clause 0x1
3804; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
3805; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3806; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
3807; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3808; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
3809; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
3810; GFX10-CU-NEXT:    s_endpgm
3811;
3812; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_store:
3813; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3814; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
3815; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
3816; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3817; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3818; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3819; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3820; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3821; SKIP-CACHE-INV-NEXT:    s_endpgm
3822;
3823; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_store:
3824; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3825; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
3826; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3827; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
3828; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3829; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3830; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
3831; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3832;
3833; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_release_store:
3834; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3835; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
3836; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3837; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
3838; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3839; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3840; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
3841; GFX90A-TGSPLIT-NEXT:    s_endpgm
3842;
3843;
3844    i32 %in, i32 addrspace(1)* %out) {
3845entry:
3846  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread-one-as") release, align 4
3847  ret void
3848}
3849
3850define amdgpu_kernel void @global_singlethread_one_as_seq_cst_store(
3851; GFX6-LABEL: global_singlethread_one_as_seq_cst_store:
3852; GFX6:       ; %bb.0: ; %entry
3853; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
3854; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
3855; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3856; GFX6-NEXT:    s_mov_b32 s2, -1
3857; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3858; GFX6-NEXT:    v_mov_b32_e32 v0, s6
3859; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3860; GFX6-NEXT:    s_endpgm
3861;
3862; GFX7-LABEL: global_singlethread_one_as_seq_cst_store:
3863; GFX7:       ; %bb.0: ; %entry
3864; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
3865; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
3866; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3867; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3868; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3869; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3870; GFX7-NEXT:    flat_store_dword v[0:1], v2
3871; GFX7-NEXT:    s_endpgm
3872;
3873; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_store:
3874; GFX10-WGP:       ; %bb.0: ; %entry
3875; GFX10-WGP-NEXT:    s_clause 0x1
3876; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
3877; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3878; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
3879; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3880; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
3881; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
3882; GFX10-WGP-NEXT:    s_endpgm
3883;
3884; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_store:
3885; GFX10-CU:       ; %bb.0: ; %entry
3886; GFX10-CU-NEXT:    s_clause 0x1
3887; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
3888; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3889; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
3890; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3891; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
3892; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
3893; GFX10-CU-NEXT:    s_endpgm
3894;
3895; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_store:
3896; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3897; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
3898; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
3899; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3900; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3901; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3902; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3903; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3904; SKIP-CACHE-INV-NEXT:    s_endpgm
3905;
3906; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_store:
3907; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3908; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
3909; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3910; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
3911; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3912; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3913; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
3914; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3915;
3916; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_store:
3917; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3918; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
3919; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3920; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
3921; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3922; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3923; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
3924; GFX90A-TGSPLIT-NEXT:    s_endpgm
3925;
3926;
3927    i32 %in, i32 addrspace(1)* %out) {
3928entry:
3929  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread-one-as") seq_cst, align 4
3930  ret void
3931}
3932
3933define amdgpu_kernel void @global_singlethread_one_as_monotonic_atomicrmw(
3934; GFX6-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
3935; GFX6:       ; %bb.0: ; %entry
3936; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3937; GFX6-NEXT:    s_load_dword s4, s[4:5], 0x2
3938; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3939; GFX6-NEXT:    s_mov_b32 s2, -1
3940; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3941; GFX6-NEXT:    v_mov_b32_e32 v0, s4
3942; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
3943; GFX6-NEXT:    s_endpgm
3944;
3945; GFX7-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
3946; GFX7:       ; %bb.0: ; %entry
3947; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3948; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
3949; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3950; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3951; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3952; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3953; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
3954; GFX7-NEXT:    s_endpgm
3955;
3956; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
3957; GFX10-WGP:       ; %bb.0: ; %entry
3958; GFX10-WGP-NEXT:    s_clause 0x1
3959; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
3960; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3961; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
3962; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3963; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
3964; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
3965; GFX10-WGP-NEXT:    s_endpgm
3966;
3967; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
3968; GFX10-CU:       ; %bb.0: ; %entry
3969; GFX10-CU-NEXT:    s_clause 0x1
3970; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
3971; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3972; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
3973; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3974; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
3975; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
3976; GFX10-CU-NEXT:    s_endpgm
3977;
3978; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
3979; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3980; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
3981; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
3982; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
3983; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
3984; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3985; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3986; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
3987; SKIP-CACHE-INV-NEXT:    s_endpgm
3988;
3989; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
3990; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3991; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3992; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
3993; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
3994; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3995; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
3996; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
3997; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3998;
3999; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
4000; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4001; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4002; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4003; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4004; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4005; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4006; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
4007; GFX90A-TGSPLIT-NEXT:    s_endpgm
4008;
4009;
4010    i32 addrspace(1)* %out, i32 %in) {
4011entry:
4012  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") monotonic
4013  ret void
4014}
4015
4016define amdgpu_kernel void @global_singlethread_one_as_acquire_atomicrmw(
4017; GFX6-LABEL: global_singlethread_one_as_acquire_atomicrmw:
4018; GFX6:       ; %bb.0: ; %entry
4019; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4020; GFX6-NEXT:    s_load_dword s4, s[4:5], 0x2
4021; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4022; GFX6-NEXT:    s_mov_b32 s2, -1
4023; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4024; GFX6-NEXT:    v_mov_b32_e32 v0, s4
4025; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
4026; GFX6-NEXT:    s_endpgm
4027;
4028; GFX7-LABEL: global_singlethread_one_as_acquire_atomicrmw:
4029; GFX7:       ; %bb.0: ; %entry
4030; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4031; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
4032; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4033; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4034; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4035; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4036; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
4037; GFX7-NEXT:    s_endpgm
4038;
4039; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_atomicrmw:
4040; GFX10-WGP:       ; %bb.0: ; %entry
4041; GFX10-WGP-NEXT:    s_clause 0x1
4042; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
4043; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4044; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
4045; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4046; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
4047; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
4048; GFX10-WGP-NEXT:    s_endpgm
4049;
4050; GFX10-CU-LABEL: global_singlethread_one_as_acquire_atomicrmw:
4051; GFX10-CU:       ; %bb.0: ; %entry
4052; GFX10-CU-NEXT:    s_clause 0x1
4053; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
4054; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4055; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
4056; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4057; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
4058; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
4059; GFX10-CU-NEXT:    s_endpgm
4060;
4061; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_atomicrmw:
4062; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4063; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
4064; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
4065; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
4066; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
4067; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4068; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4069; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
4070; SKIP-CACHE-INV-NEXT:    s_endpgm
4071;
4072; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_atomicrmw:
4073; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4074; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4075; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4076; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4077; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4078; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4079; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
4080; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4081;
4082; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_atomicrmw:
4083; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4084; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4085; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4086; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4087; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4088; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4089; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
4090; GFX90A-TGSPLIT-NEXT:    s_endpgm
4091;
4092;
4093    i32 addrspace(1)* %out, i32 %in) {
4094entry:
4095  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") acquire
4096  ret void
4097}
4098
4099define amdgpu_kernel void @global_singlethread_one_as_release_atomicrmw(
4100; GFX6-LABEL: global_singlethread_one_as_release_atomicrmw:
4101; GFX6:       ; %bb.0: ; %entry
4102; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4103; GFX6-NEXT:    s_load_dword s4, s[4:5], 0x2
4104; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4105; GFX6-NEXT:    s_mov_b32 s2, -1
4106; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4107; GFX6-NEXT:    v_mov_b32_e32 v0, s4
4108; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
4109; GFX6-NEXT:    s_endpgm
4110;
4111; GFX7-LABEL: global_singlethread_one_as_release_atomicrmw:
4112; GFX7:       ; %bb.0: ; %entry
4113; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4114; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
4115; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4116; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4117; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4118; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4119; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
4120; GFX7-NEXT:    s_endpgm
4121;
4122; GFX10-WGP-LABEL: global_singlethread_one_as_release_atomicrmw:
4123; GFX10-WGP:       ; %bb.0: ; %entry
4124; GFX10-WGP-NEXT:    s_clause 0x1
4125; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
4126; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4127; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
4128; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4129; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
4130; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
4131; GFX10-WGP-NEXT:    s_endpgm
4132;
4133; GFX10-CU-LABEL: global_singlethread_one_as_release_atomicrmw:
4134; GFX10-CU:       ; %bb.0: ; %entry
4135; GFX10-CU-NEXT:    s_clause 0x1
4136; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
4137; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4138; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
4139; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4140; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
4141; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
4142; GFX10-CU-NEXT:    s_endpgm
4143;
4144; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_atomicrmw:
4145; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4146; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
4147; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
4148; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
4149; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
4150; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4151; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4152; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
4153; SKIP-CACHE-INV-NEXT:    s_endpgm
4154;
4155; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_atomicrmw:
4156; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4157; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4158; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4159; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4160; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4161; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4162; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
4163; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4164;
4165; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_release_atomicrmw:
4166; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4167; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4168; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4169; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4170; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4171; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4172; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
4173; GFX90A-TGSPLIT-NEXT:    s_endpgm
4174;
4175;
4176    i32 addrspace(1)* %out, i32 %in) {
4177entry:
4178  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") release
4179  ret void
4180}
4181
4182define amdgpu_kernel void @global_singlethread_one_as_acq_rel_atomicrmw(
4183; GFX6-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
4184; GFX6:       ; %bb.0: ; %entry
4185; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4186; GFX6-NEXT:    s_load_dword s4, s[4:5], 0x2
4187; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4188; GFX6-NEXT:    s_mov_b32 s2, -1
4189; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4190; GFX6-NEXT:    v_mov_b32_e32 v0, s4
4191; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
4192; GFX6-NEXT:    s_endpgm
4193;
4194; GFX7-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
4195; GFX7:       ; %bb.0: ; %entry
4196; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4197; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
4198; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4199; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4200; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4201; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4202; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
4203; GFX7-NEXT:    s_endpgm
4204;
4205; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
4206; GFX10-WGP:       ; %bb.0: ; %entry
4207; GFX10-WGP-NEXT:    s_clause 0x1
4208; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
4209; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4210; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
4211; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4212; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
4213; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
4214; GFX10-WGP-NEXT:    s_endpgm
4215;
4216; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
4217; GFX10-CU:       ; %bb.0: ; %entry
4218; GFX10-CU-NEXT:    s_clause 0x1
4219; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
4220; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4221; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
4222; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4223; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
4224; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
4225; GFX10-CU-NEXT:    s_endpgm
4226;
4227; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
4228; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4229; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
4230; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
4231; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
4232; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
4233; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4234; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4235; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
4236; SKIP-CACHE-INV-NEXT:    s_endpgm
4237;
4238; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
4239; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4240; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4241; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4242; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4243; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4244; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4245; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
4246; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4247;
4248; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
4249; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4250; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4251; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4252; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4253; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4254; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4255; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
4256; GFX90A-TGSPLIT-NEXT:    s_endpgm
4257;
4258;
4259    i32 addrspace(1)* %out, i32 %in) {
4260entry:
4261  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") acq_rel
4262  ret void
4263}
4264
4265define amdgpu_kernel void @global_singlethread_one_as_seq_cst_atomicrmw(
4266; GFX6-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
4267; GFX6:       ; %bb.0: ; %entry
4268; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4269; GFX6-NEXT:    s_load_dword s4, s[4:5], 0x2
4270; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4271; GFX6-NEXT:    s_mov_b32 s2, -1
4272; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4273; GFX6-NEXT:    v_mov_b32_e32 v0, s4
4274; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
4275; GFX6-NEXT:    s_endpgm
4276;
4277; GFX7-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
4278; GFX7:       ; %bb.0: ; %entry
4279; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4280; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
4281; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4282; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4283; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4284; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4285; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
4286; GFX7-NEXT:    s_endpgm
4287;
4288; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
4289; GFX10-WGP:       ; %bb.0: ; %entry
4290; GFX10-WGP-NEXT:    s_clause 0x1
4291; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
4292; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4293; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
4294; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4295; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
4296; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
4297; GFX10-WGP-NEXT:    s_endpgm
4298;
4299; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
4300; GFX10-CU:       ; %bb.0: ; %entry
4301; GFX10-CU-NEXT:    s_clause 0x1
4302; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
4303; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4304; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
4305; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4306; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
4307; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
4308; GFX10-CU-NEXT:    s_endpgm
4309;
4310; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
4311; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4312; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
4313; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
4314; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
4315; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
4316; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4317; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4318; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
4319; SKIP-CACHE-INV-NEXT:    s_endpgm
4320;
4321; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
4322; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4323; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4324; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4325; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4326; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4327; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4328; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
4329; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4330;
4331; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
4332; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4333; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4334; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4335; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4336; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4337; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4338; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
4339; GFX90A-TGSPLIT-NEXT:    s_endpgm
4340;
4341;
4342    i32 addrspace(1)* %out, i32 %in) {
4343entry:
4344  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") seq_cst
4345  ret void
4346}
4347
4348define amdgpu_kernel void @global_singlethread_one_as_acquire_ret_atomicrmw(
4349; GFX6-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
4350; GFX6:       ; %bb.0: ; %entry
4351; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4352; GFX6-NEXT:    s_load_dword s4, s[4:5], 0x2
4353; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4354; GFX6-NEXT:    s_mov_b32 s2, -1
4355; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4356; GFX6-NEXT:    v_mov_b32_e32 v0, s4
4357; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
4358; GFX6-NEXT:    s_waitcnt vmcnt(0)
4359; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4360; GFX6-NEXT:    s_endpgm
4361;
4362; GFX7-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
4363; GFX7:       ; %bb.0: ; %entry
4364; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4365; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
4366; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4367; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4368; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4369; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4370; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4371; GFX7-NEXT:    s_waitcnt vmcnt(0)
4372; GFX7-NEXT:    flat_store_dword v[0:1], v2
4373; GFX7-NEXT:    s_endpgm
4374;
4375; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
4376; GFX10-WGP:       ; %bb.0: ; %entry
4377; GFX10-WGP-NEXT:    s_clause 0x1
4378; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
4379; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4380; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
4381; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4382; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
4383; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
4384; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4385; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
4386; GFX10-WGP-NEXT:    s_endpgm
4387;
4388; GFX10-CU-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
4389; GFX10-CU:       ; %bb.0: ; %entry
4390; GFX10-CU-NEXT:    s_clause 0x1
4391; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
4392; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4393; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
4394; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4395; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
4396; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
4397; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4398; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
4399; GFX10-CU-NEXT:    s_endpgm
4400;
4401; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
4402; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4403; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
4404; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
4405; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
4406; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
4407; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4408; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4409; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
4410; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4411; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
4412; SKIP-CACHE-INV-NEXT:    s_endpgm
4413;
4414; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
4415; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4416; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4417; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4418; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4419; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4420; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4421; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
4422; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4423; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
4424; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4425;
4426; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
4427; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4428; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4429; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4430; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4431; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4432; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4433; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
4434; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4435; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
4436; GFX90A-TGSPLIT-NEXT:    s_endpgm
4437;
4438;
4439    i32 addrspace(1)* %out, i32 %in) {
4440entry:
4441  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") acquire
4442  store i32 %val, i32 addrspace(1)* %out, align 4
4443  ret void
4444}
4445
4446define amdgpu_kernel void @global_singlethread_one_as_acq_rel_ret_atomicrmw(
4447; GFX6-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
4448; GFX6:       ; %bb.0: ; %entry
4449; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4450; GFX6-NEXT:    s_load_dword s4, s[4:5], 0x2
4451; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4452; GFX6-NEXT:    s_mov_b32 s2, -1
4453; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4454; GFX6-NEXT:    v_mov_b32_e32 v0, s4
4455; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
4456; GFX6-NEXT:    s_waitcnt vmcnt(0)
4457; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4458; GFX6-NEXT:    s_endpgm
4459;
4460; GFX7-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
4461; GFX7:       ; %bb.0: ; %entry
4462; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4463; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
4464; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4465; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4466; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4467; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4468; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4469; GFX7-NEXT:    s_waitcnt vmcnt(0)
4470; GFX7-NEXT:    flat_store_dword v[0:1], v2
4471; GFX7-NEXT:    s_endpgm
4472;
4473; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
4474; GFX10-WGP:       ; %bb.0: ; %entry
4475; GFX10-WGP-NEXT:    s_clause 0x1
4476; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
4477; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4478; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
4479; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4480; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
4481; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
4482; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4483; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
4484; GFX10-WGP-NEXT:    s_endpgm
4485;
4486; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
4487; GFX10-CU:       ; %bb.0: ; %entry
4488; GFX10-CU-NEXT:    s_clause 0x1
4489; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
4490; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4491; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
4492; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4493; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
4494; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
4495; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4496; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
4497; GFX10-CU-NEXT:    s_endpgm
4498;
4499; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
4500; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4501; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
4502; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
4503; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
4504; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
4505; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4506; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4507; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
4508; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4509; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
4510; SKIP-CACHE-INV-NEXT:    s_endpgm
4511;
4512; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
4513; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4514; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4515; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4516; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4517; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4518; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4519; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
4520; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4521; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
4522; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4523;
4524; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
4525; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4526; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4527; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4528; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4529; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4530; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4531; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
4532; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4533; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
4534; GFX90A-TGSPLIT-NEXT:    s_endpgm
4535;
4536;
4537    i32 addrspace(1)* %out, i32 %in) {
4538entry:
4539  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") acq_rel
4540  store i32 %val, i32 addrspace(1)* %out, align 4
4541  ret void
4542}
4543
4544define amdgpu_kernel void @global_singlethread_one_as_seq_cst_ret_atomicrmw(
4545; GFX6-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
4546; GFX6:       ; %bb.0: ; %entry
4547; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4548; GFX6-NEXT:    s_load_dword s4, s[4:5], 0x2
4549; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4550; GFX6-NEXT:    s_mov_b32 s2, -1
4551; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4552; GFX6-NEXT:    v_mov_b32_e32 v0, s4
4553; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
4554; GFX6-NEXT:    s_waitcnt vmcnt(0)
4555; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4556; GFX6-NEXT:    s_endpgm
4557;
4558; GFX7-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
4559; GFX7:       ; %bb.0: ; %entry
4560; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4561; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
4562; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4563; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4564; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4565; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4566; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4567; GFX7-NEXT:    s_waitcnt vmcnt(0)
4568; GFX7-NEXT:    flat_store_dword v[0:1], v2
4569; GFX7-NEXT:    s_endpgm
4570;
4571; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
4572; GFX10-WGP:       ; %bb.0: ; %entry
4573; GFX10-WGP-NEXT:    s_clause 0x1
4574; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
4575; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4576; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
4577; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4578; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
4579; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
4580; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4581; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
4582; GFX10-WGP-NEXT:    s_endpgm
4583;
4584; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
4585; GFX10-CU:       ; %bb.0: ; %entry
4586; GFX10-CU-NEXT:    s_clause 0x1
4587; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
4588; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4589; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
4590; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4591; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
4592; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
4593; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4594; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
4595; GFX10-CU-NEXT:    s_endpgm
4596;
4597; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
4598; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4599; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
4600; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
4601; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
4602; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
4603; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4604; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4605; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
4606; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4607; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
4608; SKIP-CACHE-INV-NEXT:    s_endpgm
4609;
4610; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
4611; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4612; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4613; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4614; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4615; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4616; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4617; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
4618; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4619; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
4620; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4621;
4622; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
4623; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4624; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4625; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4626; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4627; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4628; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4629; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
4630; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4631; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
4632; GFX90A-TGSPLIT-NEXT:    s_endpgm
4633;
4634;
4635    i32 addrspace(1)* %out, i32 %in) {
4636entry:
4637  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") seq_cst
4638  store i32 %val, i32 addrspace(1)* %out, align 4
4639  ret void
4640}
4641
4642define amdgpu_kernel void @global_singlethread_one_as_monotonic_monotonic_cmpxchg(
4643; GFX6-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
4644; GFX6:       ; %bb.0: ; %entry
4645; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4646; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
4647; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4648; GFX6-NEXT:    s_mov_b32 s2, -1
4649; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4650; GFX6-NEXT:    v_mov_b32_e32 v0, s4
4651; GFX6-NEXT:    v_mov_b32_e32 v1, s5
4652; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
4653; GFX6-NEXT:    s_endpgm
4654;
4655; GFX7-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
4656; GFX7:       ; %bb.0: ; %entry
4657; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4658; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4659; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4660; GFX7-NEXT:    s_add_u32 s0, s0, 16
4661; GFX7-NEXT:    s_addc_u32 s1, s1, 0
4662; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4663; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4664; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4665; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4666; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4667; GFX7-NEXT:    s_endpgm
4668;
4669; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
4670; GFX10-WGP:       ; %bb.0: ; %entry
4671; GFX10-WGP-NEXT:    s_clause 0x1
4672; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4673; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4674; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
4675; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4676; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
4677; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
4678; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
4679; GFX10-WGP-NEXT:    s_endpgm
4680;
4681; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
4682; GFX10-CU:       ; %bb.0: ; %entry
4683; GFX10-CU-NEXT:    s_clause 0x1
4684; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4685; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4686; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
4687; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4688; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
4689; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
4690; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
4691; GFX10-CU-NEXT:    s_endpgm
4692;
4693; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
4694; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4695; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
4696; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
4697; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
4698; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
4699; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4700; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4701; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
4702; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
4703; SKIP-CACHE-INV-NEXT:    s_endpgm
4704;
4705; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
4706; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4707; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4708; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4709; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4710; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4711; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
4712; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
4713; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4714;
4715; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
4716; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4717; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4718; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4719; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4720; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4721; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
4722; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
4723; GFX90A-TGSPLIT-NEXT:    s_endpgm
4724;
4725;
4726    i32 addrspace(1)* %out, i32 %in, i32 %old) {
4727entry:
4728  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
4729  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic monotonic
4730  ret void
4731}
4732
4733define amdgpu_kernel void @global_singlethread_one_as_acquire_monotonic_cmpxchg(
4734; GFX6-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
4735; GFX6:       ; %bb.0: ; %entry
4736; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4737; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
4738; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4739; GFX6-NEXT:    s_mov_b32 s2, -1
4740; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4741; GFX6-NEXT:    v_mov_b32_e32 v0, s4
4742; GFX6-NEXT:    v_mov_b32_e32 v1, s5
4743; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
4744; GFX6-NEXT:    s_endpgm
4745;
4746; GFX7-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
4747; GFX7:       ; %bb.0: ; %entry
4748; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4749; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4750; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4751; GFX7-NEXT:    s_add_u32 s0, s0, 16
4752; GFX7-NEXT:    s_addc_u32 s1, s1, 0
4753; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4754; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4755; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4756; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4757; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4758; GFX7-NEXT:    s_endpgm
4759;
4760; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
4761; GFX10-WGP:       ; %bb.0: ; %entry
4762; GFX10-WGP-NEXT:    s_clause 0x1
4763; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4764; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4765; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
4766; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4767; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
4768; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
4769; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
4770; GFX10-WGP-NEXT:    s_endpgm
4771;
4772; GFX10-CU-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
4773; GFX10-CU:       ; %bb.0: ; %entry
4774; GFX10-CU-NEXT:    s_clause 0x1
4775; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4776; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4777; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
4778; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4779; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
4780; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
4781; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
4782; GFX10-CU-NEXT:    s_endpgm
4783;
4784; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
4785; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4786; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
4787; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
4788; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
4789; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
4790; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4791; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4792; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
4793; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
4794; SKIP-CACHE-INV-NEXT:    s_endpgm
4795;
4796; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
4797; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4798; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4799; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4800; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4801; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4802; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
4803; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
4804; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4805;
4806; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
4807; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4808; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4809; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4810; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4811; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4812; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
4813; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
4814; GFX90A-TGSPLIT-NEXT:    s_endpgm
4815;
4816;
4817    i32 addrspace(1)* %out, i32 %in, i32 %old) {
4818entry:
4819  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
4820  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire monotonic
4821  ret void
4822}
4823
4824define amdgpu_kernel void @global_singlethread_one_as_release_monotonic_cmpxchg(
4825; GFX6-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
4826; GFX6:       ; %bb.0: ; %entry
4827; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4828; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
4829; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4830; GFX6-NEXT:    s_mov_b32 s2, -1
4831; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4832; GFX6-NEXT:    v_mov_b32_e32 v0, s4
4833; GFX6-NEXT:    v_mov_b32_e32 v1, s5
4834; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
4835; GFX6-NEXT:    s_endpgm
4836;
4837; GFX7-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
4838; GFX7:       ; %bb.0: ; %entry
4839; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4840; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4841; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4842; GFX7-NEXT:    s_add_u32 s0, s0, 16
4843; GFX7-NEXT:    s_addc_u32 s1, s1, 0
4844; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4845; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4846; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4847; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4848; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4849; GFX7-NEXT:    s_endpgm
4850;
4851; GFX10-WGP-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
4852; GFX10-WGP:       ; %bb.0: ; %entry
4853; GFX10-WGP-NEXT:    s_clause 0x1
4854; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4855; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4856; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
4857; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4858; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
4859; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
4860; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
4861; GFX10-WGP-NEXT:    s_endpgm
4862;
4863; GFX10-CU-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
4864; GFX10-CU:       ; %bb.0: ; %entry
4865; GFX10-CU-NEXT:    s_clause 0x1
4866; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4867; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4868; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
4869; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4870; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
4871; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
4872; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
4873; GFX10-CU-NEXT:    s_endpgm
4874;
4875; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
4876; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4877; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
4878; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
4879; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
4880; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
4881; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4882; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4883; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
4884; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
4885; SKIP-CACHE-INV-NEXT:    s_endpgm
4886;
4887; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
4888; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4889; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4890; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4891; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4892; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4893; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
4894; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
4895; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4896;
4897; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
4898; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4899; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4900; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4901; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4902; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4903; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
4904; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
4905; GFX90A-TGSPLIT-NEXT:    s_endpgm
4906;
4907;
4908    i32 addrspace(1)* %out, i32 %in, i32 %old) {
4909entry:
4910  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
4911  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release monotonic
4912  ret void
4913}
4914
4915define amdgpu_kernel void @global_singlethread_one_as_acq_rel_monotonic_cmpxchg(
4916; GFX6-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
4917; GFX6:       ; %bb.0: ; %entry
4918; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4919; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
4920; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4921; GFX6-NEXT:    s_mov_b32 s2, -1
4922; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4923; GFX6-NEXT:    v_mov_b32_e32 v0, s4
4924; GFX6-NEXT:    v_mov_b32_e32 v1, s5
4925; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
4926; GFX6-NEXT:    s_endpgm
4927;
4928; GFX7-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
4929; GFX7:       ; %bb.0: ; %entry
4930; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4931; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4932; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4933; GFX7-NEXT:    s_add_u32 s0, s0, 16
4934; GFX7-NEXT:    s_addc_u32 s1, s1, 0
4935; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4936; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4937; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4938; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4939; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4940; GFX7-NEXT:    s_endpgm
4941;
4942; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
4943; GFX10-WGP:       ; %bb.0: ; %entry
4944; GFX10-WGP-NEXT:    s_clause 0x1
4945; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4946; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4947; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
4948; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4949; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
4950; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
4951; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
4952; GFX10-WGP-NEXT:    s_endpgm
4953;
4954; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
4955; GFX10-CU:       ; %bb.0: ; %entry
4956; GFX10-CU-NEXT:    s_clause 0x1
4957; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4958; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4959; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
4960; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4961; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
4962; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
4963; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
4964; GFX10-CU-NEXT:    s_endpgm
4965;
4966; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
4967; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4968; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
4969; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
4970; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
4971; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
4972; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4973; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4974; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
4975; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
4976; SKIP-CACHE-INV-NEXT:    s_endpgm
4977;
4978; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
4979; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4980; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4981; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4982; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4983; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4984; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
4985; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
4986; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4987;
4988; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
4989; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4990; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4991; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4992; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4993; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4994; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
4995; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
4996; GFX90A-TGSPLIT-NEXT:    s_endpgm
4997;
4998;
4999    i32 addrspace(1)* %out, i32 %in, i32 %old) {
5000entry:
5001  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
5002  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel monotonic
5003  ret void
5004}
5005
5006define amdgpu_kernel void @global_singlethread_one_as_seq_cst_monotonic_cmpxchg(
5007; GFX6-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
5008; GFX6:       ; %bb.0: ; %entry
5009; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5010; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
5011; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5012; GFX6-NEXT:    s_mov_b32 s2, -1
5013; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5014; GFX6-NEXT:    v_mov_b32_e32 v0, s4
5015; GFX6-NEXT:    v_mov_b32_e32 v1, s5
5016; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
5017; GFX6-NEXT:    s_endpgm
5018;
5019; GFX7-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
5020; GFX7:       ; %bb.0: ; %entry
5021; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5022; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5023; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5024; GFX7-NEXT:    s_add_u32 s0, s0, 16
5025; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5026; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5027; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5028; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5029; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5030; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5031; GFX7-NEXT:    s_endpgm
5032;
5033; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
5034; GFX10-WGP:       ; %bb.0: ; %entry
5035; GFX10-WGP-NEXT:    s_clause 0x1
5036; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5037; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5038; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
5039; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5040; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
5041; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
5042; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
5043; GFX10-WGP-NEXT:    s_endpgm
5044;
5045; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
5046; GFX10-CU:       ; %bb.0: ; %entry
5047; GFX10-CU-NEXT:    s_clause 0x1
5048; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5049; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5050; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
5051; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5052; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
5053; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
5054; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
5055; GFX10-CU-NEXT:    s_endpgm
5056;
5057; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
5058; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5059; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
5060; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5061; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
5062; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
5063; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5064; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5065; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
5066; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
5067; SKIP-CACHE-INV-NEXT:    s_endpgm
5068;
5069; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
5070; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5071; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5072; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5073; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5074; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5075; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
5076; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
5077; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5078;
5079; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
5080; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5081; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5082; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5083; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5084; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5085; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
5086; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
5087; GFX90A-TGSPLIT-NEXT:    s_endpgm
5088;
5089;
5090    i32 addrspace(1)* %out, i32 %in, i32 %old) {
5091entry:
5092  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
5093  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst monotonic
5094  ret void
5095}
5096
5097define amdgpu_kernel void @global_singlethread_one_as_acquire_acquire_cmpxchg(
5098; GFX6-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
5099; GFX6:       ; %bb.0: ; %entry
5100; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5101; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
5102; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5103; GFX6-NEXT:    s_mov_b32 s2, -1
5104; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5105; GFX6-NEXT:    v_mov_b32_e32 v0, s4
5106; GFX6-NEXT:    v_mov_b32_e32 v1, s5
5107; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
5108; GFX6-NEXT:    s_endpgm
5109;
5110; GFX7-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
5111; GFX7:       ; %bb.0: ; %entry
5112; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5113; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5114; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5115; GFX7-NEXT:    s_add_u32 s0, s0, 16
5116; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5117; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5118; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5119; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5120; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5121; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5122; GFX7-NEXT:    s_endpgm
5123;
5124; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
5125; GFX10-WGP:       ; %bb.0: ; %entry
5126; GFX10-WGP-NEXT:    s_clause 0x1
5127; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5128; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5129; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
5130; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5131; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
5132; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
5133; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
5134; GFX10-WGP-NEXT:    s_endpgm
5135;
5136; GFX10-CU-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
5137; GFX10-CU:       ; %bb.0: ; %entry
5138; GFX10-CU-NEXT:    s_clause 0x1
5139; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5140; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5141; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
5142; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5143; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
5144; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
5145; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
5146; GFX10-CU-NEXT:    s_endpgm
5147;
5148; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
5149; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5150; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
5151; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5152; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
5153; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
5154; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5155; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5156; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
5157; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
5158; SKIP-CACHE-INV-NEXT:    s_endpgm
5159;
5160; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
5161; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5162; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5163; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5164; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5165; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5166; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
5167; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
5168; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5169;
5170; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
5171; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5172; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5173; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5174; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5175; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5176; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
5177; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
5178; GFX90A-TGSPLIT-NEXT:    s_endpgm
5179;
5180;
5181    i32 addrspace(1)* %out, i32 %in, i32 %old) {
5182entry:
5183  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
5184  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire acquire
5185  ret void
5186}
5187
5188define amdgpu_kernel void @global_singlethread_one_as_release_acquire_cmpxchg(
5189; GFX6-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
5190; GFX6:       ; %bb.0: ; %entry
5191; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5192; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
5193; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5194; GFX6-NEXT:    s_mov_b32 s2, -1
5195; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5196; GFX6-NEXT:    v_mov_b32_e32 v0, s4
5197; GFX6-NEXT:    v_mov_b32_e32 v1, s5
5198; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
5199; GFX6-NEXT:    s_endpgm
5200;
5201; GFX7-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
5202; GFX7:       ; %bb.0: ; %entry
5203; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5204; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5205; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5206; GFX7-NEXT:    s_add_u32 s0, s0, 16
5207; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5208; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5209; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5210; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5211; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5212; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5213; GFX7-NEXT:    s_endpgm
5214;
5215; GFX10-WGP-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
5216; GFX10-WGP:       ; %bb.0: ; %entry
5217; GFX10-WGP-NEXT:    s_clause 0x1
5218; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5219; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5220; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
5221; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5222; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
5223; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
5224; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
5225; GFX10-WGP-NEXT:    s_endpgm
5226;
5227; GFX10-CU-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
5228; GFX10-CU:       ; %bb.0: ; %entry
5229; GFX10-CU-NEXT:    s_clause 0x1
5230; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5231; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5232; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
5233; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5234; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
5235; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
5236; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
5237; GFX10-CU-NEXT:    s_endpgm
5238;
5239; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
5240; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5241; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
5242; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5243; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
5244; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
5245; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5246; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5247; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
5248; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
5249; SKIP-CACHE-INV-NEXT:    s_endpgm
5250;
5251; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
5252; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5253; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5254; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5255; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5256; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5257; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
5258; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
5259; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5260;
5261; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
5262; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5263; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5264; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5265; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5266; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5267; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
5268; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
5269; GFX90A-TGSPLIT-NEXT:    s_endpgm
5270;
5271;
5272    i32 addrspace(1)* %out, i32 %in, i32 %old) {
5273entry:
5274  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
5275  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release acquire
5276  ret void
5277}
5278
5279define amdgpu_kernel void @global_singlethread_one_as_acq_rel_acquire_cmpxchg(
5280; GFX6-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
5281; GFX6:       ; %bb.0: ; %entry
5282; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5283; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
5284; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5285; GFX6-NEXT:    s_mov_b32 s2, -1
5286; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5287; GFX6-NEXT:    v_mov_b32_e32 v0, s4
5288; GFX6-NEXT:    v_mov_b32_e32 v1, s5
5289; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
5290; GFX6-NEXT:    s_endpgm
5291;
5292; GFX7-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
5293; GFX7:       ; %bb.0: ; %entry
5294; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5295; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5296; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5297; GFX7-NEXT:    s_add_u32 s0, s0, 16
5298; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5299; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5300; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5301; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5302; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5303; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5304; GFX7-NEXT:    s_endpgm
5305;
5306; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
5307; GFX10-WGP:       ; %bb.0: ; %entry
5308; GFX10-WGP-NEXT:    s_clause 0x1
5309; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5310; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5311; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
5312; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5313; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
5314; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
5315; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
5316; GFX10-WGP-NEXT:    s_endpgm
5317;
5318; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
5319; GFX10-CU:       ; %bb.0: ; %entry
5320; GFX10-CU-NEXT:    s_clause 0x1
5321; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5322; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5323; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
5324; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5325; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
5326; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
5327; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
5328; GFX10-CU-NEXT:    s_endpgm
5329;
5330; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
5331; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5332; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
5333; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5334; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
5335; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
5336; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5337; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5338; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
5339; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
5340; SKIP-CACHE-INV-NEXT:    s_endpgm
5341;
5342; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
5343; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5344; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5345; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5346; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5347; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5348; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
5349; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
5350; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5351;
5352; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
5353; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5354; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5355; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5356; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5357; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5358; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
5359; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
5360; GFX90A-TGSPLIT-NEXT:    s_endpgm
5361;
5362;
5363    i32 addrspace(1)* %out, i32 %in, i32 %old) {
5364entry:
5365  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
5366  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel acquire
5367  ret void
5368}
5369
5370define amdgpu_kernel void @global_singlethread_one_as_seq_cst_acquire_cmpxchg(
5371; GFX6-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
5372; GFX6:       ; %bb.0: ; %entry
5373; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5374; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
5375; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5376; GFX6-NEXT:    s_mov_b32 s2, -1
5377; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5378; GFX6-NEXT:    v_mov_b32_e32 v0, s4
5379; GFX6-NEXT:    v_mov_b32_e32 v1, s5
5380; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
5381; GFX6-NEXT:    s_endpgm
5382;
5383; GFX7-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
5384; GFX7:       ; %bb.0: ; %entry
5385; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5386; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5387; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5388; GFX7-NEXT:    s_add_u32 s0, s0, 16
5389; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5390; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5391; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5392; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5393; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5394; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5395; GFX7-NEXT:    s_endpgm
5396;
5397; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
5398; GFX10-WGP:       ; %bb.0: ; %entry
5399; GFX10-WGP-NEXT:    s_clause 0x1
5400; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5401; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5402; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
5403; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5404; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
5405; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
5406; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
5407; GFX10-WGP-NEXT:    s_endpgm
5408;
5409; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
5410; GFX10-CU:       ; %bb.0: ; %entry
5411; GFX10-CU-NEXT:    s_clause 0x1
5412; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5413; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5414; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
5415; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5416; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
5417; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
5418; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
5419; GFX10-CU-NEXT:    s_endpgm
5420;
5421; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
5422; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5423; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
5424; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5425; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
5426; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
5427; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5428; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5429; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
5430; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
5431; SKIP-CACHE-INV-NEXT:    s_endpgm
5432;
5433; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
5434; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5435; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5436; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5437; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5438; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5439; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
5440; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
5441; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5442;
5443; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
5444; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5445; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5446; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5447; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5448; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5449; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
5450; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
5451; GFX90A-TGSPLIT-NEXT:    s_endpgm
5452;
5453;
5454    i32 addrspace(1)* %out, i32 %in, i32 %old) {
5455entry:
5456  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
5457  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst acquire
5458  ret void
5459}
5460
5461define amdgpu_kernel void @global_singlethread_one_as_seq_cst_seq_cst_cmpxchg(
5462; GFX6-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
5463; GFX6:       ; %bb.0: ; %entry
5464; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5465; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
5466; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5467; GFX6-NEXT:    s_mov_b32 s2, -1
5468; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5469; GFX6-NEXT:    v_mov_b32_e32 v0, s4
5470; GFX6-NEXT:    v_mov_b32_e32 v1, s5
5471; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
5472; GFX6-NEXT:    s_endpgm
5473;
5474; GFX7-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
5475; GFX7:       ; %bb.0: ; %entry
5476; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5477; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5478; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5479; GFX7-NEXT:    s_add_u32 s0, s0, 16
5480; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5481; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5482; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5483; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5484; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5485; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5486; GFX7-NEXT:    s_endpgm
5487;
5488; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
5489; GFX10-WGP:       ; %bb.0: ; %entry
5490; GFX10-WGP-NEXT:    s_clause 0x1
5491; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5492; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5493; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
5494; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5495; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
5496; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
5497; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
5498; GFX10-WGP-NEXT:    s_endpgm
5499;
5500; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
5501; GFX10-CU:       ; %bb.0: ; %entry
5502; GFX10-CU-NEXT:    s_clause 0x1
5503; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5504; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5505; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
5506; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5507; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
5508; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
5509; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
5510; GFX10-CU-NEXT:    s_endpgm
5511;
5512; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
5513; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5514; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
5515; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5516; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
5517; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
5518; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5519; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5520; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
5521; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
5522; SKIP-CACHE-INV-NEXT:    s_endpgm
5523;
5524; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
5525; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5526; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5527; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5528; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5529; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5530; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
5531; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
5532; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5533;
5534; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
5535; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5536; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5537; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5538; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5539; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5540; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
5541; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
5542; GFX90A-TGSPLIT-NEXT:    s_endpgm
5543;
5544;
5545    i32 addrspace(1)* %out, i32 %in, i32 %old) {
5546entry:
5547  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
5548  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst seq_cst
5549  ret void
5550}
5551
5552define amdgpu_kernel void @global_singlethread_one_as_acquire_monotonic_ret_cmpxchg(
5553; GFX6-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
5554; GFX6:       ; %bb.0: ; %entry
5555; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5556; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
5557; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5558; GFX6-NEXT:    s_mov_b32 s2, -1
5559; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5560; GFX6-NEXT:    v_mov_b32_e32 v0, s4
5561; GFX6-NEXT:    v_mov_b32_e32 v1, s5
5562; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
5563; GFX6-NEXT:    s_waitcnt vmcnt(0)
5564; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5565; GFX6-NEXT:    s_endpgm
5566;
5567; GFX7-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
5568; GFX7:       ; %bb.0: ; %entry
5569; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5570; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5571; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5572; GFX7-NEXT:    s_add_u32 s4, s0, 16
5573; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5574; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5575; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5576; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5577; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5578; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5579; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5580; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5581; GFX7-NEXT:    s_waitcnt vmcnt(0)
5582; GFX7-NEXT:    flat_store_dword v[0:1], v2
5583; GFX7-NEXT:    s_endpgm
5584;
5585; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
5586; GFX10-WGP:       ; %bb.0: ; %entry
5587; GFX10-WGP-NEXT:    s_clause 0x1
5588; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5589; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5590; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
5591; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5592; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
5593; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
5594; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
5595; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5596; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[0:1]
5597; GFX10-WGP-NEXT:    s_endpgm
5598;
5599; GFX10-CU-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
5600; GFX10-CU:       ; %bb.0: ; %entry
5601; GFX10-CU-NEXT:    s_clause 0x1
5602; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5603; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5604; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
5605; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5606; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
5607; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
5608; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
5609; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5610; GFX10-CU-NEXT:    global_store_dword v2, v0, s[0:1]
5611; GFX10-CU-NEXT:    s_endpgm
5612;
5613; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
5614; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5615; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
5616; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5617; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
5618; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
5619; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5620; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5621; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
5622; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
5623; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5624; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
5625; SKIP-CACHE-INV-NEXT:    s_endpgm
5626;
5627; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
5628; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5629; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5630; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5631; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5632; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5633; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
5634; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
5635; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5636; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
5637; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5638;
5639; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
5640; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5641; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5642; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5643; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5644; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5645; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
5646; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
5647; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5648; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
5649; GFX90A-TGSPLIT-NEXT:    s_endpgm
5650;
5651;
5652    i32 addrspace(1)* %out, i32 %in, i32 %old) {
5653entry:
5654  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
5655  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire monotonic
5656  %val0 = extractvalue { i32, i1 } %val, 0
5657  store i32 %val0, i32 addrspace(1)* %out, align 4
5658  ret void
5659}
5660
5661define amdgpu_kernel void @global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg(
5662; GFX6-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
5663; GFX6:       ; %bb.0: ; %entry
5664; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5665; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
5666; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5667; GFX6-NEXT:    s_mov_b32 s2, -1
5668; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5669; GFX6-NEXT:    v_mov_b32_e32 v0, s4
5670; GFX6-NEXT:    v_mov_b32_e32 v1, s5
5671; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
5672; GFX6-NEXT:    s_waitcnt vmcnt(0)
5673; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5674; GFX6-NEXT:    s_endpgm
5675;
5676; GFX7-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
5677; GFX7:       ; %bb.0: ; %entry
5678; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5679; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5680; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5681; GFX7-NEXT:    s_add_u32 s4, s0, 16
5682; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5683; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5684; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5685; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5686; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5687; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5688; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5689; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5690; GFX7-NEXT:    s_waitcnt vmcnt(0)
5691; GFX7-NEXT:    flat_store_dword v[0:1], v2
5692; GFX7-NEXT:    s_endpgm
5693;
5694; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
5695; GFX10-WGP:       ; %bb.0: ; %entry
5696; GFX10-WGP-NEXT:    s_clause 0x1
5697; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5698; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5699; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
5700; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5701; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
5702; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
5703; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
5704; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5705; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[0:1]
5706; GFX10-WGP-NEXT:    s_endpgm
5707;
5708; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
5709; GFX10-CU:       ; %bb.0: ; %entry
5710; GFX10-CU-NEXT:    s_clause 0x1
5711; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5712; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5713; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
5714; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5715; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
5716; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
5717; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
5718; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5719; GFX10-CU-NEXT:    global_store_dword v2, v0, s[0:1]
5720; GFX10-CU-NEXT:    s_endpgm
5721;
5722; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
5723; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5724; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
5725; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5726; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
5727; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
5728; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5729; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5730; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
5731; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
5732; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5733; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
5734; SKIP-CACHE-INV-NEXT:    s_endpgm
5735;
5736; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
5737; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5738; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5739; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5740; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5741; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5742; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
5743; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
5744; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5745; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
5746; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5747;
5748; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
5749; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5750; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5751; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5752; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5753; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5754; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
5755; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
5756; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5757; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
5758; GFX90A-TGSPLIT-NEXT:    s_endpgm
5759;
5760;
5761    i32 addrspace(1)* %out, i32 %in, i32 %old) {
5762entry:
5763  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
5764  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel monotonic
5765  %val0 = extractvalue { i32, i1 } %val, 0
5766  store i32 %val0, i32 addrspace(1)* %out, align 4
5767  ret void
5768}
5769
5770define amdgpu_kernel void @global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg(
5771; GFX6-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
5772; GFX6:       ; %bb.0: ; %entry
5773; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5774; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
5775; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5776; GFX6-NEXT:    s_mov_b32 s2, -1
5777; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5778; GFX6-NEXT:    v_mov_b32_e32 v0, s4
5779; GFX6-NEXT:    v_mov_b32_e32 v1, s5
5780; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
5781; GFX6-NEXT:    s_waitcnt vmcnt(0)
5782; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5783; GFX6-NEXT:    s_endpgm
5784;
5785; GFX7-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
5786; GFX7:       ; %bb.0: ; %entry
5787; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5788; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5789; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5790; GFX7-NEXT:    s_add_u32 s4, s0, 16
5791; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5792; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5793; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5794; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5795; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5796; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5797; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5798; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5799; GFX7-NEXT:    s_waitcnt vmcnt(0)
5800; GFX7-NEXT:    flat_store_dword v[0:1], v2
5801; GFX7-NEXT:    s_endpgm
5802;
5803; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
5804; GFX10-WGP:       ; %bb.0: ; %entry
5805; GFX10-WGP-NEXT:    s_clause 0x1
5806; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5807; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5808; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
5809; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5810; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
5811; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
5812; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
5813; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5814; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[0:1]
5815; GFX10-WGP-NEXT:    s_endpgm
5816;
5817; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
5818; GFX10-CU:       ; %bb.0: ; %entry
5819; GFX10-CU-NEXT:    s_clause 0x1
5820; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5821; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5822; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
5823; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5824; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
5825; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
5826; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
5827; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5828; GFX10-CU-NEXT:    global_store_dword v2, v0, s[0:1]
5829; GFX10-CU-NEXT:    s_endpgm
5830;
5831; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
5832; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5833; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
5834; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5835; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
5836; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
5837; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5838; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5839; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
5840; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
5841; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5842; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
5843; SKIP-CACHE-INV-NEXT:    s_endpgm
5844;
5845; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
5846; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5847; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5848; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5849; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5850; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5851; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
5852; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
5853; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5854; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
5855; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5856;
5857; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
5858; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5859; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5860; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5861; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5862; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5863; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
5864; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
5865; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5866; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
5867; GFX90A-TGSPLIT-NEXT:    s_endpgm
5868;
5869;
5870    i32 addrspace(1)* %out, i32 %in, i32 %old) {
5871entry:
5872  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
5873  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst monotonic
5874  %val0 = extractvalue { i32, i1 } %val, 0
5875  store i32 %val0, i32 addrspace(1)* %out, align 4
5876  ret void
5877}
5878
5879define amdgpu_kernel void @global_singlethread_one_as_acquire_acquire_ret_cmpxchg(
5880; GFX6-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
5881; GFX6:       ; %bb.0: ; %entry
5882; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5883; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
5884; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5885; GFX6-NEXT:    s_mov_b32 s2, -1
5886; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5887; GFX6-NEXT:    v_mov_b32_e32 v0, s4
5888; GFX6-NEXT:    v_mov_b32_e32 v1, s5
5889; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
5890; GFX6-NEXT:    s_waitcnt vmcnt(0)
5891; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5892; GFX6-NEXT:    s_endpgm
5893;
5894; GFX7-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
5895; GFX7:       ; %bb.0: ; %entry
5896; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5897; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5898; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5899; GFX7-NEXT:    s_add_u32 s4, s0, 16
5900; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5901; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5902; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5903; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5904; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5905; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5906; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5907; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5908; GFX7-NEXT:    s_waitcnt vmcnt(0)
5909; GFX7-NEXT:    flat_store_dword v[0:1], v2
5910; GFX7-NEXT:    s_endpgm
5911;
5912; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
5913; GFX10-WGP:       ; %bb.0: ; %entry
5914; GFX10-WGP-NEXT:    s_clause 0x1
5915; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5916; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5917; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
5918; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5919; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
5920; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
5921; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
5922; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5923; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[0:1]
5924; GFX10-WGP-NEXT:    s_endpgm
5925;
5926; GFX10-CU-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
5927; GFX10-CU:       ; %bb.0: ; %entry
5928; GFX10-CU-NEXT:    s_clause 0x1
5929; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5930; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5931; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
5932; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5933; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
5934; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
5935; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
5936; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5937; GFX10-CU-NEXT:    global_store_dword v2, v0, s[0:1]
5938; GFX10-CU-NEXT:    s_endpgm
5939;
5940; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
5941; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5942; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
5943; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5944; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
5945; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
5946; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5947; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5948; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
5949; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
5950; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5951; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
5952; SKIP-CACHE-INV-NEXT:    s_endpgm
5953;
5954; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
5955; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5956; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5957; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5958; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5959; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5960; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
5961; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
5962; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5963; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
5964; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5965;
5966; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
5967; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5968; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5969; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5970; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5971; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5972; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
5973; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
5974; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5975; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
5976; GFX90A-TGSPLIT-NEXT:    s_endpgm
5977;
5978;
5979    i32 addrspace(1)* %out, i32 %in, i32 %old) {
5980entry:
5981  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
5982  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire acquire
5983  %val0 = extractvalue { i32, i1 } %val, 0
5984  store i32 %val0, i32 addrspace(1)* %out, align 4
5985  ret void
5986}
5987
5988define amdgpu_kernel void @global_singlethread_one_as_release_acquire_ret_cmpxchg(
5989; GFX6-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
5990; GFX6:       ; %bb.0: ; %entry
5991; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5992; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
5993; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5994; GFX6-NEXT:    s_mov_b32 s2, -1
5995; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5996; GFX6-NEXT:    v_mov_b32_e32 v0, s4
5997; GFX6-NEXT:    v_mov_b32_e32 v1, s5
5998; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
5999; GFX6-NEXT:    s_waitcnt vmcnt(0)
6000; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
6001; GFX6-NEXT:    s_endpgm
6002;
6003; GFX7-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
6004; GFX7:       ; %bb.0: ; %entry
6005; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6006; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6007; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6008; GFX7-NEXT:    s_add_u32 s4, s0, 16
6009; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6010; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6011; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6012; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6013; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6014; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6015; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6016; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6017; GFX7-NEXT:    s_waitcnt vmcnt(0)
6018; GFX7-NEXT:    flat_store_dword v[0:1], v2
6019; GFX7-NEXT:    s_endpgm
6020;
6021; GFX10-WGP-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
6022; GFX10-WGP:       ; %bb.0: ; %entry
6023; GFX10-WGP-NEXT:    s_clause 0x1
6024; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6025; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6026; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
6027; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6028; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
6029; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
6030; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
6031; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6032; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[0:1]
6033; GFX10-WGP-NEXT:    s_endpgm
6034;
6035; GFX10-CU-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
6036; GFX10-CU:       ; %bb.0: ; %entry
6037; GFX10-CU-NEXT:    s_clause 0x1
6038; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6039; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6040; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
6041; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6042; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
6043; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
6044; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
6045; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6046; GFX10-CU-NEXT:    global_store_dword v2, v0, s[0:1]
6047; GFX10-CU-NEXT:    s_endpgm
6048;
6049; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
6050; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6051; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
6052; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
6053; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
6054; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
6055; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6056; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6057; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6058; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
6059; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6060; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
6061; SKIP-CACHE-INV-NEXT:    s_endpgm
6062;
6063; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
6064; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6065; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6066; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6067; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6068; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6069; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
6070; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
6071; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6072; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
6073; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6074;
6075; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
6076; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6077; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6078; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6079; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6080; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6081; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
6082; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
6083; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6084; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
6085; GFX90A-TGSPLIT-NEXT:    s_endpgm
6086;
6087;
6088    i32 addrspace(1)* %out, i32 %in, i32 %old) {
6089entry:
6090  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
6091  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release acquire
6092  %val0 = extractvalue { i32, i1 } %val, 0
6093  store i32 %val0, i32 addrspace(1)* %out, align 4
6094  ret void
6095}
6096
6097define amdgpu_kernel void @global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg(
6098; GFX6-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
6099; GFX6:       ; %bb.0: ; %entry
6100; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6101; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
6102; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
6103; GFX6-NEXT:    s_mov_b32 s2, -1
6104; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6105; GFX6-NEXT:    v_mov_b32_e32 v0, s4
6106; GFX6-NEXT:    v_mov_b32_e32 v1, s5
6107; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
6108; GFX6-NEXT:    s_waitcnt vmcnt(0)
6109; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
6110; GFX6-NEXT:    s_endpgm
6111;
6112; GFX7-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
6113; GFX7:       ; %bb.0: ; %entry
6114; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6115; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6116; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6117; GFX7-NEXT:    s_add_u32 s4, s0, 16
6118; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6119; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6120; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6121; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6122; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6123; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6124; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6125; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6126; GFX7-NEXT:    s_waitcnt vmcnt(0)
6127; GFX7-NEXT:    flat_store_dword v[0:1], v2
6128; GFX7-NEXT:    s_endpgm
6129;
6130; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
6131; GFX10-WGP:       ; %bb.0: ; %entry
6132; GFX10-WGP-NEXT:    s_clause 0x1
6133; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6134; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6135; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
6136; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6137; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
6138; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
6139; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
6140; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6141; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[0:1]
6142; GFX10-WGP-NEXT:    s_endpgm
6143;
6144; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
6145; GFX10-CU:       ; %bb.0: ; %entry
6146; GFX10-CU-NEXT:    s_clause 0x1
6147; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6148; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6149; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
6150; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6151; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
6152; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
6153; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
6154; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6155; GFX10-CU-NEXT:    global_store_dword v2, v0, s[0:1]
6156; GFX10-CU-NEXT:    s_endpgm
6157;
6158; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
6159; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6160; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
6161; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
6162; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
6163; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
6164; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6165; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6166; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6167; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
6168; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6169; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
6170; SKIP-CACHE-INV-NEXT:    s_endpgm
6171;
6172; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
6173; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6174; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6175; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6176; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6177; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6178; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
6179; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
6180; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6181; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
6182; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6183;
6184; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
6185; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6186; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6187; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6188; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6189; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6190; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
6191; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
6192; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6193; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
6194; GFX90A-TGSPLIT-NEXT:    s_endpgm
6195;
6196;
6197    i32 addrspace(1)* %out, i32 %in, i32 %old) {
6198entry:
6199  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
6200  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel acquire
6201  %val0 = extractvalue { i32, i1 } %val, 0
6202  store i32 %val0, i32 addrspace(1)* %out, align 4
6203  ret void
6204}
6205
6206define amdgpu_kernel void @global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg(
6207; GFX6-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
6208; GFX6:       ; %bb.0: ; %entry
6209; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6210; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
6211; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
6212; GFX6-NEXT:    s_mov_b32 s2, -1
6213; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6214; GFX6-NEXT:    v_mov_b32_e32 v0, s4
6215; GFX6-NEXT:    v_mov_b32_e32 v1, s5
6216; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
6217; GFX6-NEXT:    s_waitcnt vmcnt(0)
6218; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
6219; GFX6-NEXT:    s_endpgm
6220;
6221; GFX7-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
6222; GFX7:       ; %bb.0: ; %entry
6223; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6224; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6225; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6226; GFX7-NEXT:    s_add_u32 s4, s0, 16
6227; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6228; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6229; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6230; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6231; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6232; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6233; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6234; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6235; GFX7-NEXT:    s_waitcnt vmcnt(0)
6236; GFX7-NEXT:    flat_store_dword v[0:1], v2
6237; GFX7-NEXT:    s_endpgm
6238;
6239; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
6240; GFX10-WGP:       ; %bb.0: ; %entry
6241; GFX10-WGP-NEXT:    s_clause 0x1
6242; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6243; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6244; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
6245; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6246; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
6247; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
6248; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
6249; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6250; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[0:1]
6251; GFX10-WGP-NEXT:    s_endpgm
6252;
6253; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
6254; GFX10-CU:       ; %bb.0: ; %entry
6255; GFX10-CU-NEXT:    s_clause 0x1
6256; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6257; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6258; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
6259; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6260; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
6261; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
6262; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
6263; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6264; GFX10-CU-NEXT:    global_store_dword v2, v0, s[0:1]
6265; GFX10-CU-NEXT:    s_endpgm
6266;
6267; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
6268; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6269; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
6270; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
6271; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
6272; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
6273; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6274; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6275; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6276; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
6277; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6278; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
6279; SKIP-CACHE-INV-NEXT:    s_endpgm
6280;
6281; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
6282; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6283; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6284; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6285; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6286; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6287; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
6288; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
6289; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6290; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
6291; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6292;
6293; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
6294; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6295; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6296; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6297; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6298; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6299; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
6300; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
6301; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6302; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
6303; GFX90A-TGSPLIT-NEXT:    s_endpgm
6304;
6305;
6306    i32 addrspace(1)* %out, i32 %in, i32 %old) {
6307entry:
6308  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
6309  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst acquire
6310  %val0 = extractvalue { i32, i1 } %val, 0
6311  store i32 %val0, i32 addrspace(1)* %out, align 4
6312  ret void
6313}
6314
6315define amdgpu_kernel void @global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg(
6316; GFX6-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
6317; GFX6:       ; %bb.0: ; %entry
6318; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6319; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
6320; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
6321; GFX6-NEXT:    s_mov_b32 s2, -1
6322; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6323; GFX6-NEXT:    v_mov_b32_e32 v0, s4
6324; GFX6-NEXT:    v_mov_b32_e32 v1, s5
6325; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
6326; GFX6-NEXT:    s_waitcnt vmcnt(0)
6327; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
6328; GFX6-NEXT:    s_endpgm
6329;
6330; GFX7-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
6331; GFX7:       ; %bb.0: ; %entry
6332; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6333; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6334; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6335; GFX7-NEXT:    s_add_u32 s4, s0, 16
6336; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6337; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6338; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6339; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6340; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6341; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6342; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6343; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6344; GFX7-NEXT:    s_waitcnt vmcnt(0)
6345; GFX7-NEXT:    flat_store_dword v[0:1], v2
6346; GFX7-NEXT:    s_endpgm
6347;
6348; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
6349; GFX10-WGP:       ; %bb.0: ; %entry
6350; GFX10-WGP-NEXT:    s_clause 0x1
6351; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6352; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6353; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
6354; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6355; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
6356; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
6357; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
6358; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6359; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[0:1]
6360; GFX10-WGP-NEXT:    s_endpgm
6361;
6362; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
6363; GFX10-CU:       ; %bb.0: ; %entry
6364; GFX10-CU-NEXT:    s_clause 0x1
6365; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6366; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6367; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
6368; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6369; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
6370; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
6371; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
6372; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6373; GFX10-CU-NEXT:    global_store_dword v2, v0, s[0:1]
6374; GFX10-CU-NEXT:    s_endpgm
6375;
6376; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
6377; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6378; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
6379; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
6380; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
6381; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
6382; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6383; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6384; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6385; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
6386; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6387; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
6388; SKIP-CACHE-INV-NEXT:    s_endpgm
6389;
6390; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
6391; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6392; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6393; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6394; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6395; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6396; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
6397; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
6398; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6399; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
6400; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6401;
6402; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
6403; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6404; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6405; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6406; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6407; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6408; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
6409; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
6410; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6411; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
6412; GFX90A-TGSPLIT-NEXT:    s_endpgm
6413;
6414;
6415    i32 addrspace(1)* %out, i32 %in, i32 %old) {
6416entry:
6417  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
6418  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst seq_cst
6419  %val0 = extractvalue { i32, i1 } %val, 0
6420  store i32 %val0, i32 addrspace(1)* %out, align 4
6421  ret void
6422}
6423
6424