1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx600 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX6 %s
3; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
4; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
5; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
6; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
7; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-NOTTGSPLIT %s
8; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-TGSPLIT %s
9; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-NOTTGSPLIT %s
10; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
11
12define amdgpu_kernel void @global_singlethread_unordered_load(
13; GFX6-LABEL: global_singlethread_unordered_load:
14; GFX6:       ; %bb.0: ; %entry
15; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
16; GFX6-NEXT:    s_mov_b32 s7, 0x100f000
17; GFX6-NEXT:    s_mov_b32 s6, -1
18; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
19; GFX6-NEXT:    s_mov_b32 s4, s0
20; GFX6-NEXT:    s_mov_b32 s5, s1
21; GFX6-NEXT:    buffer_load_dword v0, off, s[4:7], 0
22; GFX6-NEXT:    s_mov_b32 s4, s2
23; GFX6-NEXT:    s_mov_b32 s5, s3
24; GFX6-NEXT:    s_waitcnt vmcnt(0)
25; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
26; GFX6-NEXT:    s_endpgm
27;
28; GFX7-LABEL: global_singlethread_unordered_load:
29; GFX7:       ; %bb.0: ; %entry
30; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
31; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
32; GFX7-NEXT:    v_mov_b32_e32 v0, s0
33; GFX7-NEXT:    v_mov_b32_e32 v1, s1
34; GFX7-NEXT:    flat_load_dword v2, v[0:1]
35; GFX7-NEXT:    v_mov_b32_e32 v0, s2
36; GFX7-NEXT:    v_mov_b32_e32 v1, s3
37; GFX7-NEXT:    s_waitcnt vmcnt(0)
38; GFX7-NEXT:    flat_store_dword v[0:1], v2
39; GFX7-NEXT:    s_endpgm
40;
41; GFX10-WGP-LABEL: global_singlethread_unordered_load:
42; GFX10-WGP:       ; %bb.0: ; %entry
43; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
44; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
45; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
46; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
47; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
48; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
49; GFX10-WGP-NEXT:    s_endpgm
50;
51; GFX10-CU-LABEL: global_singlethread_unordered_load:
52; GFX10-CU:       ; %bb.0: ; %entry
53; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
54; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
55; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
56; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
57; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
58; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
59; GFX10-CU-NEXT:    s_endpgm
60;
61; SKIP-CACHE-INV-LABEL: global_singlethread_unordered_load:
62; SKIP-CACHE-INV:       ; %bb.0: ; %entry
63; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
64; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
65; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
66; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
67; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s0
68; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s1
69; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[4:7], 0
70; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s2
71; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s3
72; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
73; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
74; SKIP-CACHE-INV-NEXT:    s_endpgm
75;
76; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_unordered_load:
77; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
78; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
79; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
80; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
81; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
82; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
83; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
84; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
85;
86; GFX90A-TGSPLIT-LABEL: global_singlethread_unordered_load:
87; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
88; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
89; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
90; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
91; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
92; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
93; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
94; GFX90A-TGSPLIT-NEXT:    s_endpgm
95;
96; GFX940-NOTTGSPLIT-LABEL: global_singlethread_unordered_load:
97; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
98; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
99; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
100; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
101; GFX940-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
102; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
103; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
104; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
105;
106; GFX940-TGSPLIT-LABEL: global_singlethread_unordered_load:
107; GFX940-TGSPLIT:       ; %bb.0: ; %entry
108; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
109; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
110; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
111; GFX940-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
112; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
113; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
114; GFX940-TGSPLIT-NEXT:    s_endpgm
115    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
116entry:
117  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread") unordered, align 4
118  store i32 %val, i32 addrspace(1)* %out
119  ret void
120}
121
122define amdgpu_kernel void @global_singlethread_monotonic_load(
123; GFX6-LABEL: global_singlethread_monotonic_load:
124; GFX6:       ; %bb.0: ; %entry
125; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
126; GFX6-NEXT:    s_mov_b32 s7, 0x100f000
127; GFX6-NEXT:    s_mov_b32 s6, -1
128; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
129; GFX6-NEXT:    s_mov_b32 s4, s0
130; GFX6-NEXT:    s_mov_b32 s5, s1
131; GFX6-NEXT:    buffer_load_dword v0, off, s[4:7], 0
132; GFX6-NEXT:    s_mov_b32 s4, s2
133; GFX6-NEXT:    s_mov_b32 s5, s3
134; GFX6-NEXT:    s_waitcnt vmcnt(0)
135; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
136; GFX6-NEXT:    s_endpgm
137;
138; GFX7-LABEL: global_singlethread_monotonic_load:
139; GFX7:       ; %bb.0: ; %entry
140; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
141; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
142; GFX7-NEXT:    v_mov_b32_e32 v0, s0
143; GFX7-NEXT:    v_mov_b32_e32 v1, s1
144; GFX7-NEXT:    flat_load_dword v2, v[0:1]
145; GFX7-NEXT:    v_mov_b32_e32 v0, s2
146; GFX7-NEXT:    v_mov_b32_e32 v1, s3
147; GFX7-NEXT:    s_waitcnt vmcnt(0)
148; GFX7-NEXT:    flat_store_dword v[0:1], v2
149; GFX7-NEXT:    s_endpgm
150;
151; GFX10-WGP-LABEL: global_singlethread_monotonic_load:
152; GFX10-WGP:       ; %bb.0: ; %entry
153; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
154; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
155; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
156; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
157; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
158; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
159; GFX10-WGP-NEXT:    s_endpgm
160;
161; GFX10-CU-LABEL: global_singlethread_monotonic_load:
162; GFX10-CU:       ; %bb.0: ; %entry
163; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
164; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
165; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
166; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
167; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
168; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
169; GFX10-CU-NEXT:    s_endpgm
170;
171; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_load:
172; SKIP-CACHE-INV:       ; %bb.0: ; %entry
173; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
174; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
175; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
176; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
177; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s0
178; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s1
179; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[4:7], 0
180; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s2
181; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s3
182; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
183; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
184; SKIP-CACHE-INV-NEXT:    s_endpgm
185;
186; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_load:
187; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
188; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
189; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
190; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
191; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
192; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
193; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
194; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
195;
196; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_load:
197; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
198; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
199; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
200; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
201; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
202; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
203; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
204; GFX90A-TGSPLIT-NEXT:    s_endpgm
205;
206; GFX940-NOTTGSPLIT-LABEL: global_singlethread_monotonic_load:
207; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
208; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
209; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
210; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
211; GFX940-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
212; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
213; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
214; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
215;
216; GFX940-TGSPLIT-LABEL: global_singlethread_monotonic_load:
217; GFX940-TGSPLIT:       ; %bb.0: ; %entry
218; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
219; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
220; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
221; GFX940-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
222; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
223; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
224; GFX940-TGSPLIT-NEXT:    s_endpgm
225    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
226entry:
227  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread") monotonic, align 4
228  store i32 %val, i32 addrspace(1)* %out
229  ret void
230}
231
232define amdgpu_kernel void @global_singlethread_acquire_load(
233; GFX6-LABEL: global_singlethread_acquire_load:
234; GFX6:       ; %bb.0: ; %entry
235; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
236; GFX6-NEXT:    s_mov_b32 s7, 0x100f000
237; GFX6-NEXT:    s_mov_b32 s6, -1
238; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
239; GFX6-NEXT:    s_mov_b32 s4, s0
240; GFX6-NEXT:    s_mov_b32 s5, s1
241; GFX6-NEXT:    buffer_load_dword v0, off, s[4:7], 0
242; GFX6-NEXT:    s_mov_b32 s4, s2
243; GFX6-NEXT:    s_mov_b32 s5, s3
244; GFX6-NEXT:    s_waitcnt vmcnt(0)
245; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
246; GFX6-NEXT:    s_endpgm
247;
248; GFX7-LABEL: global_singlethread_acquire_load:
249; GFX7:       ; %bb.0: ; %entry
250; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
251; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
252; GFX7-NEXT:    v_mov_b32_e32 v0, s0
253; GFX7-NEXT:    v_mov_b32_e32 v1, s1
254; GFX7-NEXT:    flat_load_dword v2, v[0:1]
255; GFX7-NEXT:    v_mov_b32_e32 v0, s2
256; GFX7-NEXT:    v_mov_b32_e32 v1, s3
257; GFX7-NEXT:    s_waitcnt vmcnt(0)
258; GFX7-NEXT:    flat_store_dword v[0:1], v2
259; GFX7-NEXT:    s_endpgm
260;
261; GFX10-WGP-LABEL: global_singlethread_acquire_load:
262; GFX10-WGP:       ; %bb.0: ; %entry
263; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
264; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
265; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
266; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
267; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
268; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
269; GFX10-WGP-NEXT:    s_endpgm
270;
271; GFX10-CU-LABEL: global_singlethread_acquire_load:
272; GFX10-CU:       ; %bb.0: ; %entry
273; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
274; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
275; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
276; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
277; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
278; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
279; GFX10-CU-NEXT:    s_endpgm
280;
281; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_load:
282; SKIP-CACHE-INV:       ; %bb.0: ; %entry
283; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
284; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
285; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
286; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
287; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s0
288; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s1
289; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[4:7], 0
290; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s2
291; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s3
292; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
293; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
294; SKIP-CACHE-INV-NEXT:    s_endpgm
295;
296; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_load:
297; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
298; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
299; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
300; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
301; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
302; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
303; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
304; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
305;
306; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_load:
307; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
308; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
309; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
310; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
311; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
312; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
313; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
314; GFX90A-TGSPLIT-NEXT:    s_endpgm
315;
316; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acquire_load:
317; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
318; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
319; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
320; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
321; GFX940-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
322; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
323; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
324; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
325;
326; GFX940-TGSPLIT-LABEL: global_singlethread_acquire_load:
327; GFX940-TGSPLIT:       ; %bb.0: ; %entry
328; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
329; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
330; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
331; GFX940-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
332; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
333; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
334; GFX940-TGSPLIT-NEXT:    s_endpgm
335    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
336entry:
337  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread") acquire, align 4
338  store i32 %val, i32 addrspace(1)* %out
339  ret void
340}
341
342define amdgpu_kernel void @global_singlethread_seq_cst_load(
343; GFX6-LABEL: global_singlethread_seq_cst_load:
344; GFX6:       ; %bb.0: ; %entry
345; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
346; GFX6-NEXT:    s_mov_b32 s7, 0x100f000
347; GFX6-NEXT:    s_mov_b32 s6, -1
348; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
349; GFX6-NEXT:    s_mov_b32 s4, s0
350; GFX6-NEXT:    s_mov_b32 s5, s1
351; GFX6-NEXT:    buffer_load_dword v0, off, s[4:7], 0
352; GFX6-NEXT:    s_mov_b32 s4, s2
353; GFX6-NEXT:    s_mov_b32 s5, s3
354; GFX6-NEXT:    s_waitcnt vmcnt(0)
355; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
356; GFX6-NEXT:    s_endpgm
357;
358; GFX7-LABEL: global_singlethread_seq_cst_load:
359; GFX7:       ; %bb.0: ; %entry
360; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
361; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
362; GFX7-NEXT:    v_mov_b32_e32 v0, s0
363; GFX7-NEXT:    v_mov_b32_e32 v1, s1
364; GFX7-NEXT:    flat_load_dword v2, v[0:1]
365; GFX7-NEXT:    v_mov_b32_e32 v0, s2
366; GFX7-NEXT:    v_mov_b32_e32 v1, s3
367; GFX7-NEXT:    s_waitcnt vmcnt(0)
368; GFX7-NEXT:    flat_store_dword v[0:1], v2
369; GFX7-NEXT:    s_endpgm
370;
371; GFX10-WGP-LABEL: global_singlethread_seq_cst_load:
372; GFX10-WGP:       ; %bb.0: ; %entry
373; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
374; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
375; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
376; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
377; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
378; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
379; GFX10-WGP-NEXT:    s_endpgm
380;
381; GFX10-CU-LABEL: global_singlethread_seq_cst_load:
382; GFX10-CU:       ; %bb.0: ; %entry
383; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
384; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
385; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
386; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
387; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
388; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
389; GFX10-CU-NEXT:    s_endpgm
390;
391; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_load:
392; SKIP-CACHE-INV:       ; %bb.0: ; %entry
393; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
394; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
395; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
396; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
397; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s0
398; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s1
399; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[4:7], 0
400; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s2
401; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s3
402; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
403; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
404; SKIP-CACHE-INV-NEXT:    s_endpgm
405;
406; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_load:
407; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
408; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
409; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
410; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
411; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
412; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
413; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
414; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
415;
416; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_load:
417; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
418; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
419; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
420; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
421; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
422; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
423; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
424; GFX90A-TGSPLIT-NEXT:    s_endpgm
425;
426; GFX940-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_load:
427; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
428; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
429; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
430; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
431; GFX940-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
432; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
433; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
434; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
435;
436; GFX940-TGSPLIT-LABEL: global_singlethread_seq_cst_load:
437; GFX940-TGSPLIT:       ; %bb.0: ; %entry
438; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
439; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
440; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
441; GFX940-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
442; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
443; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
444; GFX940-TGSPLIT-NEXT:    s_endpgm
445    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
446entry:
447  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread") seq_cst, align 4
448  store i32 %val, i32 addrspace(1)* %out
449  ret void
450}
451
452define amdgpu_kernel void @global_singlethread_unordered_store(
453; GFX6-LABEL: global_singlethread_unordered_store:
454; GFX6:       ; %bb.0: ; %entry
455; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
456; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
457; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
458; GFX6-NEXT:    s_mov_b32 s2, -1
459; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
460; GFX6-NEXT:    v_mov_b32_e32 v0, s6
461; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
462; GFX6-NEXT:    s_endpgm
463;
464; GFX7-LABEL: global_singlethread_unordered_store:
465; GFX7:       ; %bb.0: ; %entry
466; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
467; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
468; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
469; GFX7-NEXT:    v_mov_b32_e32 v0, s0
470; GFX7-NEXT:    v_mov_b32_e32 v1, s1
471; GFX7-NEXT:    v_mov_b32_e32 v2, s2
472; GFX7-NEXT:    flat_store_dword v[0:1], v2
473; GFX7-NEXT:    s_endpgm
474;
475; GFX10-WGP-LABEL: global_singlethread_unordered_store:
476; GFX10-WGP:       ; %bb.0: ; %entry
477; GFX10-WGP-NEXT:    s_clause 0x1
478; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
479; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
480; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
481; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
482; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
483; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
484; GFX10-WGP-NEXT:    s_endpgm
485;
486; GFX10-CU-LABEL: global_singlethread_unordered_store:
487; GFX10-CU:       ; %bb.0: ; %entry
488; GFX10-CU-NEXT:    s_clause 0x1
489; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
490; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
491; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
492; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
493; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
494; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
495; GFX10-CU-NEXT:    s_endpgm
496;
497; SKIP-CACHE-INV-LABEL: global_singlethread_unordered_store:
498; SKIP-CACHE-INV:       ; %bb.0: ; %entry
499; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x0
500; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
501; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
502; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
503; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
504; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
505; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
506; SKIP-CACHE-INV-NEXT:    s_endpgm
507;
508; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_unordered_store:
509; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
510; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
511; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
512; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
513; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
514; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
515; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
516; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
517;
518; GFX90A-TGSPLIT-LABEL: global_singlethread_unordered_store:
519; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
520; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
521; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
522; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
523; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
524; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
525; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
526; GFX90A-TGSPLIT-NEXT:    s_endpgm
527;
528; GFX940-NOTTGSPLIT-LABEL: global_singlethread_unordered_store:
529; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
530; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
531; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
532; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
533; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
534; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
535; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
536; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
537;
538; GFX940-TGSPLIT-LABEL: global_singlethread_unordered_store:
539; GFX940-TGSPLIT:       ; %bb.0: ; %entry
540; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
541; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
542; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
543; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
544; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
545; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
546; GFX940-TGSPLIT-NEXT:    s_endpgm
547    i32 %in, i32 addrspace(1)* %out) {
548entry:
549  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread") unordered, align 4
550  ret void
551}
552
553define amdgpu_kernel void @global_singlethread_monotonic_store(
554; GFX6-LABEL: global_singlethread_monotonic_store:
555; GFX6:       ; %bb.0: ; %entry
556; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
557; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
558; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
559; GFX6-NEXT:    s_mov_b32 s2, -1
560; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
561; GFX6-NEXT:    v_mov_b32_e32 v0, s6
562; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
563; GFX6-NEXT:    s_endpgm
564;
565; GFX7-LABEL: global_singlethread_monotonic_store:
566; GFX7:       ; %bb.0: ; %entry
567; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
568; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
569; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
570; GFX7-NEXT:    v_mov_b32_e32 v0, s0
571; GFX7-NEXT:    v_mov_b32_e32 v1, s1
572; GFX7-NEXT:    v_mov_b32_e32 v2, s2
573; GFX7-NEXT:    flat_store_dword v[0:1], v2
574; GFX7-NEXT:    s_endpgm
575;
576; GFX10-WGP-LABEL: global_singlethread_monotonic_store:
577; GFX10-WGP:       ; %bb.0: ; %entry
578; GFX10-WGP-NEXT:    s_clause 0x1
579; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
580; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
581; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
582; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
583; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
584; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
585; GFX10-WGP-NEXT:    s_endpgm
586;
587; GFX10-CU-LABEL: global_singlethread_monotonic_store:
588; GFX10-CU:       ; %bb.0: ; %entry
589; GFX10-CU-NEXT:    s_clause 0x1
590; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
591; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
592; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
593; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
594; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
595; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
596; GFX10-CU-NEXT:    s_endpgm
597;
598; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_store:
599; SKIP-CACHE-INV:       ; %bb.0: ; %entry
600; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x0
601; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
602; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
603; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
604; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
605; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
606; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
607; SKIP-CACHE-INV-NEXT:    s_endpgm
608;
609; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_store:
610; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
611; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
612; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
613; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
614; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
615; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
616; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
617; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
618;
619; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_store:
620; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
621; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
622; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
623; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
624; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
625; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
626; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
627; GFX90A-TGSPLIT-NEXT:    s_endpgm
628;
629; GFX940-NOTTGSPLIT-LABEL: global_singlethread_monotonic_store:
630; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
631; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
632; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
633; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
634; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
635; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
636; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
637; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
638;
639; GFX940-TGSPLIT-LABEL: global_singlethread_monotonic_store:
640; GFX940-TGSPLIT:       ; %bb.0: ; %entry
641; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
642; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
643; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
644; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
645; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
646; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
647; GFX940-TGSPLIT-NEXT:    s_endpgm
648    i32 %in, i32 addrspace(1)* %out) {
649entry:
650  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread") monotonic, align 4
651  ret void
652}
653
654define amdgpu_kernel void @global_singlethread_release_store(
655; GFX6-LABEL: global_singlethread_release_store:
656; GFX6:       ; %bb.0: ; %entry
657; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
658; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
659; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
660; GFX6-NEXT:    s_mov_b32 s2, -1
661; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
662; GFX6-NEXT:    v_mov_b32_e32 v0, s6
663; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
664; GFX6-NEXT:    s_endpgm
665;
666; GFX7-LABEL: global_singlethread_release_store:
667; GFX7:       ; %bb.0: ; %entry
668; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
669; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
670; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
671; GFX7-NEXT:    v_mov_b32_e32 v0, s0
672; GFX7-NEXT:    v_mov_b32_e32 v1, s1
673; GFX7-NEXT:    v_mov_b32_e32 v2, s2
674; GFX7-NEXT:    flat_store_dword v[0:1], v2
675; GFX7-NEXT:    s_endpgm
676;
677; GFX10-WGP-LABEL: global_singlethread_release_store:
678; GFX10-WGP:       ; %bb.0: ; %entry
679; GFX10-WGP-NEXT:    s_clause 0x1
680; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
681; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
682; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
683; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
684; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
685; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
686; GFX10-WGP-NEXT:    s_endpgm
687;
688; GFX10-CU-LABEL: global_singlethread_release_store:
689; GFX10-CU:       ; %bb.0: ; %entry
690; GFX10-CU-NEXT:    s_clause 0x1
691; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
692; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
693; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
694; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
695; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
696; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
697; GFX10-CU-NEXT:    s_endpgm
698;
699; SKIP-CACHE-INV-LABEL: global_singlethread_release_store:
700; SKIP-CACHE-INV:       ; %bb.0: ; %entry
701; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x0
702; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
703; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
704; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
705; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
706; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
707; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
708; SKIP-CACHE-INV-NEXT:    s_endpgm
709;
710; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_release_store:
711; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
712; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
713; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
714; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
715; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
716; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
717; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
718; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
719;
720; GFX90A-TGSPLIT-LABEL: global_singlethread_release_store:
721; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
722; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
723; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
724; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
725; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
726; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
727; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
728; GFX90A-TGSPLIT-NEXT:    s_endpgm
729;
730; GFX940-NOTTGSPLIT-LABEL: global_singlethread_release_store:
731; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
732; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
733; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
734; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
735; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
736; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
737; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
738; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
739;
740; GFX940-TGSPLIT-LABEL: global_singlethread_release_store:
741; GFX940-TGSPLIT:       ; %bb.0: ; %entry
742; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
743; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
744; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
745; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
746; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
747; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
748; GFX940-TGSPLIT-NEXT:    s_endpgm
749    i32 %in, i32 addrspace(1)* %out) {
750entry:
751  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread") release, align 4
752  ret void
753}
754
755define amdgpu_kernel void @global_singlethread_seq_cst_store(
756; GFX6-LABEL: global_singlethread_seq_cst_store:
757; GFX6:       ; %bb.0: ; %entry
758; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
759; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
760; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
761; GFX6-NEXT:    s_mov_b32 s2, -1
762; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
763; GFX6-NEXT:    v_mov_b32_e32 v0, s6
764; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
765; GFX6-NEXT:    s_endpgm
766;
767; GFX7-LABEL: global_singlethread_seq_cst_store:
768; GFX7:       ; %bb.0: ; %entry
769; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
770; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
771; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
772; GFX7-NEXT:    v_mov_b32_e32 v0, s0
773; GFX7-NEXT:    v_mov_b32_e32 v1, s1
774; GFX7-NEXT:    v_mov_b32_e32 v2, s2
775; GFX7-NEXT:    flat_store_dword v[0:1], v2
776; GFX7-NEXT:    s_endpgm
777;
778; GFX10-WGP-LABEL: global_singlethread_seq_cst_store:
779; GFX10-WGP:       ; %bb.0: ; %entry
780; GFX10-WGP-NEXT:    s_clause 0x1
781; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
782; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
783; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
784; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
785; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
786; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
787; GFX10-WGP-NEXT:    s_endpgm
788;
789; GFX10-CU-LABEL: global_singlethread_seq_cst_store:
790; GFX10-CU:       ; %bb.0: ; %entry
791; GFX10-CU-NEXT:    s_clause 0x1
792; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
793; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
794; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
795; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
796; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
797; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
798; GFX10-CU-NEXT:    s_endpgm
799;
800; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_store:
801; SKIP-CACHE-INV:       ; %bb.0: ; %entry
802; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x0
803; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
804; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
805; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
806; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
807; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
808; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
809; SKIP-CACHE-INV-NEXT:    s_endpgm
810;
811; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_store:
812; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
813; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
814; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
815; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
816; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
817; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
818; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
819; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
820;
821; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_store:
822; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
823; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
824; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
825; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
826; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
827; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
828; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
829; GFX90A-TGSPLIT-NEXT:    s_endpgm
830;
831; GFX940-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_store:
832; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
833; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
834; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
835; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
836; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
837; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
838; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
839; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
840;
841; GFX940-TGSPLIT-LABEL: global_singlethread_seq_cst_store:
842; GFX940-TGSPLIT:       ; %bb.0: ; %entry
843; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
844; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
845; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
846; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
847; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
848; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
849; GFX940-TGSPLIT-NEXT:    s_endpgm
850    i32 %in, i32 addrspace(1)* %out) {
851entry:
852  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread") seq_cst, align 4
853  ret void
854}
855
856define amdgpu_kernel void @global_singlethread_monotonic_atomicrmw(
857; GFX6-LABEL: global_singlethread_monotonic_atomicrmw:
858; GFX6:       ; %bb.0: ; %entry
859; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
860; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
861; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
862; GFX6-NEXT:    s_mov_b32 s2, -1
863; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
864; GFX6-NEXT:    v_mov_b32_e32 v0, s6
865; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
866; GFX6-NEXT:    s_endpgm
867;
868; GFX7-LABEL: global_singlethread_monotonic_atomicrmw:
869; GFX7:       ; %bb.0: ; %entry
870; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
871; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
872; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
873; GFX7-NEXT:    v_mov_b32_e32 v0, s0
874; GFX7-NEXT:    v_mov_b32_e32 v1, s1
875; GFX7-NEXT:    v_mov_b32_e32 v2, s2
876; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
877; GFX7-NEXT:    s_endpgm
878;
879; GFX10-WGP-LABEL: global_singlethread_monotonic_atomicrmw:
880; GFX10-WGP:       ; %bb.0: ; %entry
881; GFX10-WGP-NEXT:    s_clause 0x1
882; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
883; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
884; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
885; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
886; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
887; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
888; GFX10-WGP-NEXT:    s_endpgm
889;
890; GFX10-CU-LABEL: global_singlethread_monotonic_atomicrmw:
891; GFX10-CU:       ; %bb.0: ; %entry
892; GFX10-CU-NEXT:    s_clause 0x1
893; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
894; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
895; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
896; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
897; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
898; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
899; GFX10-CU-NEXT:    s_endpgm
900;
901; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_atomicrmw:
902; SKIP-CACHE-INV:       ; %bb.0: ; %entry
903; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
904; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
905; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
906; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
907; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
908; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
909; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
910; SKIP-CACHE-INV-NEXT:    s_endpgm
911;
912; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_atomicrmw:
913; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
914; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
915; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
916; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
917; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
918; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
919; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
920; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
921;
922; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_atomicrmw:
923; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
924; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
925; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
926; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
927; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
928; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
929; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
930; GFX90A-TGSPLIT-NEXT:    s_endpgm
931;
932; GFX940-NOTTGSPLIT-LABEL: global_singlethread_monotonic_atomicrmw:
933; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
934; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
935; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
936; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
937; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
938; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
939; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
940; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
941;
942; GFX940-TGSPLIT-LABEL: global_singlethread_monotonic_atomicrmw:
943; GFX940-TGSPLIT:       ; %bb.0: ; %entry
944; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
945; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
946; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
947; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
948; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
949; GFX940-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
950; GFX940-TGSPLIT-NEXT:    s_endpgm
951    i32 addrspace(1)* %out, i32 %in) {
952entry:
953  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") monotonic
954  ret void
955}
956
957define amdgpu_kernel void @global_singlethread_acquire_atomicrmw(
958; GFX6-LABEL: global_singlethread_acquire_atomicrmw:
959; GFX6:       ; %bb.0: ; %entry
960; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
961; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
962; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
963; GFX6-NEXT:    s_mov_b32 s2, -1
964; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
965; GFX6-NEXT:    v_mov_b32_e32 v0, s6
966; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
967; GFX6-NEXT:    s_endpgm
968;
969; GFX7-LABEL: global_singlethread_acquire_atomicrmw:
970; GFX7:       ; %bb.0: ; %entry
971; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
972; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
973; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
974; GFX7-NEXT:    v_mov_b32_e32 v0, s0
975; GFX7-NEXT:    v_mov_b32_e32 v1, s1
976; GFX7-NEXT:    v_mov_b32_e32 v2, s2
977; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
978; GFX7-NEXT:    s_endpgm
979;
980; GFX10-WGP-LABEL: global_singlethread_acquire_atomicrmw:
981; GFX10-WGP:       ; %bb.0: ; %entry
982; GFX10-WGP-NEXT:    s_clause 0x1
983; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
984; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
985; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
986; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
987; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
988; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
989; GFX10-WGP-NEXT:    s_endpgm
990;
991; GFX10-CU-LABEL: global_singlethread_acquire_atomicrmw:
992; GFX10-CU:       ; %bb.0: ; %entry
993; GFX10-CU-NEXT:    s_clause 0x1
994; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
995; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
996; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
997; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
998; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
999; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
1000; GFX10-CU-NEXT:    s_endpgm
1001;
1002; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_atomicrmw:
1003; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1004; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
1005; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1006; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
1007; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
1008; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1009; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
1010; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
1011; SKIP-CACHE-INV-NEXT:    s_endpgm
1012;
1013; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_atomicrmw:
1014; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1015; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1016; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1017; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1018; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1019; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1020; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1021; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1022;
1023; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_atomicrmw:
1024; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1025; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1026; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1027; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1028; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1029; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1030; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1031; GFX90A-TGSPLIT-NEXT:    s_endpgm
1032;
1033; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acquire_atomicrmw:
1034; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1035; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1036; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1037; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1038; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1039; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1040; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
1041; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1042;
1043; GFX940-TGSPLIT-LABEL: global_singlethread_acquire_atomicrmw:
1044; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1045; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1046; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1047; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1048; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1049; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1050; GFX940-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
1051; GFX940-TGSPLIT-NEXT:    s_endpgm
1052    i32 addrspace(1)* %out, i32 %in) {
1053entry:
1054  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") acquire
1055  ret void
1056}
1057
1058define amdgpu_kernel void @global_singlethread_release_atomicrmw(
1059; GFX6-LABEL: global_singlethread_release_atomicrmw:
1060; GFX6:       ; %bb.0: ; %entry
1061; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
1062; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1063; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1064; GFX6-NEXT:    s_mov_b32 s2, -1
1065; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1066; GFX6-NEXT:    v_mov_b32_e32 v0, s6
1067; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
1068; GFX6-NEXT:    s_endpgm
1069;
1070; GFX7-LABEL: global_singlethread_release_atomicrmw:
1071; GFX7:       ; %bb.0: ; %entry
1072; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1073; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1074; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1075; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1076; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1077; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1078; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1079; GFX7-NEXT:    s_endpgm
1080;
1081; GFX10-WGP-LABEL: global_singlethread_release_atomicrmw:
1082; GFX10-WGP:       ; %bb.0: ; %entry
1083; GFX10-WGP-NEXT:    s_clause 0x1
1084; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1085; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1086; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
1087; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1088; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1089; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
1090; GFX10-WGP-NEXT:    s_endpgm
1091;
1092; GFX10-CU-LABEL: global_singlethread_release_atomicrmw:
1093; GFX10-CU:       ; %bb.0: ; %entry
1094; GFX10-CU-NEXT:    s_clause 0x1
1095; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1096; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1097; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
1098; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1099; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1100; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
1101; GFX10-CU-NEXT:    s_endpgm
1102;
1103; SKIP-CACHE-INV-LABEL: global_singlethread_release_atomicrmw:
1104; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1105; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
1106; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1107; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
1108; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
1109; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1110; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
1111; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
1112; SKIP-CACHE-INV-NEXT:    s_endpgm
1113;
1114; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_release_atomicrmw:
1115; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1116; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1117; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1118; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1119; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1120; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1121; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1122; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1123;
1124; GFX90A-TGSPLIT-LABEL: global_singlethread_release_atomicrmw:
1125; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1126; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1127; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1128; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1129; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1130; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1131; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1132; GFX90A-TGSPLIT-NEXT:    s_endpgm
1133;
1134; GFX940-NOTTGSPLIT-LABEL: global_singlethread_release_atomicrmw:
1135; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1136; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1137; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1138; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1139; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1140; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1141; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
1142; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1143;
1144; GFX940-TGSPLIT-LABEL: global_singlethread_release_atomicrmw:
1145; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1146; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1147; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1148; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1149; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1150; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1151; GFX940-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
1152; GFX940-TGSPLIT-NEXT:    s_endpgm
1153    i32 addrspace(1)* %out, i32 %in) {
1154entry:
1155  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") release
1156  ret void
1157}
1158
1159define amdgpu_kernel void @global_singlethread_acq_rel_atomicrmw(
1160; GFX6-LABEL: global_singlethread_acq_rel_atomicrmw:
1161; GFX6:       ; %bb.0: ; %entry
1162; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
1163; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1164; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1165; GFX6-NEXT:    s_mov_b32 s2, -1
1166; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1167; GFX6-NEXT:    v_mov_b32_e32 v0, s6
1168; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
1169; GFX6-NEXT:    s_endpgm
1170;
1171; GFX7-LABEL: global_singlethread_acq_rel_atomicrmw:
1172; GFX7:       ; %bb.0: ; %entry
1173; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1174; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1175; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1176; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1177; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1178; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1179; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1180; GFX7-NEXT:    s_endpgm
1181;
1182; GFX10-WGP-LABEL: global_singlethread_acq_rel_atomicrmw:
1183; GFX10-WGP:       ; %bb.0: ; %entry
1184; GFX10-WGP-NEXT:    s_clause 0x1
1185; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1186; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1187; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
1188; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1189; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1190; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
1191; GFX10-WGP-NEXT:    s_endpgm
1192;
1193; GFX10-CU-LABEL: global_singlethread_acq_rel_atomicrmw:
1194; GFX10-CU:       ; %bb.0: ; %entry
1195; GFX10-CU-NEXT:    s_clause 0x1
1196; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1197; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1198; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
1199; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1200; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1201; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
1202; GFX10-CU-NEXT:    s_endpgm
1203;
1204; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_atomicrmw:
1205; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1206; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
1207; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1208; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
1209; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
1210; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1211; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
1212; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
1213; SKIP-CACHE-INV-NEXT:    s_endpgm
1214;
1215; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_atomicrmw:
1216; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1217; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1218; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1219; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1220; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1221; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1222; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1223; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1224;
1225; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_atomicrmw:
1226; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1227; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1228; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1229; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1230; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1231; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1232; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1233; GFX90A-TGSPLIT-NEXT:    s_endpgm
1234;
1235; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_atomicrmw:
1236; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1237; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1238; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1239; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1240; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1241; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1242; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
1243; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1244;
1245; GFX940-TGSPLIT-LABEL: global_singlethread_acq_rel_atomicrmw:
1246; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1247; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1248; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1249; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1250; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1251; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1252; GFX940-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
1253; GFX940-TGSPLIT-NEXT:    s_endpgm
1254    i32 addrspace(1)* %out, i32 %in) {
1255entry:
1256  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") acq_rel
1257  ret void
1258}
1259
1260define amdgpu_kernel void @global_singlethread_seq_cst_atomicrmw(
1261; GFX6-LABEL: global_singlethread_seq_cst_atomicrmw:
1262; GFX6:       ; %bb.0: ; %entry
1263; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
1264; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1265; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1266; GFX6-NEXT:    s_mov_b32 s2, -1
1267; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1268; GFX6-NEXT:    v_mov_b32_e32 v0, s6
1269; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
1270; GFX6-NEXT:    s_endpgm
1271;
1272; GFX7-LABEL: global_singlethread_seq_cst_atomicrmw:
1273; GFX7:       ; %bb.0: ; %entry
1274; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1275; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1276; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1277; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1278; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1279; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1280; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1281; GFX7-NEXT:    s_endpgm
1282;
1283; GFX10-WGP-LABEL: global_singlethread_seq_cst_atomicrmw:
1284; GFX10-WGP:       ; %bb.0: ; %entry
1285; GFX10-WGP-NEXT:    s_clause 0x1
1286; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1287; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1288; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
1289; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1290; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1291; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
1292; GFX10-WGP-NEXT:    s_endpgm
1293;
1294; GFX10-CU-LABEL: global_singlethread_seq_cst_atomicrmw:
1295; GFX10-CU:       ; %bb.0: ; %entry
1296; GFX10-CU-NEXT:    s_clause 0x1
1297; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1298; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1299; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
1300; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1301; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1302; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
1303; GFX10-CU-NEXT:    s_endpgm
1304;
1305; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_atomicrmw:
1306; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1307; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
1308; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1309; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
1310; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
1311; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1312; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
1313; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
1314; SKIP-CACHE-INV-NEXT:    s_endpgm
1315;
1316; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_atomicrmw:
1317; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1318; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1319; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1320; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1321; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1322; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1323; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1324; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1325;
1326; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_atomicrmw:
1327; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1328; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1329; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1330; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1331; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1332; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1333; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1334; GFX90A-TGSPLIT-NEXT:    s_endpgm
1335;
1336; GFX940-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_atomicrmw:
1337; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1338; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1339; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1340; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1341; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1342; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1343; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
1344; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1345;
1346; GFX940-TGSPLIT-LABEL: global_singlethread_seq_cst_atomicrmw:
1347; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1348; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1349; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1350; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1351; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1352; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1353; GFX940-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
1354; GFX940-TGSPLIT-NEXT:    s_endpgm
1355    i32 addrspace(1)* %out, i32 %in) {
1356entry:
1357  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") seq_cst
1358  ret void
1359}
1360
1361define amdgpu_kernel void @global_singlethread_acquire_ret_atomicrmw(
1362; GFX6-LABEL: global_singlethread_acquire_ret_atomicrmw:
1363; GFX6:       ; %bb.0: ; %entry
1364; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
1365; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1366; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1367; GFX6-NEXT:    s_mov_b32 s2, -1
1368; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1369; GFX6-NEXT:    v_mov_b32_e32 v0, s6
1370; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
1371; GFX6-NEXT:    s_waitcnt vmcnt(0)
1372; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1373; GFX6-NEXT:    s_endpgm
1374;
1375; GFX7-LABEL: global_singlethread_acquire_ret_atomicrmw:
1376; GFX7:       ; %bb.0: ; %entry
1377; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1378; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1379; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1380; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1381; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1382; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1383; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1384; GFX7-NEXT:    s_waitcnt vmcnt(0)
1385; GFX7-NEXT:    flat_store_dword v[0:1], v2
1386; GFX7-NEXT:    s_endpgm
1387;
1388; GFX10-WGP-LABEL: global_singlethread_acquire_ret_atomicrmw:
1389; GFX10-WGP:       ; %bb.0: ; %entry
1390; GFX10-WGP-NEXT:    s_clause 0x1
1391; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1392; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1393; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
1394; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1395; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1396; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1397; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
1398; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
1399; GFX10-WGP-NEXT:    s_endpgm
1400;
1401; GFX10-CU-LABEL: global_singlethread_acquire_ret_atomicrmw:
1402; GFX10-CU:       ; %bb.0: ; %entry
1403; GFX10-CU-NEXT:    s_clause 0x1
1404; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1405; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1406; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
1407; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1408; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1409; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1410; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
1411; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
1412; GFX10-CU-NEXT:    s_endpgm
1413;
1414; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_ret_atomicrmw:
1415; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1416; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
1417; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1418; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
1419; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
1420; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1421; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
1422; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
1423; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
1424; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1425; SKIP-CACHE-INV-NEXT:    s_endpgm
1426;
1427; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_ret_atomicrmw:
1428; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1429; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1430; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1431; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1432; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1433; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1434; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1435; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1436; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
1437; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1438;
1439; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_ret_atomicrmw:
1440; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1441; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1442; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1443; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1444; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1445; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1446; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1447; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1448; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
1449; GFX90A-TGSPLIT-NEXT:    s_endpgm
1450;
1451; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acquire_ret_atomicrmw:
1452; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1453; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1454; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1455; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1456; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1457; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1458; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[2:3] sc0
1459; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1460; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
1461; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1462;
1463; GFX940-TGSPLIT-LABEL: global_singlethread_acquire_ret_atomicrmw:
1464; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1465; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1466; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1467; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1468; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1469; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1470; GFX940-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[2:3] sc0
1471; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1472; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
1473; GFX940-TGSPLIT-NEXT:    s_endpgm
1474    i32 addrspace(1)* %out, i32 %in) {
1475entry:
1476  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") acquire
1477  store i32 %val, i32 addrspace(1)* %out, align 4
1478  ret void
1479}
1480
1481define amdgpu_kernel void @global_singlethread_acq_rel_ret_atomicrmw(
1482; GFX6-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1483; GFX6:       ; %bb.0: ; %entry
1484; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
1485; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1486; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1487; GFX6-NEXT:    s_mov_b32 s2, -1
1488; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1489; GFX6-NEXT:    v_mov_b32_e32 v0, s6
1490; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
1491; GFX6-NEXT:    s_waitcnt vmcnt(0)
1492; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1493; GFX6-NEXT:    s_endpgm
1494;
1495; GFX7-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1496; GFX7:       ; %bb.0: ; %entry
1497; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1498; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1499; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1500; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1501; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1502; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1503; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1504; GFX7-NEXT:    s_waitcnt vmcnt(0)
1505; GFX7-NEXT:    flat_store_dword v[0:1], v2
1506; GFX7-NEXT:    s_endpgm
1507;
1508; GFX10-WGP-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1509; GFX10-WGP:       ; %bb.0: ; %entry
1510; GFX10-WGP-NEXT:    s_clause 0x1
1511; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1512; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1513; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
1514; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1515; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1516; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1517; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
1518; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
1519; GFX10-WGP-NEXT:    s_endpgm
1520;
1521; GFX10-CU-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1522; GFX10-CU:       ; %bb.0: ; %entry
1523; GFX10-CU-NEXT:    s_clause 0x1
1524; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1525; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1526; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
1527; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1528; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1529; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1530; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
1531; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
1532; GFX10-CU-NEXT:    s_endpgm
1533;
1534; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1535; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1536; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
1537; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1538; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
1539; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
1540; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1541; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
1542; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
1543; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
1544; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1545; SKIP-CACHE-INV-NEXT:    s_endpgm
1546;
1547; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1548; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1549; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1550; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1551; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1552; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1553; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1554; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1555; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1556; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
1557; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1558;
1559; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1560; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1561; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1562; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1563; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1564; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1565; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1566; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1567; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1568; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
1569; GFX90A-TGSPLIT-NEXT:    s_endpgm
1570;
1571; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1572; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1573; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1574; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1575; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1576; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1577; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1578; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[2:3] sc0
1579; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1580; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
1581; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1582;
1583; GFX940-TGSPLIT-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1584; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1585; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1586; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1587; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1588; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1589; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1590; GFX940-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[2:3] sc0
1591; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1592; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
1593; GFX940-TGSPLIT-NEXT:    s_endpgm
1594    i32 addrspace(1)* %out, i32 %in) {
1595entry:
1596  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") acq_rel
1597  store i32 %val, i32 addrspace(1)* %out, align 4
1598  ret void
1599}
1600
1601define amdgpu_kernel void @global_singlethread_seq_cst_ret_atomicrmw(
1602; GFX6-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
1603; GFX6:       ; %bb.0: ; %entry
1604; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
1605; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1606; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1607; GFX6-NEXT:    s_mov_b32 s2, -1
1608; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1609; GFX6-NEXT:    v_mov_b32_e32 v0, s6
1610; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
1611; GFX6-NEXT:    s_waitcnt vmcnt(0)
1612; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1613; GFX6-NEXT:    s_endpgm
1614;
1615; GFX7-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
1616; GFX7:       ; %bb.0: ; %entry
1617; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1618; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1619; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1620; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1621; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1622; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1623; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1624; GFX7-NEXT:    s_waitcnt vmcnt(0)
1625; GFX7-NEXT:    flat_store_dword v[0:1], v2
1626; GFX7-NEXT:    s_endpgm
1627;
1628; GFX10-WGP-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
1629; GFX10-WGP:       ; %bb.0: ; %entry
1630; GFX10-WGP-NEXT:    s_clause 0x1
1631; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1632; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1633; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
1634; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1635; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1636; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1637; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
1638; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
1639; GFX10-WGP-NEXT:    s_endpgm
1640;
1641; GFX10-CU-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
1642; GFX10-CU:       ; %bb.0: ; %entry
1643; GFX10-CU-NEXT:    s_clause 0x1
1644; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1645; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1646; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
1647; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1648; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1649; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1650; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
1651; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
1652; GFX10-CU-NEXT:    s_endpgm
1653;
1654; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
1655; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1656; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
1657; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1658; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
1659; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
1660; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1661; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
1662; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
1663; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
1664; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1665; SKIP-CACHE-INV-NEXT:    s_endpgm
1666;
1667; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
1668; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1669; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1670; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1671; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1672; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1673; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1674; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1675; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1676; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
1677; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1678;
1679; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
1680; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1681; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1682; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1683; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1684; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1685; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1686; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1687; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1688; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
1689; GFX90A-TGSPLIT-NEXT:    s_endpgm
1690;
1691; GFX940-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
1692; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1693; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1694; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1695; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1696; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1697; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1698; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[2:3] sc0
1699; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1700; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
1701; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1702;
1703; GFX940-TGSPLIT-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
1704; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1705; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1706; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1707; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1708; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1709; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1710; GFX940-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[2:3] sc0
1711; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1712; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
1713; GFX940-TGSPLIT-NEXT:    s_endpgm
1714    i32 addrspace(1)* %out, i32 %in) {
1715entry:
1716  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") seq_cst
1717  store i32 %val, i32 addrspace(1)* %out, align 4
1718  ret void
1719}
1720
1721define amdgpu_kernel void @global_singlethread_monotonic_monotonic_cmpxchg(
1722; GFX6-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
1723; GFX6:       ; %bb.0: ; %entry
1724; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
1725; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1726; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1727; GFX6-NEXT:    s_mov_b32 s2, -1
1728; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1729; GFX6-NEXT:    v_mov_b32_e32 v0, s6
1730; GFX6-NEXT:    v_mov_b32_e32 v1, s7
1731; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
1732; GFX6-NEXT:    s_endpgm
1733;
1734; GFX7-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
1735; GFX7:       ; %bb.0: ; %entry
1736; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1737; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1738; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1739; GFX7-NEXT:    s_add_u32 s0, s0, 16
1740; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1741; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1742; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1743; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1744; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1745; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1746; GFX7-NEXT:    s_endpgm
1747;
1748; GFX10-WGP-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
1749; GFX10-WGP:       ; %bb.0: ; %entry
1750; GFX10-WGP-NEXT:    s_clause 0x1
1751; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1752; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1753; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
1754; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1755; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1756; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1757; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1758; GFX10-WGP-NEXT:    s_endpgm
1759;
1760; GFX10-CU-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
1761; GFX10-CU:       ; %bb.0: ; %entry
1762; GFX10-CU-NEXT:    s_clause 0x1
1763; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1764; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1765; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
1766; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1767; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1768; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1769; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1770; GFX10-CU-NEXT:    s_endpgm
1771;
1772; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
1773; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1774; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
1775; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1776; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
1777; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
1778; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1779; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
1780; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
1781; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
1782; SKIP-CACHE-INV-NEXT:    s_endpgm
1783;
1784; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
1785; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1786; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1787; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1788; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1789; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1790; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1791; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1792; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1793;
1794; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
1795; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1796; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1797; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1798; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1799; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1800; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1801; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1802; GFX90A-TGSPLIT-NEXT:    s_endpgm
1803;
1804; GFX940-NOTTGSPLIT-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
1805; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1806; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
1807; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
1808; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1809; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1810; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1811; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
1812; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1813;
1814; GFX940-TGSPLIT-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
1815; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1816; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
1817; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
1818; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1819; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1820; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1821; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
1822; GFX940-TGSPLIT-NEXT:    s_endpgm
1823    i32 addrspace(1)* %out, i32 %in, i32 %old) {
1824entry:
1825  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
1826  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic monotonic
1827  ret void
1828}
1829
1830define amdgpu_kernel void @global_singlethread_acquire_monotonic_cmpxchg(
1831; GFX6-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
1832; GFX6:       ; %bb.0: ; %entry
1833; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
1834; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1835; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1836; GFX6-NEXT:    s_mov_b32 s2, -1
1837; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1838; GFX6-NEXT:    v_mov_b32_e32 v0, s6
1839; GFX6-NEXT:    v_mov_b32_e32 v1, s7
1840; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
1841; GFX6-NEXT:    s_endpgm
1842;
1843; GFX7-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
1844; GFX7:       ; %bb.0: ; %entry
1845; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1846; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1847; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1848; GFX7-NEXT:    s_add_u32 s0, s0, 16
1849; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1850; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1851; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1852; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1853; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1854; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1855; GFX7-NEXT:    s_endpgm
1856;
1857; GFX10-WGP-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
1858; GFX10-WGP:       ; %bb.0: ; %entry
1859; GFX10-WGP-NEXT:    s_clause 0x1
1860; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1861; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1862; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
1863; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1864; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1865; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1866; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1867; GFX10-WGP-NEXT:    s_endpgm
1868;
1869; GFX10-CU-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
1870; GFX10-CU:       ; %bb.0: ; %entry
1871; GFX10-CU-NEXT:    s_clause 0x1
1872; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1873; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1874; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
1875; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1876; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1877; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1878; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1879; GFX10-CU-NEXT:    s_endpgm
1880;
1881; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
1882; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1883; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
1884; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1885; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
1886; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
1887; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1888; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
1889; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
1890; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
1891; SKIP-CACHE-INV-NEXT:    s_endpgm
1892;
1893; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
1894; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1895; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1896; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1897; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1898; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1899; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1900; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1901; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1902;
1903; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
1904; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1905; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1906; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1907; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1908; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1909; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1910; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1911; GFX90A-TGSPLIT-NEXT:    s_endpgm
1912;
1913; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
1914; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1915; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
1916; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
1917; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1918; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1919; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1920; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
1921; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1922;
1923; GFX940-TGSPLIT-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
1924; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1925; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
1926; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
1927; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1928; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1929; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1930; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
1931; GFX940-TGSPLIT-NEXT:    s_endpgm
1932    i32 addrspace(1)* %out, i32 %in, i32 %old) {
1933entry:
1934  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
1935  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire monotonic
1936  ret void
1937}
1938
1939define amdgpu_kernel void @global_singlethread_release_monotonic_cmpxchg(
1940; GFX6-LABEL: global_singlethread_release_monotonic_cmpxchg:
1941; GFX6:       ; %bb.0: ; %entry
1942; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
1943; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1944; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1945; GFX6-NEXT:    s_mov_b32 s2, -1
1946; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1947; GFX6-NEXT:    v_mov_b32_e32 v0, s6
1948; GFX6-NEXT:    v_mov_b32_e32 v1, s7
1949; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
1950; GFX6-NEXT:    s_endpgm
1951;
1952; GFX7-LABEL: global_singlethread_release_monotonic_cmpxchg:
1953; GFX7:       ; %bb.0: ; %entry
1954; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1955; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1956; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1957; GFX7-NEXT:    s_add_u32 s0, s0, 16
1958; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1959; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1960; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1961; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1962; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1963; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1964; GFX7-NEXT:    s_endpgm
1965;
1966; GFX10-WGP-LABEL: global_singlethread_release_monotonic_cmpxchg:
1967; GFX10-WGP:       ; %bb.0: ; %entry
1968; GFX10-WGP-NEXT:    s_clause 0x1
1969; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1970; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1971; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
1972; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1973; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1974; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1975; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1976; GFX10-WGP-NEXT:    s_endpgm
1977;
1978; GFX10-CU-LABEL: global_singlethread_release_monotonic_cmpxchg:
1979; GFX10-CU:       ; %bb.0: ; %entry
1980; GFX10-CU-NEXT:    s_clause 0x1
1981; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1982; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1983; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
1984; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1985; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1986; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1987; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1988; GFX10-CU-NEXT:    s_endpgm
1989;
1990; SKIP-CACHE-INV-LABEL: global_singlethread_release_monotonic_cmpxchg:
1991; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1992; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
1993; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1994; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
1995; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
1996; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1997; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
1998; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
1999; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2000; SKIP-CACHE-INV-NEXT:    s_endpgm
2001;
2002; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_release_monotonic_cmpxchg:
2003; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2004; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2005; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2006; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2007; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2008; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2009; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2010; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2011;
2012; GFX90A-TGSPLIT-LABEL: global_singlethread_release_monotonic_cmpxchg:
2013; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2014; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2015; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2016; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2017; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2018; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2019; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2020; GFX90A-TGSPLIT-NEXT:    s_endpgm
2021;
2022; GFX940-NOTTGSPLIT-LABEL: global_singlethread_release_monotonic_cmpxchg:
2023; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2024; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2025; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2026; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2027; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2028; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2029; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2030; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2031;
2032; GFX940-TGSPLIT-LABEL: global_singlethread_release_monotonic_cmpxchg:
2033; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2034; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2035; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2036; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2037; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2038; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2039; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2040; GFX940-TGSPLIT-NEXT:    s_endpgm
2041    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2042entry:
2043  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2044  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") release monotonic
2045  ret void
2046}
2047
2048define amdgpu_kernel void @global_singlethread_acq_rel_monotonic_cmpxchg(
2049; GFX6-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
2050; GFX6:       ; %bb.0: ; %entry
2051; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
2052; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2053; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2054; GFX6-NEXT:    s_mov_b32 s2, -1
2055; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2056; GFX6-NEXT:    v_mov_b32_e32 v0, s6
2057; GFX6-NEXT:    v_mov_b32_e32 v1, s7
2058; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2059; GFX6-NEXT:    s_endpgm
2060;
2061; GFX7-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
2062; GFX7:       ; %bb.0: ; %entry
2063; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2064; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2065; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2066; GFX7-NEXT:    s_add_u32 s0, s0, 16
2067; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2068; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2069; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2070; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2071; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2072; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2073; GFX7-NEXT:    s_endpgm
2074;
2075; GFX10-WGP-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
2076; GFX10-WGP:       ; %bb.0: ; %entry
2077; GFX10-WGP-NEXT:    s_clause 0x1
2078; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2079; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2080; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2081; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2082; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2083; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2084; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2085; GFX10-WGP-NEXT:    s_endpgm
2086;
2087; GFX10-CU-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
2088; GFX10-CU:       ; %bb.0: ; %entry
2089; GFX10-CU-NEXT:    s_clause 0x1
2090; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2091; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2092; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2093; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2094; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2095; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2096; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2097; GFX10-CU-NEXT:    s_endpgm
2098;
2099; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
2100; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2101; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
2102; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
2103; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
2104; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
2105; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2106; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2107; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2108; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2109; SKIP-CACHE-INV-NEXT:    s_endpgm
2110;
2111; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
2112; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2113; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2114; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2115; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2116; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2117; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2118; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2119; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2120;
2121; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
2122; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2123; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2124; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2125; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2126; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2127; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2128; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2129; GFX90A-TGSPLIT-NEXT:    s_endpgm
2130;
2131; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
2132; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2133; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2134; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2135; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2136; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2137; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2138; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2139; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2140;
2141; GFX940-TGSPLIT-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
2142; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2143; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2144; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2145; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2146; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2147; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2148; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2149; GFX940-TGSPLIT-NEXT:    s_endpgm
2150    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2151entry:
2152  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2153  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel monotonic
2154  ret void
2155}
2156
2157define amdgpu_kernel void @global_singlethread_seq_cst_monotonic_cmpxchg(
2158; GFX6-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
2159; GFX6:       ; %bb.0: ; %entry
2160; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
2161; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2162; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2163; GFX6-NEXT:    s_mov_b32 s2, -1
2164; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2165; GFX6-NEXT:    v_mov_b32_e32 v0, s6
2166; GFX6-NEXT:    v_mov_b32_e32 v1, s7
2167; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2168; GFX6-NEXT:    s_endpgm
2169;
2170; GFX7-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
2171; GFX7:       ; %bb.0: ; %entry
2172; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2173; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2174; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2175; GFX7-NEXT:    s_add_u32 s0, s0, 16
2176; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2177; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2178; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2179; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2180; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2181; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2182; GFX7-NEXT:    s_endpgm
2183;
2184; GFX10-WGP-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
2185; GFX10-WGP:       ; %bb.0: ; %entry
2186; GFX10-WGP-NEXT:    s_clause 0x1
2187; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2188; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2189; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2190; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2191; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2192; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2193; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2194; GFX10-WGP-NEXT:    s_endpgm
2195;
2196; GFX10-CU-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
2197; GFX10-CU:       ; %bb.0: ; %entry
2198; GFX10-CU-NEXT:    s_clause 0x1
2199; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2200; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2201; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2202; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2203; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2204; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2205; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2206; GFX10-CU-NEXT:    s_endpgm
2207;
2208; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
2209; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2210; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
2211; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
2212; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
2213; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
2214; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2215; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2216; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2217; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2218; SKIP-CACHE-INV-NEXT:    s_endpgm
2219;
2220; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
2221; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2222; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2223; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2224; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2225; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2226; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2227; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2228; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2229;
2230; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
2231; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2232; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2233; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2234; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2235; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2236; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2237; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2238; GFX90A-TGSPLIT-NEXT:    s_endpgm
2239;
2240; GFX940-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
2241; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2242; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2243; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2244; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2245; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2246; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2247; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2248; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2249;
2250; GFX940-TGSPLIT-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
2251; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2252; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2253; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2254; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2255; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2256; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2257; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2258; GFX940-TGSPLIT-NEXT:    s_endpgm
2259    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2260entry:
2261  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2262  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst monotonic
2263  ret void
2264}
2265
2266define amdgpu_kernel void @global_singlethread_monotonic_acquire_cmpxchg(
2267; GFX6-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
2268; GFX6:       ; %bb.0: ; %entry
2269; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
2270; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2271; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2272; GFX6-NEXT:    s_mov_b32 s2, -1
2273; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2274; GFX6-NEXT:    v_mov_b32_e32 v0, s6
2275; GFX6-NEXT:    v_mov_b32_e32 v1, s7
2276; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2277; GFX6-NEXT:    s_endpgm
2278;
2279; GFX7-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
2280; GFX7:       ; %bb.0: ; %entry
2281; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2282; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2283; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2284; GFX7-NEXT:    s_add_u32 s0, s0, 16
2285; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2286; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2287; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2288; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2289; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2290; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2291; GFX7-NEXT:    s_endpgm
2292;
2293; GFX10-WGP-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
2294; GFX10-WGP:       ; %bb.0: ; %entry
2295; GFX10-WGP-NEXT:    s_clause 0x1
2296; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2297; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2298; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2299; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2300; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2301; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2302; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2303; GFX10-WGP-NEXT:    s_endpgm
2304;
2305; GFX10-CU-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
2306; GFX10-CU:       ; %bb.0: ; %entry
2307; GFX10-CU-NEXT:    s_clause 0x1
2308; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2309; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2310; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2311; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2312; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2313; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2314; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2315; GFX10-CU-NEXT:    s_endpgm
2316;
2317; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
2318; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2319; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
2320; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
2321; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
2322; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
2323; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2324; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2325; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2326; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2327; SKIP-CACHE-INV-NEXT:    s_endpgm
2328;
2329; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
2330; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2331; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2332; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2333; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2334; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2335; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2336; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2337; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2338;
2339; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
2340; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2341; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2342; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2343; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2344; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2345; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2346; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2347; GFX90A-TGSPLIT-NEXT:    s_endpgm
2348;
2349; GFX940-NOTTGSPLIT-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
2350; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2351; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2352; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2353; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2354; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2355; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2356; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2357; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2358;
2359; GFX940-TGSPLIT-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
2360; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2361; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2362; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2363; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2364; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2365; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2366; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2367; GFX940-TGSPLIT-NEXT:    s_endpgm
2368    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2369entry:
2370  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2371  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic acquire
2372  ret void
2373}
2374
2375define amdgpu_kernel void @global_singlethread_acquire_acquire_cmpxchg(
2376; GFX6-LABEL: global_singlethread_acquire_acquire_cmpxchg:
2377; GFX6:       ; %bb.0: ; %entry
2378; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
2379; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2380; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2381; GFX6-NEXT:    s_mov_b32 s2, -1
2382; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2383; GFX6-NEXT:    v_mov_b32_e32 v0, s6
2384; GFX6-NEXT:    v_mov_b32_e32 v1, s7
2385; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2386; GFX6-NEXT:    s_endpgm
2387;
2388; GFX7-LABEL: global_singlethread_acquire_acquire_cmpxchg:
2389; GFX7:       ; %bb.0: ; %entry
2390; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2391; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2392; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2393; GFX7-NEXT:    s_add_u32 s0, s0, 16
2394; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2395; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2396; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2397; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2398; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2399; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2400; GFX7-NEXT:    s_endpgm
2401;
2402; GFX10-WGP-LABEL: global_singlethread_acquire_acquire_cmpxchg:
2403; GFX10-WGP:       ; %bb.0: ; %entry
2404; GFX10-WGP-NEXT:    s_clause 0x1
2405; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2406; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2407; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2408; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2409; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2410; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2411; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2412; GFX10-WGP-NEXT:    s_endpgm
2413;
2414; GFX10-CU-LABEL: global_singlethread_acquire_acquire_cmpxchg:
2415; GFX10-CU:       ; %bb.0: ; %entry
2416; GFX10-CU-NEXT:    s_clause 0x1
2417; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2418; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2419; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2420; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2421; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2422; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2423; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2424; GFX10-CU-NEXT:    s_endpgm
2425;
2426; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_acquire_cmpxchg:
2427; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2428; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
2429; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
2430; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
2431; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
2432; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2433; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2434; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2435; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2436; SKIP-CACHE-INV-NEXT:    s_endpgm
2437;
2438; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_acquire_cmpxchg:
2439; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2440; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2441; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2442; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2443; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2444; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2445; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2446; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2447;
2448; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_acquire_cmpxchg:
2449; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2450; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2451; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2452; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2453; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2454; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2455; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2456; GFX90A-TGSPLIT-NEXT:    s_endpgm
2457;
2458; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acquire_acquire_cmpxchg:
2459; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2460; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2461; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2462; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2463; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2464; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2465; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2466; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2467;
2468; GFX940-TGSPLIT-LABEL: global_singlethread_acquire_acquire_cmpxchg:
2469; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2470; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2471; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2472; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2473; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2474; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2475; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2476; GFX940-TGSPLIT-NEXT:    s_endpgm
2477    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2478entry:
2479  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2480  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire acquire
2481  ret void
2482}
2483
2484define amdgpu_kernel void @global_singlethread_release_acquire_cmpxchg(
2485; GFX6-LABEL: global_singlethread_release_acquire_cmpxchg:
2486; GFX6:       ; %bb.0: ; %entry
2487; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
2488; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2489; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2490; GFX6-NEXT:    s_mov_b32 s2, -1
2491; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2492; GFX6-NEXT:    v_mov_b32_e32 v0, s6
2493; GFX6-NEXT:    v_mov_b32_e32 v1, s7
2494; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2495; GFX6-NEXT:    s_endpgm
2496;
2497; GFX7-LABEL: global_singlethread_release_acquire_cmpxchg:
2498; GFX7:       ; %bb.0: ; %entry
2499; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2500; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2501; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2502; GFX7-NEXT:    s_add_u32 s0, s0, 16
2503; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2504; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2505; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2506; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2507; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2508; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2509; GFX7-NEXT:    s_endpgm
2510;
2511; GFX10-WGP-LABEL: global_singlethread_release_acquire_cmpxchg:
2512; GFX10-WGP:       ; %bb.0: ; %entry
2513; GFX10-WGP-NEXT:    s_clause 0x1
2514; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2515; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2516; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2517; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2518; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2519; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2520; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2521; GFX10-WGP-NEXT:    s_endpgm
2522;
2523; GFX10-CU-LABEL: global_singlethread_release_acquire_cmpxchg:
2524; GFX10-CU:       ; %bb.0: ; %entry
2525; GFX10-CU-NEXT:    s_clause 0x1
2526; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2527; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2528; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2529; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2530; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2531; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2532; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2533; GFX10-CU-NEXT:    s_endpgm
2534;
2535; SKIP-CACHE-INV-LABEL: global_singlethread_release_acquire_cmpxchg:
2536; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2537; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
2538; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
2539; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
2540; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
2541; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2542; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2543; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2544; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2545; SKIP-CACHE-INV-NEXT:    s_endpgm
2546;
2547; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_release_acquire_cmpxchg:
2548; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2549; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2550; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2551; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2552; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2553; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2554; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2555; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2556;
2557; GFX90A-TGSPLIT-LABEL: global_singlethread_release_acquire_cmpxchg:
2558; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2559; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2560; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2561; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2562; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2563; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2564; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2565; GFX90A-TGSPLIT-NEXT:    s_endpgm
2566;
2567; GFX940-NOTTGSPLIT-LABEL: global_singlethread_release_acquire_cmpxchg:
2568; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2569; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2570; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2571; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2572; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2573; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2574; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2575; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2576;
2577; GFX940-TGSPLIT-LABEL: global_singlethread_release_acquire_cmpxchg:
2578; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2579; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2580; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2581; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2582; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2583; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2584; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2585; GFX940-TGSPLIT-NEXT:    s_endpgm
2586    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2587entry:
2588  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2589  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") release acquire
2590  ret void
2591}
2592
2593define amdgpu_kernel void @global_singlethread_acq_rel_acquire_cmpxchg(
2594; GFX6-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
2595; GFX6:       ; %bb.0: ; %entry
2596; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
2597; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2598; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2599; GFX6-NEXT:    s_mov_b32 s2, -1
2600; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2601; GFX6-NEXT:    v_mov_b32_e32 v0, s6
2602; GFX6-NEXT:    v_mov_b32_e32 v1, s7
2603; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2604; GFX6-NEXT:    s_endpgm
2605;
2606; GFX7-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
2607; GFX7:       ; %bb.0: ; %entry
2608; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2609; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2610; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2611; GFX7-NEXT:    s_add_u32 s0, s0, 16
2612; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2613; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2614; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2615; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2616; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2617; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2618; GFX7-NEXT:    s_endpgm
2619;
2620; GFX10-WGP-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
2621; GFX10-WGP:       ; %bb.0: ; %entry
2622; GFX10-WGP-NEXT:    s_clause 0x1
2623; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2624; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2625; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2626; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2627; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2628; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2629; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2630; GFX10-WGP-NEXT:    s_endpgm
2631;
2632; GFX10-CU-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
2633; GFX10-CU:       ; %bb.0: ; %entry
2634; GFX10-CU-NEXT:    s_clause 0x1
2635; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2636; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2637; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2638; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2639; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2640; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2641; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2642; GFX10-CU-NEXT:    s_endpgm
2643;
2644; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
2645; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2646; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
2647; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
2648; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
2649; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
2650; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2651; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2652; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2653; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2654; SKIP-CACHE-INV-NEXT:    s_endpgm
2655;
2656; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
2657; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2658; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2659; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2660; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2661; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2662; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2663; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2664; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2665;
2666; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
2667; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2668; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2669; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2670; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2671; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2672; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2673; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2674; GFX90A-TGSPLIT-NEXT:    s_endpgm
2675;
2676; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
2677; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2678; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2679; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2680; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2681; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2682; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2683; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2684; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2685;
2686; GFX940-TGSPLIT-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
2687; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2688; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2689; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2690; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2691; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2692; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2693; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2694; GFX940-TGSPLIT-NEXT:    s_endpgm
2695    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2696entry:
2697  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2698  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel acquire
2699  ret void
2700}
2701
2702define amdgpu_kernel void @global_singlethread_seq_cst_acquire_cmpxchg(
2703; GFX6-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
2704; GFX6:       ; %bb.0: ; %entry
2705; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
2706; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2707; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2708; GFX6-NEXT:    s_mov_b32 s2, -1
2709; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2710; GFX6-NEXT:    v_mov_b32_e32 v0, s6
2711; GFX6-NEXT:    v_mov_b32_e32 v1, s7
2712; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2713; GFX6-NEXT:    s_endpgm
2714;
2715; GFX7-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
2716; GFX7:       ; %bb.0: ; %entry
2717; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2718; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2719; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2720; GFX7-NEXT:    s_add_u32 s0, s0, 16
2721; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2722; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2723; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2724; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2725; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2726; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2727; GFX7-NEXT:    s_endpgm
2728;
2729; GFX10-WGP-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
2730; GFX10-WGP:       ; %bb.0: ; %entry
2731; GFX10-WGP-NEXT:    s_clause 0x1
2732; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2733; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2734; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2735; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2736; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2737; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2738; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2739; GFX10-WGP-NEXT:    s_endpgm
2740;
2741; GFX10-CU-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
2742; GFX10-CU:       ; %bb.0: ; %entry
2743; GFX10-CU-NEXT:    s_clause 0x1
2744; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2745; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2746; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2747; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2748; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2749; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2750; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2751; GFX10-CU-NEXT:    s_endpgm
2752;
2753; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
2754; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2755; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
2756; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
2757; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
2758; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
2759; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2760; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2761; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2762; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2763; SKIP-CACHE-INV-NEXT:    s_endpgm
2764;
2765; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
2766; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2767; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2768; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2769; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2770; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2771; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2772; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2773; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2774;
2775; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
2776; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2777; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2778; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2779; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2780; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2781; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2782; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2783; GFX90A-TGSPLIT-NEXT:    s_endpgm
2784;
2785; GFX940-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
2786; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2787; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2788; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2789; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2790; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2791; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2792; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2793; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2794;
2795; GFX940-TGSPLIT-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
2796; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2797; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2798; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2799; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2800; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2801; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2802; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2803; GFX940-TGSPLIT-NEXT:    s_endpgm
2804    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2805entry:
2806  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2807  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst acquire
2808  ret void
2809}
2810
2811define amdgpu_kernel void @global_singlethread_monotonic_seq_cst_cmpxchg(
2812; GFX6-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
2813; GFX6:       ; %bb.0: ; %entry
2814; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
2815; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2816; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2817; GFX6-NEXT:    s_mov_b32 s2, -1
2818; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2819; GFX6-NEXT:    v_mov_b32_e32 v0, s6
2820; GFX6-NEXT:    v_mov_b32_e32 v1, s7
2821; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2822; GFX6-NEXT:    s_endpgm
2823;
2824; GFX7-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
2825; GFX7:       ; %bb.0: ; %entry
2826; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2827; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2828; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2829; GFX7-NEXT:    s_add_u32 s0, s0, 16
2830; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2831; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2832; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2833; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2834; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2835; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2836; GFX7-NEXT:    s_endpgm
2837;
2838; GFX10-WGP-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
2839; GFX10-WGP:       ; %bb.0: ; %entry
2840; GFX10-WGP-NEXT:    s_clause 0x1
2841; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2842; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2843; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2844; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2845; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2846; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2847; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2848; GFX10-WGP-NEXT:    s_endpgm
2849;
2850; GFX10-CU-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
2851; GFX10-CU:       ; %bb.0: ; %entry
2852; GFX10-CU-NEXT:    s_clause 0x1
2853; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2854; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2855; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2856; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2857; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2858; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2859; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2860; GFX10-CU-NEXT:    s_endpgm
2861;
2862; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
2863; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2864; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
2865; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
2866; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
2867; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
2868; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2869; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2870; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2871; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2872; SKIP-CACHE-INV-NEXT:    s_endpgm
2873;
2874; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
2875; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2876; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2877; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2878; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2879; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2880; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2881; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2882; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2883;
2884; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
2885; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2886; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2887; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2888; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2889; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2890; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2891; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2892; GFX90A-TGSPLIT-NEXT:    s_endpgm
2893;
2894; GFX940-NOTTGSPLIT-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
2895; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2896; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2897; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2898; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2899; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2900; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2901; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2902; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2903;
2904; GFX940-TGSPLIT-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
2905; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2906; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2907; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2908; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2909; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2910; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2911; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2912; GFX940-TGSPLIT-NEXT:    s_endpgm
2913    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2914entry:
2915  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2916  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic seq_cst
2917  ret void
2918}
2919
2920define amdgpu_kernel void @global_singlethread_acquire_seq_cst_cmpxchg(
2921; GFX6-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
2922; GFX6:       ; %bb.0: ; %entry
2923; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
2924; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2925; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2926; GFX6-NEXT:    s_mov_b32 s2, -1
2927; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2928; GFX6-NEXT:    v_mov_b32_e32 v0, s6
2929; GFX6-NEXT:    v_mov_b32_e32 v1, s7
2930; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2931; GFX6-NEXT:    s_endpgm
2932;
2933; GFX7-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
2934; GFX7:       ; %bb.0: ; %entry
2935; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2936; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2937; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2938; GFX7-NEXT:    s_add_u32 s0, s0, 16
2939; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2940; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2941; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2942; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2943; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2944; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2945; GFX7-NEXT:    s_endpgm
2946;
2947; GFX10-WGP-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
2948; GFX10-WGP:       ; %bb.0: ; %entry
2949; GFX10-WGP-NEXT:    s_clause 0x1
2950; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2951; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2952; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2953; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2954; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2955; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2956; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2957; GFX10-WGP-NEXT:    s_endpgm
2958;
2959; GFX10-CU-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
2960; GFX10-CU:       ; %bb.0: ; %entry
2961; GFX10-CU-NEXT:    s_clause 0x1
2962; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2963; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2964; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2965; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2966; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2967; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2968; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2969; GFX10-CU-NEXT:    s_endpgm
2970;
2971; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
2972; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2973; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
2974; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
2975; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
2976; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
2977; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2978; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2979; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2980; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2981; SKIP-CACHE-INV-NEXT:    s_endpgm
2982;
2983; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
2984; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2985; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2986; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2987; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2988; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2989; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2990; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2991; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2992;
2993; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
2994; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2995; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2996; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2997; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2998; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2999; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3000; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3001; GFX90A-TGSPLIT-NEXT:    s_endpgm
3002;
3003; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
3004; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3005; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3006; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3007; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3008; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3009; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3010; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
3011; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3012;
3013; GFX940-TGSPLIT-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
3014; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3015; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3016; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3017; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3018; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3019; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3020; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
3021; GFX940-TGSPLIT-NEXT:    s_endpgm
3022    i32 addrspace(1)* %out, i32 %in, i32 %old) {
3023entry:
3024  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3025  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire seq_cst
3026  ret void
3027}
3028
3029define amdgpu_kernel void @global_singlethread_release_seq_cst_cmpxchg(
3030; GFX6-LABEL: global_singlethread_release_seq_cst_cmpxchg:
3031; GFX6:       ; %bb.0: ; %entry
3032; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
3033; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3034; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3035; GFX6-NEXT:    s_mov_b32 s2, -1
3036; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3037; GFX6-NEXT:    v_mov_b32_e32 v0, s6
3038; GFX6-NEXT:    v_mov_b32_e32 v1, s7
3039; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3040; GFX6-NEXT:    s_endpgm
3041;
3042; GFX7-LABEL: global_singlethread_release_seq_cst_cmpxchg:
3043; GFX7:       ; %bb.0: ; %entry
3044; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3045; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3046; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3047; GFX7-NEXT:    s_add_u32 s0, s0, 16
3048; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3049; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3050; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3051; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3052; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3053; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3054; GFX7-NEXT:    s_endpgm
3055;
3056; GFX10-WGP-LABEL: global_singlethread_release_seq_cst_cmpxchg:
3057; GFX10-WGP:       ; %bb.0: ; %entry
3058; GFX10-WGP-NEXT:    s_clause 0x1
3059; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3060; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3061; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3062; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3063; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3064; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3065; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3066; GFX10-WGP-NEXT:    s_endpgm
3067;
3068; GFX10-CU-LABEL: global_singlethread_release_seq_cst_cmpxchg:
3069; GFX10-CU:       ; %bb.0: ; %entry
3070; GFX10-CU-NEXT:    s_clause 0x1
3071; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3072; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3073; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3074; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3075; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3076; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3077; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3078; GFX10-CU-NEXT:    s_endpgm
3079;
3080; SKIP-CACHE-INV-LABEL: global_singlethread_release_seq_cst_cmpxchg:
3081; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3082; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
3083; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
3084; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3085; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3086; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3087; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3088; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3089; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3090; SKIP-CACHE-INV-NEXT:    s_endpgm
3091;
3092; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_release_seq_cst_cmpxchg:
3093; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3094; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3095; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3096; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3097; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3098; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3099; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3100; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3101;
3102; GFX90A-TGSPLIT-LABEL: global_singlethread_release_seq_cst_cmpxchg:
3103; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3104; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3105; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3106; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3107; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3108; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3109; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3110; GFX90A-TGSPLIT-NEXT:    s_endpgm
3111;
3112; GFX940-NOTTGSPLIT-LABEL: global_singlethread_release_seq_cst_cmpxchg:
3113; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3114; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3115; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3116; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3117; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3118; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3119; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
3120; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3121;
3122; GFX940-TGSPLIT-LABEL: global_singlethread_release_seq_cst_cmpxchg:
3123; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3124; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3125; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3126; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3127; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3128; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3129; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
3130; GFX940-TGSPLIT-NEXT:    s_endpgm
3131    i32 addrspace(1)* %out, i32 %in, i32 %old) {
3132entry:
3133  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3134  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") release seq_cst
3135  ret void
3136}
3137
3138define amdgpu_kernel void @global_singlethread_acq_rel_seq_cst_cmpxchg(
3139; GFX6-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
3140; GFX6:       ; %bb.0: ; %entry
3141; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
3142; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3143; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3144; GFX6-NEXT:    s_mov_b32 s2, -1
3145; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3146; GFX6-NEXT:    v_mov_b32_e32 v0, s6
3147; GFX6-NEXT:    v_mov_b32_e32 v1, s7
3148; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3149; GFX6-NEXT:    s_endpgm
3150;
3151; GFX7-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
3152; GFX7:       ; %bb.0: ; %entry
3153; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3154; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3155; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3156; GFX7-NEXT:    s_add_u32 s0, s0, 16
3157; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3158; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3159; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3160; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3161; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3162; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3163; GFX7-NEXT:    s_endpgm
3164;
3165; GFX10-WGP-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
3166; GFX10-WGP:       ; %bb.0: ; %entry
3167; GFX10-WGP-NEXT:    s_clause 0x1
3168; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3169; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3170; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3171; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3172; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3173; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3174; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3175; GFX10-WGP-NEXT:    s_endpgm
3176;
3177; GFX10-CU-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
3178; GFX10-CU:       ; %bb.0: ; %entry
3179; GFX10-CU-NEXT:    s_clause 0x1
3180; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3181; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3182; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3183; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3184; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3185; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3186; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3187; GFX10-CU-NEXT:    s_endpgm
3188;
3189; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
3190; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3191; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
3192; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
3193; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3194; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3195; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3196; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3197; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3198; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3199; SKIP-CACHE-INV-NEXT:    s_endpgm
3200;
3201; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
3202; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3203; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3204; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3205; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3206; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3207; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3208; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3209; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3210;
3211; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
3212; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3213; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3214; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3215; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3216; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3217; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3218; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3219; GFX90A-TGSPLIT-NEXT:    s_endpgm
3220;
3221; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
3222; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3223; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3224; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3225; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3226; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3227; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3228; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
3229; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3230;
3231; GFX940-TGSPLIT-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
3232; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3233; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3234; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3235; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3236; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3237; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3238; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
3239; GFX940-TGSPLIT-NEXT:    s_endpgm
3240    i32 addrspace(1)* %out, i32 %in, i32 %old) {
3241entry:
3242  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3243  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel seq_cst
3244  ret void
3245}
3246
3247define amdgpu_kernel void @global_singlethread_seq_cst_seq_cst_cmpxchg(
3248; GFX6-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
3249; GFX6:       ; %bb.0: ; %entry
3250; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
3251; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3252; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3253; GFX6-NEXT:    s_mov_b32 s2, -1
3254; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3255; GFX6-NEXT:    v_mov_b32_e32 v0, s6
3256; GFX6-NEXT:    v_mov_b32_e32 v1, s7
3257; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3258; GFX6-NEXT:    s_endpgm
3259;
3260; GFX7-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
3261; GFX7:       ; %bb.0: ; %entry
3262; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3263; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3264; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3265; GFX7-NEXT:    s_add_u32 s0, s0, 16
3266; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3267; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3268; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3269; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3270; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3271; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3272; GFX7-NEXT:    s_endpgm
3273;
3274; GFX10-WGP-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
3275; GFX10-WGP:       ; %bb.0: ; %entry
3276; GFX10-WGP-NEXT:    s_clause 0x1
3277; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3278; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3279; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3280; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3281; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3282; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3283; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3284; GFX10-WGP-NEXT:    s_endpgm
3285;
3286; GFX10-CU-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
3287; GFX10-CU:       ; %bb.0: ; %entry
3288; GFX10-CU-NEXT:    s_clause 0x1
3289; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3290; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3291; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3292; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3293; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3294; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3295; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3296; GFX10-CU-NEXT:    s_endpgm
3297;
3298; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
3299; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3300; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
3301; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
3302; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3303; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3304; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3305; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3306; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3307; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3308; SKIP-CACHE-INV-NEXT:    s_endpgm
3309;
3310; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
3311; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3312; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3313; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3314; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3315; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3316; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3317; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3318; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3319;
3320; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
3321; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3322; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3323; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3324; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3325; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3326; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3327; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3328; GFX90A-TGSPLIT-NEXT:    s_endpgm
3329;
3330; GFX940-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
3331; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3332; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3333; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3334; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3335; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3336; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3337; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
3338; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3339;
3340; GFX940-TGSPLIT-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
3341; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3342; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3343; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3344; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3345; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3346; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3347; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
3348; GFX940-TGSPLIT-NEXT:    s_endpgm
3349    i32 addrspace(1)* %out, i32 %in, i32 %old) {
3350entry:
3351  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3352  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst seq_cst
3353  ret void
3354}
3355
3356define amdgpu_kernel void @global_singlethread_monotonic_monotonic_ret_cmpxchg(
3357; GFX6-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
3358; GFX6:       ; %bb.0: ; %entry
3359; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
3360; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3361; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3362; GFX6-NEXT:    s_mov_b32 s2, -1
3363; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3364; GFX6-NEXT:    v_mov_b32_e32 v0, s6
3365; GFX6-NEXT:    v_mov_b32_e32 v1, s7
3366; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3367; GFX6-NEXT:    s_waitcnt vmcnt(0)
3368; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3369; GFX6-NEXT:    s_endpgm
3370;
3371; GFX7-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
3372; GFX7:       ; %bb.0: ; %entry
3373; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3374; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3375; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3376; GFX7-NEXT:    s_add_u32 s4, s0, 16
3377; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3378; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3379; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3380; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3381; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3382; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3383; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3384; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3385; GFX7-NEXT:    s_waitcnt vmcnt(0)
3386; GFX7-NEXT:    flat_store_dword v[0:1], v2
3387; GFX7-NEXT:    s_endpgm
3388;
3389; GFX10-WGP-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
3390; GFX10-WGP:       ; %bb.0: ; %entry
3391; GFX10-WGP-NEXT:    s_clause 0x1
3392; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3393; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3394; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3395; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3396; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3397; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3398; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3399; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3400; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
3401; GFX10-WGP-NEXT:    s_endpgm
3402;
3403; GFX10-CU-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
3404; GFX10-CU:       ; %bb.0: ; %entry
3405; GFX10-CU-NEXT:    s_clause 0x1
3406; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3407; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3408; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3409; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3410; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3411; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3412; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3413; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3414; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
3415; GFX10-CU-NEXT:    s_endpgm
3416;
3417; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
3418; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3419; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
3420; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
3421; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3422; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3423; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3424; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3425; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3426; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3427; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3428; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3429; SKIP-CACHE-INV-NEXT:    s_endpgm
3430;
3431; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
3432; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3433; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3434; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3435; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3436; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3437; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3438; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3439; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3440; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
3441; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3442;
3443; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
3444; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3445; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3446; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3447; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3448; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3449; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3450; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3451; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3452; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
3453; GFX90A-TGSPLIT-NEXT:    s_endpgm
3454;
3455; GFX940-NOTTGSPLIT-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
3456; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3457; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3458; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3459; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3460; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3461; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3462; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
3463; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3464; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
3465; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3466;
3467; GFX940-TGSPLIT-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
3468; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3469; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3470; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3471; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3472; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3473; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3474; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
3475; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3476; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
3477; GFX940-TGSPLIT-NEXT:    s_endpgm
3478    i32 addrspace(1)* %out, i32 %in, i32 %old) {
3479entry:
3480  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3481  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic monotonic
3482  %val0 = extractvalue { i32, i1 } %val, 0
3483  store i32 %val0, i32 addrspace(1)* %out, align 4
3484  ret void
3485}
3486
3487define amdgpu_kernel void @global_singlethread_acquire_monotonic_ret_cmpxchg(
3488; GFX6-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
3489; GFX6:       ; %bb.0: ; %entry
3490; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
3491; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3492; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3493; GFX6-NEXT:    s_mov_b32 s2, -1
3494; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3495; GFX6-NEXT:    v_mov_b32_e32 v0, s6
3496; GFX6-NEXT:    v_mov_b32_e32 v1, s7
3497; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3498; GFX6-NEXT:    s_waitcnt vmcnt(0)
3499; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3500; GFX6-NEXT:    s_endpgm
3501;
3502; GFX7-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
3503; GFX7:       ; %bb.0: ; %entry
3504; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3505; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3506; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3507; GFX7-NEXT:    s_add_u32 s4, s0, 16
3508; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3509; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3510; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3511; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3512; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3513; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3514; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3515; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3516; GFX7-NEXT:    s_waitcnt vmcnt(0)
3517; GFX7-NEXT:    flat_store_dword v[0:1], v2
3518; GFX7-NEXT:    s_endpgm
3519;
3520; GFX10-WGP-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
3521; GFX10-WGP:       ; %bb.0: ; %entry
3522; GFX10-WGP-NEXT:    s_clause 0x1
3523; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3524; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3525; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3526; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3527; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3528; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3529; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3530; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3531; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
3532; GFX10-WGP-NEXT:    s_endpgm
3533;
3534; GFX10-CU-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
3535; GFX10-CU:       ; %bb.0: ; %entry
3536; GFX10-CU-NEXT:    s_clause 0x1
3537; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3538; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3539; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3540; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3541; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3542; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3543; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3544; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3545; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
3546; GFX10-CU-NEXT:    s_endpgm
3547;
3548; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
3549; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3550; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
3551; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
3552; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3553; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3554; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3555; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3556; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3557; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3558; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3559; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3560; SKIP-CACHE-INV-NEXT:    s_endpgm
3561;
3562; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
3563; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3564; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3565; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3566; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3567; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3568; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3569; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3570; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3571; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
3572; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3573;
3574; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
3575; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3576; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3577; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3578; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3579; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3580; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3581; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3582; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3583; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
3584; GFX90A-TGSPLIT-NEXT:    s_endpgm
3585;
3586; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
3587; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3588; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3589; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3590; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3591; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3592; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3593; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
3594; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3595; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
3596; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3597;
3598; GFX940-TGSPLIT-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
3599; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3600; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3601; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3602; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3603; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3604; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3605; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
3606; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3607; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
3608; GFX940-TGSPLIT-NEXT:    s_endpgm
3609    i32 addrspace(1)* %out, i32 %in, i32 %old) {
3610entry:
3611  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3612  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire monotonic
3613  %val0 = extractvalue { i32, i1 } %val, 0
3614  store i32 %val0, i32 addrspace(1)* %out, align 4
3615  ret void
3616}
3617
3618define amdgpu_kernel void @global_singlethread_release_monotonic_ret_cmpxchg(
3619; GFX6-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
3620; GFX6:       ; %bb.0: ; %entry
3621; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
3622; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3623; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3624; GFX6-NEXT:    s_mov_b32 s2, -1
3625; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3626; GFX6-NEXT:    v_mov_b32_e32 v0, s6
3627; GFX6-NEXT:    v_mov_b32_e32 v1, s7
3628; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3629; GFX6-NEXT:    s_waitcnt vmcnt(0)
3630; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3631; GFX6-NEXT:    s_endpgm
3632;
3633; GFX7-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
3634; GFX7:       ; %bb.0: ; %entry
3635; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3636; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3637; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3638; GFX7-NEXT:    s_add_u32 s4, s0, 16
3639; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3640; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3641; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3642; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3643; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3644; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3645; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3646; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3647; GFX7-NEXT:    s_waitcnt vmcnt(0)
3648; GFX7-NEXT:    flat_store_dword v[0:1], v2
3649; GFX7-NEXT:    s_endpgm
3650;
3651; GFX10-WGP-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
3652; GFX10-WGP:       ; %bb.0: ; %entry
3653; GFX10-WGP-NEXT:    s_clause 0x1
3654; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3655; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3656; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3657; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3658; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3659; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3660; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3661; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3662; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
3663; GFX10-WGP-NEXT:    s_endpgm
3664;
3665; GFX10-CU-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
3666; GFX10-CU:       ; %bb.0: ; %entry
3667; GFX10-CU-NEXT:    s_clause 0x1
3668; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3669; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3670; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3671; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3672; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3673; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3674; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3675; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3676; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
3677; GFX10-CU-NEXT:    s_endpgm
3678;
3679; SKIP-CACHE-INV-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
3680; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3681; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
3682; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
3683; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3684; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3685; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3686; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3687; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3688; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3689; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3690; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3691; SKIP-CACHE-INV-NEXT:    s_endpgm
3692;
3693; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
3694; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3695; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3696; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3697; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3698; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3699; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3700; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3701; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3702; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
3703; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3704;
3705; GFX90A-TGSPLIT-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
3706; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3707; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3708; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3709; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3710; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3711; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3712; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3713; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3714; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
3715; GFX90A-TGSPLIT-NEXT:    s_endpgm
3716;
3717; GFX940-NOTTGSPLIT-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
3718; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3719; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3720; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3721; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3722; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3723; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3724; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
3725; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3726; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
3727; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3728;
3729; GFX940-TGSPLIT-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
3730; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3731; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3732; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3733; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3734; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3735; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3736; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
3737; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3738; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
3739; GFX940-TGSPLIT-NEXT:    s_endpgm
3740    i32 addrspace(1)* %out, i32 %in, i32 %old) {
3741entry:
3742  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3743  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") release monotonic
3744  %val0 = extractvalue { i32, i1 } %val, 0
3745  store i32 %val0, i32 addrspace(1)* %out, align 4
3746  ret void
3747}
3748
3749define amdgpu_kernel void @global_singlethread_acq_rel_monotonic_ret_cmpxchg(
3750; GFX6-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
3751; GFX6:       ; %bb.0: ; %entry
3752; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
3753; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3754; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3755; GFX6-NEXT:    s_mov_b32 s2, -1
3756; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3757; GFX6-NEXT:    v_mov_b32_e32 v0, s6
3758; GFX6-NEXT:    v_mov_b32_e32 v1, s7
3759; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3760; GFX6-NEXT:    s_waitcnt vmcnt(0)
3761; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3762; GFX6-NEXT:    s_endpgm
3763;
3764; GFX7-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
3765; GFX7:       ; %bb.0: ; %entry
3766; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3767; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3768; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3769; GFX7-NEXT:    s_add_u32 s4, s0, 16
3770; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3771; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3772; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3773; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3774; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3775; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3776; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3777; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3778; GFX7-NEXT:    s_waitcnt vmcnt(0)
3779; GFX7-NEXT:    flat_store_dword v[0:1], v2
3780; GFX7-NEXT:    s_endpgm
3781;
3782; GFX10-WGP-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
3783; GFX10-WGP:       ; %bb.0: ; %entry
3784; GFX10-WGP-NEXT:    s_clause 0x1
3785; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3786; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3787; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3788; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3789; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3790; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3791; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3792; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3793; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
3794; GFX10-WGP-NEXT:    s_endpgm
3795;
3796; GFX10-CU-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
3797; GFX10-CU:       ; %bb.0: ; %entry
3798; GFX10-CU-NEXT:    s_clause 0x1
3799; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3800; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3801; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3802; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3803; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3804; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3805; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3806; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3807; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
3808; GFX10-CU-NEXT:    s_endpgm
3809;
3810; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
3811; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3812; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
3813; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
3814; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3815; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3816; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3817; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3818; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3819; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3820; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3821; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3822; SKIP-CACHE-INV-NEXT:    s_endpgm
3823;
3824; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
3825; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3826; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3827; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3828; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3829; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3830; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3831; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3832; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3833; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
3834; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3835;
3836; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
3837; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3838; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3839; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3840; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3841; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3842; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3843; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3844; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3845; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
3846; GFX90A-TGSPLIT-NEXT:    s_endpgm
3847;
3848; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
3849; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3850; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3851; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3852; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3853; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3854; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3855; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
3856; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3857; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
3858; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3859;
3860; GFX940-TGSPLIT-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
3861; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3862; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3863; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3864; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3865; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3866; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3867; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
3868; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3869; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
3870; GFX940-TGSPLIT-NEXT:    s_endpgm
3871    i32 addrspace(1)* %out, i32 %in, i32 %old) {
3872entry:
3873  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3874  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel monotonic
3875  %val0 = extractvalue { i32, i1 } %val, 0
3876  store i32 %val0, i32 addrspace(1)* %out, align 4
3877  ret void
3878}
3879
3880define amdgpu_kernel void @global_singlethread_seq_cst_monotonic_ret_cmpxchg(
3881; GFX6-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
3882; GFX6:       ; %bb.0: ; %entry
3883; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
3884; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3885; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3886; GFX6-NEXT:    s_mov_b32 s2, -1
3887; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3888; GFX6-NEXT:    v_mov_b32_e32 v0, s6
3889; GFX6-NEXT:    v_mov_b32_e32 v1, s7
3890; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3891; GFX6-NEXT:    s_waitcnt vmcnt(0)
3892; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3893; GFX6-NEXT:    s_endpgm
3894;
3895; GFX7-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
3896; GFX7:       ; %bb.0: ; %entry
3897; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3898; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3899; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3900; GFX7-NEXT:    s_add_u32 s4, s0, 16
3901; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3902; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3903; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3904; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3905; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3906; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3907; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3908; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3909; GFX7-NEXT:    s_waitcnt vmcnt(0)
3910; GFX7-NEXT:    flat_store_dword v[0:1], v2
3911; GFX7-NEXT:    s_endpgm
3912;
3913; GFX10-WGP-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
3914; GFX10-WGP:       ; %bb.0: ; %entry
3915; GFX10-WGP-NEXT:    s_clause 0x1
3916; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3917; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3918; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3919; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3920; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3921; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3922; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3923; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3924; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
3925; GFX10-WGP-NEXT:    s_endpgm
3926;
3927; GFX10-CU-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
3928; GFX10-CU:       ; %bb.0: ; %entry
3929; GFX10-CU-NEXT:    s_clause 0x1
3930; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3931; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3932; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3933; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3934; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3935; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3936; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3937; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3938; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
3939; GFX10-CU-NEXT:    s_endpgm
3940;
3941; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
3942; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3943; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
3944; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
3945; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3946; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3947; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3948; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3949; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3950; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3951; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3952; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3953; SKIP-CACHE-INV-NEXT:    s_endpgm
3954;
3955; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
3956; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3957; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3958; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3959; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3960; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3961; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3962; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3963; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3964; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
3965; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3966;
3967; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
3968; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3969; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3970; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3971; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3972; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3973; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3974; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3975; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3976; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
3977; GFX90A-TGSPLIT-NEXT:    s_endpgm
3978;
3979; GFX940-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
3980; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3981; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3982; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3983; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3984; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3985; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3986; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
3987; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3988; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
3989; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3990;
3991; GFX940-TGSPLIT-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
3992; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3993; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3994; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3995; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3996; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3997; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3998; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
3999; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4000; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4001; GFX940-TGSPLIT-NEXT:    s_endpgm
4002    i32 addrspace(1)* %out, i32 %in, i32 %old) {
4003entry:
4004  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
4005  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst monotonic
4006  %val0 = extractvalue { i32, i1 } %val, 0
4007  store i32 %val0, i32 addrspace(1)* %out, align 4
4008  ret void
4009}
4010
4011define amdgpu_kernel void @global_singlethread_monotonic_acquire_ret_cmpxchg(
4012; GFX6-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
4013; GFX6:       ; %bb.0: ; %entry
4014; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
4015; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4016; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4017; GFX6-NEXT:    s_mov_b32 s2, -1
4018; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4019; GFX6-NEXT:    v_mov_b32_e32 v0, s6
4020; GFX6-NEXT:    v_mov_b32_e32 v1, s7
4021; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4022; GFX6-NEXT:    s_waitcnt vmcnt(0)
4023; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4024; GFX6-NEXT:    s_endpgm
4025;
4026; GFX7-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
4027; GFX7:       ; %bb.0: ; %entry
4028; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4029; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4030; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4031; GFX7-NEXT:    s_add_u32 s4, s0, 16
4032; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4033; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4034; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4035; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4036; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4037; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4038; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4039; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4040; GFX7-NEXT:    s_waitcnt vmcnt(0)
4041; GFX7-NEXT:    flat_store_dword v[0:1], v2
4042; GFX7-NEXT:    s_endpgm
4043;
4044; GFX10-WGP-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
4045; GFX10-WGP:       ; %bb.0: ; %entry
4046; GFX10-WGP-NEXT:    s_clause 0x1
4047; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4048; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4049; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
4050; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4051; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4052; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4053; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4054; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4055; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
4056; GFX10-WGP-NEXT:    s_endpgm
4057;
4058; GFX10-CU-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
4059; GFX10-CU:       ; %bb.0: ; %entry
4060; GFX10-CU-NEXT:    s_clause 0x1
4061; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4062; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4063; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
4064; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4065; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4066; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4067; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4068; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4069; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
4070; GFX10-CU-NEXT:    s_endpgm
4071;
4072; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
4073; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4074; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
4075; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
4076; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
4077; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
4078; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4079; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4080; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4081; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4082; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4083; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4084; SKIP-CACHE-INV-NEXT:    s_endpgm
4085;
4086; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
4087; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4088; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4089; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4090; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4091; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4092; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4093; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4094; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4095; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4096; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4097;
4098; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
4099; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4100; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4101; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4102; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4103; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4104; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4105; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4106; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4107; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4108; GFX90A-TGSPLIT-NEXT:    s_endpgm
4109;
4110; GFX940-NOTTGSPLIT-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
4111; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4112; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4113; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4114; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4115; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4116; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4117; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4118; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4119; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4120; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4121;
4122; GFX940-TGSPLIT-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
4123; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4124; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4125; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4126; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4127; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4128; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4129; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4130; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4131; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4132; GFX940-TGSPLIT-NEXT:    s_endpgm
4133    i32 addrspace(1)* %out, i32 %in, i32 %old) {
4134entry:
4135  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
4136  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic acquire
4137  %val0 = extractvalue { i32, i1 } %val, 0
4138  store i32 %val0, i32 addrspace(1)* %out, align 4
4139  ret void
4140}
4141
4142define amdgpu_kernel void @global_singlethread_acquire_acquire_ret_cmpxchg(
4143; GFX6-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
4144; GFX6:       ; %bb.0: ; %entry
4145; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
4146; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4147; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4148; GFX6-NEXT:    s_mov_b32 s2, -1
4149; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4150; GFX6-NEXT:    v_mov_b32_e32 v0, s6
4151; GFX6-NEXT:    v_mov_b32_e32 v1, s7
4152; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4153; GFX6-NEXT:    s_waitcnt vmcnt(0)
4154; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4155; GFX6-NEXT:    s_endpgm
4156;
4157; GFX7-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
4158; GFX7:       ; %bb.0: ; %entry
4159; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4160; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4161; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4162; GFX7-NEXT:    s_add_u32 s4, s0, 16
4163; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4164; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4165; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4166; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4167; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4168; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4169; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4170; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4171; GFX7-NEXT:    s_waitcnt vmcnt(0)
4172; GFX7-NEXT:    flat_store_dword v[0:1], v2
4173; GFX7-NEXT:    s_endpgm
4174;
4175; GFX10-WGP-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
4176; GFX10-WGP:       ; %bb.0: ; %entry
4177; GFX10-WGP-NEXT:    s_clause 0x1
4178; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4179; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4180; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
4181; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4182; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4183; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4184; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4185; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4186; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
4187; GFX10-WGP-NEXT:    s_endpgm
4188;
4189; GFX10-CU-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
4190; GFX10-CU:       ; %bb.0: ; %entry
4191; GFX10-CU-NEXT:    s_clause 0x1
4192; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4193; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4194; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
4195; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4196; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4197; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4198; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4199; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4200; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
4201; GFX10-CU-NEXT:    s_endpgm
4202;
4203; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
4204; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4205; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
4206; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
4207; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
4208; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
4209; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4210; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4211; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4212; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4213; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4214; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4215; SKIP-CACHE-INV-NEXT:    s_endpgm
4216;
4217; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
4218; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4219; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4220; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4221; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4222; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4223; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4224; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4225; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4226; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4227; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4228;
4229; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
4230; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4231; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4232; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4233; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4234; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4235; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4236; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4237; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4238; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4239; GFX90A-TGSPLIT-NEXT:    s_endpgm
4240;
4241; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
4242; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4243; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4244; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4245; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4246; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4247; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4248; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4249; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4250; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4251; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4252;
4253; GFX940-TGSPLIT-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
4254; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4255; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4256; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4257; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4258; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4259; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4260; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4261; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4262; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4263; GFX940-TGSPLIT-NEXT:    s_endpgm
4264    i32 addrspace(1)* %out, i32 %in, i32 %old) {
4265entry:
4266  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
4267  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire acquire
4268  %val0 = extractvalue { i32, i1 } %val, 0
4269  store i32 %val0, i32 addrspace(1)* %out, align 4
4270  ret void
4271}
4272
4273define amdgpu_kernel void @global_singlethread_release_acquire_ret_cmpxchg(
4274; GFX6-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
4275; GFX6:       ; %bb.0: ; %entry
4276; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
4277; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4278; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4279; GFX6-NEXT:    s_mov_b32 s2, -1
4280; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4281; GFX6-NEXT:    v_mov_b32_e32 v0, s6
4282; GFX6-NEXT:    v_mov_b32_e32 v1, s7
4283; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4284; GFX6-NEXT:    s_waitcnt vmcnt(0)
4285; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4286; GFX6-NEXT:    s_endpgm
4287;
4288; GFX7-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
4289; GFX7:       ; %bb.0: ; %entry
4290; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4291; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4292; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4293; GFX7-NEXT:    s_add_u32 s4, s0, 16
4294; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4295; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4296; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4297; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4298; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4299; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4300; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4301; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4302; GFX7-NEXT:    s_waitcnt vmcnt(0)
4303; GFX7-NEXT:    flat_store_dword v[0:1], v2
4304; GFX7-NEXT:    s_endpgm
4305;
4306; GFX10-WGP-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
4307; GFX10-WGP:       ; %bb.0: ; %entry
4308; GFX10-WGP-NEXT:    s_clause 0x1
4309; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4310; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4311; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
4312; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4313; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4314; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4315; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4316; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4317; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
4318; GFX10-WGP-NEXT:    s_endpgm
4319;
4320; GFX10-CU-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
4321; GFX10-CU:       ; %bb.0: ; %entry
4322; GFX10-CU-NEXT:    s_clause 0x1
4323; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4324; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4325; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
4326; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4327; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4328; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4329; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4330; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4331; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
4332; GFX10-CU-NEXT:    s_endpgm
4333;
4334; SKIP-CACHE-INV-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
4335; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4336; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
4337; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
4338; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
4339; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
4340; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4341; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4342; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4343; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4344; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4345; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4346; SKIP-CACHE-INV-NEXT:    s_endpgm
4347;
4348; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
4349; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4350; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4351; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4352; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4353; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4354; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4355; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4356; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4357; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4358; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4359;
4360; GFX90A-TGSPLIT-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
4361; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4362; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4363; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4364; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4365; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4366; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4367; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4368; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4369; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4370; GFX90A-TGSPLIT-NEXT:    s_endpgm
4371;
4372; GFX940-NOTTGSPLIT-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
4373; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4374; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4375; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4376; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4377; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4378; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4379; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4380; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4381; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4382; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4383;
4384; GFX940-TGSPLIT-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
4385; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4386; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4387; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4388; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4389; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4390; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4391; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4392; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4393; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4394; GFX940-TGSPLIT-NEXT:    s_endpgm
4395    i32 addrspace(1)* %out, i32 %in, i32 %old) {
4396entry:
4397  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
4398  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") release acquire
4399  %val0 = extractvalue { i32, i1 } %val, 0
4400  store i32 %val0, i32 addrspace(1)* %out, align 4
4401  ret void
4402}
4403
4404define amdgpu_kernel void @global_singlethread_acq_rel_acquire_ret_cmpxchg(
4405; GFX6-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
4406; GFX6:       ; %bb.0: ; %entry
4407; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
4408; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4409; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4410; GFX6-NEXT:    s_mov_b32 s2, -1
4411; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4412; GFX6-NEXT:    v_mov_b32_e32 v0, s6
4413; GFX6-NEXT:    v_mov_b32_e32 v1, s7
4414; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4415; GFX6-NEXT:    s_waitcnt vmcnt(0)
4416; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4417; GFX6-NEXT:    s_endpgm
4418;
4419; GFX7-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
4420; GFX7:       ; %bb.0: ; %entry
4421; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4422; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4423; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4424; GFX7-NEXT:    s_add_u32 s4, s0, 16
4425; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4426; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4427; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4428; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4429; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4430; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4431; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4432; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4433; GFX7-NEXT:    s_waitcnt vmcnt(0)
4434; GFX7-NEXT:    flat_store_dword v[0:1], v2
4435; GFX7-NEXT:    s_endpgm
4436;
4437; GFX10-WGP-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
4438; GFX10-WGP:       ; %bb.0: ; %entry
4439; GFX10-WGP-NEXT:    s_clause 0x1
4440; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4441; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4442; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
4443; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4444; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4445; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4446; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4447; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4448; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
4449; GFX10-WGP-NEXT:    s_endpgm
4450;
4451; GFX10-CU-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
4452; GFX10-CU:       ; %bb.0: ; %entry
4453; GFX10-CU-NEXT:    s_clause 0x1
4454; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4455; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4456; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
4457; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4458; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4459; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4460; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4461; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4462; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
4463; GFX10-CU-NEXT:    s_endpgm
4464;
4465; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
4466; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4467; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
4468; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
4469; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
4470; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
4471; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4472; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4473; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4474; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4475; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4476; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4477; SKIP-CACHE-INV-NEXT:    s_endpgm
4478;
4479; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
4480; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4481; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4482; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4483; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4484; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4485; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4486; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4487; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4488; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4489; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4490;
4491; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
4492; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4493; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4494; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4495; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4496; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4497; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4498; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4499; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4500; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4501; GFX90A-TGSPLIT-NEXT:    s_endpgm
4502;
4503; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
4504; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4505; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4506; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4507; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4508; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4509; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4510; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4511; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4512; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4513; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4514;
4515; GFX940-TGSPLIT-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
4516; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4517; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4518; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4519; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4520; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4521; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4522; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4523; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4524; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4525; GFX940-TGSPLIT-NEXT:    s_endpgm
4526    i32 addrspace(1)* %out, i32 %in, i32 %old) {
4527entry:
4528  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
4529  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel acquire
4530  %val0 = extractvalue { i32, i1 } %val, 0
4531  store i32 %val0, i32 addrspace(1)* %out, align 4
4532  ret void
4533}
4534
4535define amdgpu_kernel void @global_singlethread_seq_cst_acquire_ret_cmpxchg(
4536; GFX6-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
4537; GFX6:       ; %bb.0: ; %entry
4538; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
4539; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4540; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4541; GFX6-NEXT:    s_mov_b32 s2, -1
4542; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4543; GFX6-NEXT:    v_mov_b32_e32 v0, s6
4544; GFX6-NEXT:    v_mov_b32_e32 v1, s7
4545; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4546; GFX6-NEXT:    s_waitcnt vmcnt(0)
4547; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4548; GFX6-NEXT:    s_endpgm
4549;
4550; GFX7-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
4551; GFX7:       ; %bb.0: ; %entry
4552; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4553; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4554; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4555; GFX7-NEXT:    s_add_u32 s4, s0, 16
4556; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4557; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4558; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4559; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4560; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4561; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4562; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4563; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4564; GFX7-NEXT:    s_waitcnt vmcnt(0)
4565; GFX7-NEXT:    flat_store_dword v[0:1], v2
4566; GFX7-NEXT:    s_endpgm
4567;
4568; GFX10-WGP-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
4569; GFX10-WGP:       ; %bb.0: ; %entry
4570; GFX10-WGP-NEXT:    s_clause 0x1
4571; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4572; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4573; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
4574; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4575; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4576; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4577; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4578; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4579; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
4580; GFX10-WGP-NEXT:    s_endpgm
4581;
4582; GFX10-CU-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
4583; GFX10-CU:       ; %bb.0: ; %entry
4584; GFX10-CU-NEXT:    s_clause 0x1
4585; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4586; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4587; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
4588; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4589; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4590; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4591; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4592; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4593; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
4594; GFX10-CU-NEXT:    s_endpgm
4595;
4596; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
4597; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4598; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
4599; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
4600; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
4601; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
4602; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4603; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4604; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4605; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4606; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4607; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4608; SKIP-CACHE-INV-NEXT:    s_endpgm
4609;
4610; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
4611; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4612; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4613; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4614; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4615; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4616; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4617; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4618; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4619; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4620; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4621;
4622; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
4623; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4624; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4625; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4626; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4627; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4628; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4629; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4630; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4631; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4632; GFX90A-TGSPLIT-NEXT:    s_endpgm
4633;
4634; GFX940-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
4635; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4636; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4637; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4638; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4639; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4640; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4641; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4642; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4643; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4644; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4645;
4646; GFX940-TGSPLIT-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
4647; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4648; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4649; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4650; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4651; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4652; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4653; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4654; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4655; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4656; GFX940-TGSPLIT-NEXT:    s_endpgm
4657    i32 addrspace(1)* %out, i32 %in, i32 %old) {
4658entry:
4659  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
4660  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst acquire
4661  %val0 = extractvalue { i32, i1 } %val, 0
4662  store i32 %val0, i32 addrspace(1)* %out, align 4
4663  ret void
4664}
4665
4666define amdgpu_kernel void @global_singlethread_monotonic_seq_cst_ret_cmpxchg(
4667; GFX6-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
4668; GFX6:       ; %bb.0: ; %entry
4669; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
4670; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4671; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4672; GFX6-NEXT:    s_mov_b32 s2, -1
4673; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4674; GFX6-NEXT:    v_mov_b32_e32 v0, s6
4675; GFX6-NEXT:    v_mov_b32_e32 v1, s7
4676; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4677; GFX6-NEXT:    s_waitcnt vmcnt(0)
4678; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4679; GFX6-NEXT:    s_endpgm
4680;
4681; GFX7-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
4682; GFX7:       ; %bb.0: ; %entry
4683; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4684; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4685; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4686; GFX7-NEXT:    s_add_u32 s4, s0, 16
4687; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4688; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4689; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4690; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4691; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4692; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4693; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4694; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4695; GFX7-NEXT:    s_waitcnt vmcnt(0)
4696; GFX7-NEXT:    flat_store_dword v[0:1], v2
4697; GFX7-NEXT:    s_endpgm
4698;
4699; GFX10-WGP-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
4700; GFX10-WGP:       ; %bb.0: ; %entry
4701; GFX10-WGP-NEXT:    s_clause 0x1
4702; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4703; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4704; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
4705; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4706; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4707; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4708; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4709; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4710; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
4711; GFX10-WGP-NEXT:    s_endpgm
4712;
4713; GFX10-CU-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
4714; GFX10-CU:       ; %bb.0: ; %entry
4715; GFX10-CU-NEXT:    s_clause 0x1
4716; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4717; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4718; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
4719; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4720; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4721; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4722; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4723; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4724; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
4725; GFX10-CU-NEXT:    s_endpgm
4726;
4727; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
4728; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4729; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
4730; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
4731; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
4732; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
4733; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4734; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4735; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4736; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4737; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4738; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4739; SKIP-CACHE-INV-NEXT:    s_endpgm
4740;
4741; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
4742; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4743; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4744; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4745; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4746; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4747; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4748; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4749; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4750; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4751; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4752;
4753; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
4754; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4755; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4756; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4757; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4758; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4759; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4760; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4761; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4762; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4763; GFX90A-TGSPLIT-NEXT:    s_endpgm
4764;
4765; GFX940-NOTTGSPLIT-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
4766; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4767; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4768; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4769; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4770; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4771; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4772; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4773; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4774; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4775; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4776;
4777; GFX940-TGSPLIT-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
4778; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4779; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4780; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4781; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4782; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4783; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4784; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4785; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4786; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4787; GFX940-TGSPLIT-NEXT:    s_endpgm
4788    i32 addrspace(1)* %out, i32 %in, i32 %old) {
4789entry:
4790  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
4791  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic seq_cst
4792  %val0 = extractvalue { i32, i1 } %val, 0
4793  store i32 %val0, i32 addrspace(1)* %out, align 4
4794  ret void
4795}
4796
4797define amdgpu_kernel void @global_singlethread_acquire_seq_cst_ret_cmpxchg(
4798; GFX6-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
4799; GFX6:       ; %bb.0: ; %entry
4800; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
4801; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4802; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4803; GFX6-NEXT:    s_mov_b32 s2, -1
4804; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4805; GFX6-NEXT:    v_mov_b32_e32 v0, s6
4806; GFX6-NEXT:    v_mov_b32_e32 v1, s7
4807; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4808; GFX6-NEXT:    s_waitcnt vmcnt(0)
4809; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4810; GFX6-NEXT:    s_endpgm
4811;
4812; GFX7-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
4813; GFX7:       ; %bb.0: ; %entry
4814; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4815; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4816; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4817; GFX7-NEXT:    s_add_u32 s4, s0, 16
4818; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4819; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4820; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4821; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4822; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4823; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4824; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4825; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4826; GFX7-NEXT:    s_waitcnt vmcnt(0)
4827; GFX7-NEXT:    flat_store_dword v[0:1], v2
4828; GFX7-NEXT:    s_endpgm
4829;
4830; GFX10-WGP-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
4831; GFX10-WGP:       ; %bb.0: ; %entry
4832; GFX10-WGP-NEXT:    s_clause 0x1
4833; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4834; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4835; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
4836; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4837; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4838; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4839; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4840; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4841; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
4842; GFX10-WGP-NEXT:    s_endpgm
4843;
4844; GFX10-CU-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
4845; GFX10-CU:       ; %bb.0: ; %entry
4846; GFX10-CU-NEXT:    s_clause 0x1
4847; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4848; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4849; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
4850; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4851; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4852; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4853; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4854; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4855; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
4856; GFX10-CU-NEXT:    s_endpgm
4857;
4858; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
4859; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4860; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
4861; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
4862; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
4863; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
4864; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4865; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4866; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4867; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4868; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4869; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4870; SKIP-CACHE-INV-NEXT:    s_endpgm
4871;
4872; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
4873; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4874; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4875; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4876; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4877; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4878; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4879; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4880; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4881; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4882; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4883;
4884; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
4885; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4886; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4887; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4888; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4889; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4890; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4891; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4892; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4893; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4894; GFX90A-TGSPLIT-NEXT:    s_endpgm
4895;
4896; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
4897; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4898; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4899; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4900; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4901; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4902; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4903; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4904; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4905; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4906; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4907;
4908; GFX940-TGSPLIT-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
4909; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4910; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4911; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4912; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4913; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4914; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4915; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4916; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4917; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4918; GFX940-TGSPLIT-NEXT:    s_endpgm
4919    i32 addrspace(1)* %out, i32 %in, i32 %old) {
4920entry:
4921  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
4922  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire seq_cst
4923  %val0 = extractvalue { i32, i1 } %val, 0
4924  store i32 %val0, i32 addrspace(1)* %out, align 4
4925  ret void
4926}
4927
4928define amdgpu_kernel void @global_singlethread_release_seq_cst_ret_cmpxchg(
4929; GFX6-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
4930; GFX6:       ; %bb.0: ; %entry
4931; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
4932; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4933; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4934; GFX6-NEXT:    s_mov_b32 s2, -1
4935; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4936; GFX6-NEXT:    v_mov_b32_e32 v0, s6
4937; GFX6-NEXT:    v_mov_b32_e32 v1, s7
4938; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4939; GFX6-NEXT:    s_waitcnt vmcnt(0)
4940; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4941; GFX6-NEXT:    s_endpgm
4942;
4943; GFX7-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
4944; GFX7:       ; %bb.0: ; %entry
4945; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4946; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4947; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4948; GFX7-NEXT:    s_add_u32 s4, s0, 16
4949; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4950; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4951; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4952; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4953; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4954; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4955; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4956; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4957; GFX7-NEXT:    s_waitcnt vmcnt(0)
4958; GFX7-NEXT:    flat_store_dword v[0:1], v2
4959; GFX7-NEXT:    s_endpgm
4960;
4961; GFX10-WGP-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
4962; GFX10-WGP:       ; %bb.0: ; %entry
4963; GFX10-WGP-NEXT:    s_clause 0x1
4964; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4965; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4966; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
4967; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4968; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4969; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4970; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4971; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4972; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
4973; GFX10-WGP-NEXT:    s_endpgm
4974;
4975; GFX10-CU-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
4976; GFX10-CU:       ; %bb.0: ; %entry
4977; GFX10-CU-NEXT:    s_clause 0x1
4978; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4979; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4980; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
4981; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4982; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4983; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4984; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4985; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4986; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
4987; GFX10-CU-NEXT:    s_endpgm
4988;
4989; SKIP-CACHE-INV-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
4990; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4991; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
4992; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
4993; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
4994; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
4995; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4996; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4997; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4998; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4999; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5000; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5001; SKIP-CACHE-INV-NEXT:    s_endpgm
5002;
5003; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
5004; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5005; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5006; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5007; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5008; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5009; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5010; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5011; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5012; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
5013; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5014;
5015; GFX90A-TGSPLIT-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
5016; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5017; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5018; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5019; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5020; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5021; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5022; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5023; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5024; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
5025; GFX90A-TGSPLIT-NEXT:    s_endpgm
5026;
5027; GFX940-NOTTGSPLIT-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
5028; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5029; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5030; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
5031; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5032; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5033; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5034; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
5035; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5036; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
5037; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5038;
5039; GFX940-TGSPLIT-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
5040; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5041; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5042; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
5043; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5044; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5045; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5046; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
5047; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5048; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
5049; GFX940-TGSPLIT-NEXT:    s_endpgm
5050    i32 addrspace(1)* %out, i32 %in, i32 %old) {
5051entry:
5052  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
5053  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") release seq_cst
5054  %val0 = extractvalue { i32, i1 } %val, 0
5055  store i32 %val0, i32 addrspace(1)* %out, align 4
5056  ret void
5057}
5058
5059define amdgpu_kernel void @global_singlethread_acq_rel_seq_cst_ret_cmpxchg(
5060; GFX6-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
5061; GFX6:       ; %bb.0: ; %entry
5062; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
5063; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5064; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5065; GFX6-NEXT:    s_mov_b32 s2, -1
5066; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5067; GFX6-NEXT:    v_mov_b32_e32 v0, s6
5068; GFX6-NEXT:    v_mov_b32_e32 v1, s7
5069; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
5070; GFX6-NEXT:    s_waitcnt vmcnt(0)
5071; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5072; GFX6-NEXT:    s_endpgm
5073;
5074; GFX7-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
5075; GFX7:       ; %bb.0: ; %entry
5076; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5077; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5078; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5079; GFX7-NEXT:    s_add_u32 s4, s0, 16
5080; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5081; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5082; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5083; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5084; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5085; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5086; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5087; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5088; GFX7-NEXT:    s_waitcnt vmcnt(0)
5089; GFX7-NEXT:    flat_store_dword v[0:1], v2
5090; GFX7-NEXT:    s_endpgm
5091;
5092; GFX10-WGP-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
5093; GFX10-WGP:       ; %bb.0: ; %entry
5094; GFX10-WGP-NEXT:    s_clause 0x1
5095; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5096; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5097; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
5098; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5099; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5100; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5101; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5102; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5103; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
5104; GFX10-WGP-NEXT:    s_endpgm
5105;
5106; GFX10-CU-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
5107; GFX10-CU:       ; %bb.0: ; %entry
5108; GFX10-CU-NEXT:    s_clause 0x1
5109; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5110; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5111; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
5112; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5113; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5114; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5115; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5116; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5117; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
5118; GFX10-CU-NEXT:    s_endpgm
5119;
5120; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
5121; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5122; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
5123; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
5124; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
5125; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
5126; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5127; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5128; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5129; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
5130; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5131; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5132; SKIP-CACHE-INV-NEXT:    s_endpgm
5133;
5134; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
5135; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5136; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5137; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5138; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5139; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5140; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5141; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5142; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5143; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
5144; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5145;
5146; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
5147; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5148; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5149; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5150; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5151; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5152; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5153; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5154; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5155; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
5156; GFX90A-TGSPLIT-NEXT:    s_endpgm
5157;
5158; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
5159; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5160; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5161; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
5162; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5163; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5164; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5165; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
5166; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5167; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
5168; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5169;
5170; GFX940-TGSPLIT-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
5171; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5172; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5173; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
5174; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5175; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5176; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5177; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
5178; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5179; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
5180; GFX940-TGSPLIT-NEXT:    s_endpgm
5181    i32 addrspace(1)* %out, i32 %in, i32 %old) {
5182entry:
5183  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
5184  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel seq_cst
5185  %val0 = extractvalue { i32, i1 } %val, 0
5186  store i32 %val0, i32 addrspace(1)* %out, align 4
5187  ret void
5188}
5189
5190define amdgpu_kernel void @global_singlethread_seq_cst_seq_cst_ret_cmpxchg(
5191; GFX6-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5192; GFX6:       ; %bb.0: ; %entry
5193; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
5194; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5195; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5196; GFX6-NEXT:    s_mov_b32 s2, -1
5197; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5198; GFX6-NEXT:    v_mov_b32_e32 v0, s6
5199; GFX6-NEXT:    v_mov_b32_e32 v1, s7
5200; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
5201; GFX6-NEXT:    s_waitcnt vmcnt(0)
5202; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5203; GFX6-NEXT:    s_endpgm
5204;
5205; GFX7-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5206; GFX7:       ; %bb.0: ; %entry
5207; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5208; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5209; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5210; GFX7-NEXT:    s_add_u32 s4, s0, 16
5211; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5212; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5213; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5214; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5215; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5216; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5217; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5218; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5219; GFX7-NEXT:    s_waitcnt vmcnt(0)
5220; GFX7-NEXT:    flat_store_dword v[0:1], v2
5221; GFX7-NEXT:    s_endpgm
5222;
5223; GFX10-WGP-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5224; GFX10-WGP:       ; %bb.0: ; %entry
5225; GFX10-WGP-NEXT:    s_clause 0x1
5226; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5227; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5228; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
5229; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5230; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5231; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5232; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5233; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5234; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
5235; GFX10-WGP-NEXT:    s_endpgm
5236;
5237; GFX10-CU-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5238; GFX10-CU:       ; %bb.0: ; %entry
5239; GFX10-CU-NEXT:    s_clause 0x1
5240; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5241; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5242; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
5243; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5244; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5245; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5246; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5247; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5248; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
5249; GFX10-CU-NEXT:    s_endpgm
5250;
5251; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5252; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5253; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
5254; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
5255; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
5256; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
5257; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5258; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5259; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5260; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
5261; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5262; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5263; SKIP-CACHE-INV-NEXT:    s_endpgm
5264;
5265; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5266; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5267; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5268; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5269; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5270; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5271; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5272; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5273; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5274; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
5275; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5276;
5277; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5278; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5279; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5280; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5281; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5282; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5283; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5284; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5285; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5286; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
5287; GFX90A-TGSPLIT-NEXT:    s_endpgm
5288;
5289; GFX940-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5290; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5291; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5292; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
5293; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5294; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5295; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5296; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
5297; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5298; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
5299; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5300;
5301; GFX940-TGSPLIT-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5302; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5303; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5304; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
5305; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5306; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5307; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5308; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
5309; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5310; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
5311; GFX940-TGSPLIT-NEXT:    s_endpgm
5312    i32 addrspace(1)* %out, i32 %in, i32 %old) {
5313entry:
5314  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
5315  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst seq_cst
5316  %val0 = extractvalue { i32, i1 } %val, 0
5317  store i32 %val0, i32 addrspace(1)* %out, align 4
5318  ret void
5319}
5320
5321define amdgpu_kernel void @global_singlethread_one_as_unordered_load(
5322; GFX6-LABEL: global_singlethread_one_as_unordered_load:
5323; GFX6:       ; %bb.0: ; %entry
5324; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5325; GFX6-NEXT:    s_mov_b32 s7, 0x100f000
5326; GFX6-NEXT:    s_mov_b32 s6, -1
5327; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5328; GFX6-NEXT:    s_mov_b32 s4, s0
5329; GFX6-NEXT:    s_mov_b32 s5, s1
5330; GFX6-NEXT:    buffer_load_dword v0, off, s[4:7], 0
5331; GFX6-NEXT:    s_mov_b32 s4, s2
5332; GFX6-NEXT:    s_mov_b32 s5, s3
5333; GFX6-NEXT:    s_waitcnt vmcnt(0)
5334; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
5335; GFX6-NEXT:    s_endpgm
5336;
5337; GFX7-LABEL: global_singlethread_one_as_unordered_load:
5338; GFX7:       ; %bb.0: ; %entry
5339; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5340; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5341; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5342; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5343; GFX7-NEXT:    flat_load_dword v2, v[0:1]
5344; GFX7-NEXT:    v_mov_b32_e32 v0, s2
5345; GFX7-NEXT:    v_mov_b32_e32 v1, s3
5346; GFX7-NEXT:    s_waitcnt vmcnt(0)
5347; GFX7-NEXT:    flat_store_dword v[0:1], v2
5348; GFX7-NEXT:    s_endpgm
5349;
5350; GFX10-WGP-LABEL: global_singlethread_one_as_unordered_load:
5351; GFX10-WGP:       ; %bb.0: ; %entry
5352; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5353; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
5354; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5355; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
5356; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5357; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
5358; GFX10-WGP-NEXT:    s_endpgm
5359;
5360; GFX10-CU-LABEL: global_singlethread_one_as_unordered_load:
5361; GFX10-CU:       ; %bb.0: ; %entry
5362; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5363; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
5364; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5365; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
5366; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5367; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
5368; GFX10-CU-NEXT:    s_endpgm
5369;
5370; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_unordered_load:
5371; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5372; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5373; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
5374; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
5375; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5376; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s0
5377; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s1
5378; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[4:7], 0
5379; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s2
5380; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s3
5381; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5382; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
5383; SKIP-CACHE-INV-NEXT:    s_endpgm
5384;
5385; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_unordered_load:
5386; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5387; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5388; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5389; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5390; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
5391; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5392; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
5393; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5394;
5395; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_unordered_load:
5396; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5397; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5398; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5399; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5400; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
5401; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5402; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
5403; GFX90A-TGSPLIT-NEXT:    s_endpgm
5404;
5405; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_unordered_load:
5406; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5407; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5408; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5409; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5410; GFX940-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
5411; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5412; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
5413; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5414;
5415; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_unordered_load:
5416; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5417; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5418; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5419; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5420; GFX940-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
5421; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5422; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
5423; GFX940-TGSPLIT-NEXT:    s_endpgm
5424    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
5425entry:
5426  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread-one-as") unordered, align 4
5427  store i32 %val, i32 addrspace(1)* %out
5428  ret void
5429}
5430
5431define amdgpu_kernel void @global_singlethread_one_as_monotonic_load(
5432; GFX6-LABEL: global_singlethread_one_as_monotonic_load:
5433; GFX6:       ; %bb.0: ; %entry
5434; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5435; GFX6-NEXT:    s_mov_b32 s7, 0x100f000
5436; GFX6-NEXT:    s_mov_b32 s6, -1
5437; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5438; GFX6-NEXT:    s_mov_b32 s4, s0
5439; GFX6-NEXT:    s_mov_b32 s5, s1
5440; GFX6-NEXT:    buffer_load_dword v0, off, s[4:7], 0
5441; GFX6-NEXT:    s_mov_b32 s4, s2
5442; GFX6-NEXT:    s_mov_b32 s5, s3
5443; GFX6-NEXT:    s_waitcnt vmcnt(0)
5444; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
5445; GFX6-NEXT:    s_endpgm
5446;
5447; GFX7-LABEL: global_singlethread_one_as_monotonic_load:
5448; GFX7:       ; %bb.0: ; %entry
5449; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5450; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5451; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5452; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5453; GFX7-NEXT:    flat_load_dword v2, v[0:1]
5454; GFX7-NEXT:    v_mov_b32_e32 v0, s2
5455; GFX7-NEXT:    v_mov_b32_e32 v1, s3
5456; GFX7-NEXT:    s_waitcnt vmcnt(0)
5457; GFX7-NEXT:    flat_store_dword v[0:1], v2
5458; GFX7-NEXT:    s_endpgm
5459;
5460; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_load:
5461; GFX10-WGP:       ; %bb.0: ; %entry
5462; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5463; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
5464; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5465; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
5466; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5467; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
5468; GFX10-WGP-NEXT:    s_endpgm
5469;
5470; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_load:
5471; GFX10-CU:       ; %bb.0: ; %entry
5472; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5473; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
5474; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5475; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
5476; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5477; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
5478; GFX10-CU-NEXT:    s_endpgm
5479;
5480; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_load:
5481; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5482; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5483; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
5484; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
5485; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5486; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s0
5487; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s1
5488; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[4:7], 0
5489; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s2
5490; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s3
5491; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5492; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
5493; SKIP-CACHE-INV-NEXT:    s_endpgm
5494;
5495; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_load:
5496; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5497; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5498; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5499; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5500; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
5501; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5502; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
5503; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5504;
5505; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_load:
5506; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5507; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5508; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5509; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5510; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
5511; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5512; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
5513; GFX90A-TGSPLIT-NEXT:    s_endpgm
5514;
5515; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_load:
5516; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5517; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5518; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5519; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5520; GFX940-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
5521; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5522; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
5523; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5524;
5525; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_load:
5526; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5527; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5528; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5529; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5530; GFX940-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
5531; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5532; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
5533; GFX940-TGSPLIT-NEXT:    s_endpgm
5534    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
5535entry:
5536  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread-one-as") monotonic, align 4
5537  store i32 %val, i32 addrspace(1)* %out
5538  ret void
5539}
5540
5541define amdgpu_kernel void @global_singlethread_one_as_acquire_load(
5542; GFX6-LABEL: global_singlethread_one_as_acquire_load:
5543; GFX6:       ; %bb.0: ; %entry
5544; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5545; GFX6-NEXT:    s_mov_b32 s7, 0x100f000
5546; GFX6-NEXT:    s_mov_b32 s6, -1
5547; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5548; GFX6-NEXT:    s_mov_b32 s4, s0
5549; GFX6-NEXT:    s_mov_b32 s5, s1
5550; GFX6-NEXT:    buffer_load_dword v0, off, s[4:7], 0
5551; GFX6-NEXT:    s_mov_b32 s4, s2
5552; GFX6-NEXT:    s_mov_b32 s5, s3
5553; GFX6-NEXT:    s_waitcnt vmcnt(0)
5554; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
5555; GFX6-NEXT:    s_endpgm
5556;
5557; GFX7-LABEL: global_singlethread_one_as_acquire_load:
5558; GFX7:       ; %bb.0: ; %entry
5559; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5560; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5561; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5562; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5563; GFX7-NEXT:    flat_load_dword v2, v[0:1]
5564; GFX7-NEXT:    v_mov_b32_e32 v0, s2
5565; GFX7-NEXT:    v_mov_b32_e32 v1, s3
5566; GFX7-NEXT:    s_waitcnt vmcnt(0)
5567; GFX7-NEXT:    flat_store_dword v[0:1], v2
5568; GFX7-NEXT:    s_endpgm
5569;
5570; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_load:
5571; GFX10-WGP:       ; %bb.0: ; %entry
5572; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5573; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
5574; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5575; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
5576; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5577; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
5578; GFX10-WGP-NEXT:    s_endpgm
5579;
5580; GFX10-CU-LABEL: global_singlethread_one_as_acquire_load:
5581; GFX10-CU:       ; %bb.0: ; %entry
5582; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5583; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
5584; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5585; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
5586; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5587; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
5588; GFX10-CU-NEXT:    s_endpgm
5589;
5590; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_load:
5591; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5592; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5593; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
5594; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
5595; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5596; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s0
5597; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s1
5598; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[4:7], 0
5599; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s2
5600; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s3
5601; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5602; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
5603; SKIP-CACHE-INV-NEXT:    s_endpgm
5604;
5605; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_load:
5606; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5607; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5608; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5609; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5610; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
5611; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5612; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
5613; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5614;
5615; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_load:
5616; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5617; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5618; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5619; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5620; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
5621; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5622; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
5623; GFX90A-TGSPLIT-NEXT:    s_endpgm
5624;
5625; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_load:
5626; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5627; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5628; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5629; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5630; GFX940-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
5631; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5632; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
5633; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5634;
5635; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acquire_load:
5636; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5637; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5638; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5639; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5640; GFX940-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
5641; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5642; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
5643; GFX940-TGSPLIT-NEXT:    s_endpgm
5644    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
5645entry:
5646  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread-one-as") acquire, align 4
5647  store i32 %val, i32 addrspace(1)* %out
5648  ret void
5649}
5650
5651define amdgpu_kernel void @global_singlethread_one_as_seq_cst_load(
5652; GFX6-LABEL: global_singlethread_one_as_seq_cst_load:
5653; GFX6:       ; %bb.0: ; %entry
5654; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5655; GFX6-NEXT:    s_mov_b32 s7, 0x100f000
5656; GFX6-NEXT:    s_mov_b32 s6, -1
5657; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5658; GFX6-NEXT:    s_mov_b32 s4, s0
5659; GFX6-NEXT:    s_mov_b32 s5, s1
5660; GFX6-NEXT:    buffer_load_dword v0, off, s[4:7], 0
5661; GFX6-NEXT:    s_mov_b32 s4, s2
5662; GFX6-NEXT:    s_mov_b32 s5, s3
5663; GFX6-NEXT:    s_waitcnt vmcnt(0)
5664; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
5665; GFX6-NEXT:    s_endpgm
5666;
5667; GFX7-LABEL: global_singlethread_one_as_seq_cst_load:
5668; GFX7:       ; %bb.0: ; %entry
5669; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5670; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5671; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5672; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5673; GFX7-NEXT:    flat_load_dword v2, v[0:1]
5674; GFX7-NEXT:    v_mov_b32_e32 v0, s2
5675; GFX7-NEXT:    v_mov_b32_e32 v1, s3
5676; GFX7-NEXT:    s_waitcnt vmcnt(0)
5677; GFX7-NEXT:    flat_store_dword v[0:1], v2
5678; GFX7-NEXT:    s_endpgm
5679;
5680; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_load:
5681; GFX10-WGP:       ; %bb.0: ; %entry
5682; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5683; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
5684; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5685; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
5686; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5687; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
5688; GFX10-WGP-NEXT:    s_endpgm
5689;
5690; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_load:
5691; GFX10-CU:       ; %bb.0: ; %entry
5692; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5693; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
5694; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5695; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
5696; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5697; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
5698; GFX10-CU-NEXT:    s_endpgm
5699;
5700; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_load:
5701; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5702; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5703; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
5704; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
5705; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5706; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s0
5707; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s1
5708; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[4:7], 0
5709; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s2
5710; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s3
5711; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5712; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
5713; SKIP-CACHE-INV-NEXT:    s_endpgm
5714;
5715; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_load:
5716; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5717; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5718; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5719; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5720; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
5721; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5722; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
5723; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5724;
5725; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_load:
5726; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5727; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5728; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5729; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5730; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
5731; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5732; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
5733; GFX90A-TGSPLIT-NEXT:    s_endpgm
5734;
5735; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_load:
5736; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5737; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5738; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5739; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5740; GFX940-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
5741; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5742; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
5743; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5744;
5745; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_load:
5746; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5747; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5748; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5749; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5750; GFX940-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
5751; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5752; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
5753; GFX940-TGSPLIT-NEXT:    s_endpgm
5754    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
5755entry:
5756  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread-one-as") seq_cst, align 4
5757  store i32 %val, i32 addrspace(1)* %out
5758  ret void
5759}
5760
5761define amdgpu_kernel void @global_singlethread_one_as_unordered_store(
5762; GFX6-LABEL: global_singlethread_one_as_unordered_store:
5763; GFX6:       ; %bb.0: ; %entry
5764; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
5765; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
5766; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5767; GFX6-NEXT:    s_mov_b32 s2, -1
5768; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5769; GFX6-NEXT:    v_mov_b32_e32 v0, s6
5770; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5771; GFX6-NEXT:    s_endpgm
5772;
5773; GFX7-LABEL: global_singlethread_one_as_unordered_store:
5774; GFX7:       ; %bb.0: ; %entry
5775; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
5776; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
5777; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5778; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5779; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5780; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5781; GFX7-NEXT:    flat_store_dword v[0:1], v2
5782; GFX7-NEXT:    s_endpgm
5783;
5784; GFX10-WGP-LABEL: global_singlethread_one_as_unordered_store:
5785; GFX10-WGP:       ; %bb.0: ; %entry
5786; GFX10-WGP-NEXT:    s_clause 0x1
5787; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
5788; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5789; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
5790; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5791; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5792; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
5793; GFX10-WGP-NEXT:    s_endpgm
5794;
5795; GFX10-CU-LABEL: global_singlethread_one_as_unordered_store:
5796; GFX10-CU:       ; %bb.0: ; %entry
5797; GFX10-CU-NEXT:    s_clause 0x1
5798; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
5799; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5800; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
5801; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5802; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5803; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
5804; GFX10-CU-NEXT:    s_endpgm
5805;
5806; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_unordered_store:
5807; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5808; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x0
5809; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5810; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
5811; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
5812; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5813; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5814; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5815; SKIP-CACHE-INV-NEXT:    s_endpgm
5816;
5817; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_unordered_store:
5818; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5819; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
5820; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5821; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5822; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5823; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5824; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
5825; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5826;
5827; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_unordered_store:
5828; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5829; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
5830; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5831; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5832; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5833; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5834; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
5835; GFX90A-TGSPLIT-NEXT:    s_endpgm
5836;
5837; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_unordered_store:
5838; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5839; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
5840; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5841; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5842; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5843; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
5844; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
5845; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5846;
5847; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_unordered_store:
5848; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5849; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
5850; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5851; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5852; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5853; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
5854; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
5855; GFX940-TGSPLIT-NEXT:    s_endpgm
5856    i32 %in, i32 addrspace(1)* %out) {
5857entry:
5858  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread-one-as") unordered, align 4
5859  ret void
5860}
5861
5862define amdgpu_kernel void @global_singlethread_one_as_monotonic_store(
5863; GFX6-LABEL: global_singlethread_one_as_monotonic_store:
5864; GFX6:       ; %bb.0: ; %entry
5865; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
5866; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
5867; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5868; GFX6-NEXT:    s_mov_b32 s2, -1
5869; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5870; GFX6-NEXT:    v_mov_b32_e32 v0, s6
5871; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5872; GFX6-NEXT:    s_endpgm
5873;
5874; GFX7-LABEL: global_singlethread_one_as_monotonic_store:
5875; GFX7:       ; %bb.0: ; %entry
5876; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
5877; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
5878; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5879; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5880; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5881; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5882; GFX7-NEXT:    flat_store_dword v[0:1], v2
5883; GFX7-NEXT:    s_endpgm
5884;
5885; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_store:
5886; GFX10-WGP:       ; %bb.0: ; %entry
5887; GFX10-WGP-NEXT:    s_clause 0x1
5888; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
5889; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5890; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
5891; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5892; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5893; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
5894; GFX10-WGP-NEXT:    s_endpgm
5895;
5896; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_store:
5897; GFX10-CU:       ; %bb.0: ; %entry
5898; GFX10-CU-NEXT:    s_clause 0x1
5899; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
5900; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5901; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
5902; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5903; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5904; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
5905; GFX10-CU-NEXT:    s_endpgm
5906;
5907; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_store:
5908; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5909; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x0
5910; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5911; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
5912; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
5913; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5914; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5915; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5916; SKIP-CACHE-INV-NEXT:    s_endpgm
5917;
5918; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_store:
5919; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5920; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
5921; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5922; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5923; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5924; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5925; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
5926; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5927;
5928; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_store:
5929; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5930; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
5931; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5932; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5933; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5934; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5935; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
5936; GFX90A-TGSPLIT-NEXT:    s_endpgm
5937;
5938; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_store:
5939; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5940; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
5941; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5942; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5943; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5944; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
5945; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
5946; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5947;
5948; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_store:
5949; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5950; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
5951; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5952; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5953; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5954; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
5955; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
5956; GFX940-TGSPLIT-NEXT:    s_endpgm
5957    i32 %in, i32 addrspace(1)* %out) {
5958entry:
5959  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread-one-as") monotonic, align 4
5960  ret void
5961}
5962
5963define amdgpu_kernel void @global_singlethread_one_as_release_store(
5964; GFX6-LABEL: global_singlethread_one_as_release_store:
5965; GFX6:       ; %bb.0: ; %entry
5966; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
5967; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
5968; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5969; GFX6-NEXT:    s_mov_b32 s2, -1
5970; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5971; GFX6-NEXT:    v_mov_b32_e32 v0, s6
5972; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5973; GFX6-NEXT:    s_endpgm
5974;
5975; GFX7-LABEL: global_singlethread_one_as_release_store:
5976; GFX7:       ; %bb.0: ; %entry
5977; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
5978; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
5979; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5980; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5981; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5982; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5983; GFX7-NEXT:    flat_store_dword v[0:1], v2
5984; GFX7-NEXT:    s_endpgm
5985;
5986; GFX10-WGP-LABEL: global_singlethread_one_as_release_store:
5987; GFX10-WGP:       ; %bb.0: ; %entry
5988; GFX10-WGP-NEXT:    s_clause 0x1
5989; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
5990; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5991; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
5992; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5993; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5994; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
5995; GFX10-WGP-NEXT:    s_endpgm
5996;
5997; GFX10-CU-LABEL: global_singlethread_one_as_release_store:
5998; GFX10-CU:       ; %bb.0: ; %entry
5999; GFX10-CU-NEXT:    s_clause 0x1
6000; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
6001; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6002; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
6003; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6004; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
6005; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
6006; GFX10-CU-NEXT:    s_endpgm
6007;
6008; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_store:
6009; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6010; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x0
6011; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6012; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
6013; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
6014; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6015; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6016; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
6017; SKIP-CACHE-INV-NEXT:    s_endpgm
6018;
6019; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_store:
6020; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6021; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
6022; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6023; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6024; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6025; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6026; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
6027; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6028;
6029; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_release_store:
6030; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6031; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
6032; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6033; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6034; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6035; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6036; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
6037; GFX90A-TGSPLIT-NEXT:    s_endpgm
6038;
6039; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_store:
6040; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6041; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
6042; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
6043; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6044; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6045; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
6046; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6047; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6048;
6049; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_release_store:
6050; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6051; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
6052; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
6053; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6054; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6055; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
6056; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6057; GFX940-TGSPLIT-NEXT:    s_endpgm
6058    i32 %in, i32 addrspace(1)* %out) {
6059entry:
6060  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread-one-as") release, align 4
6061  ret void
6062}
6063
6064define amdgpu_kernel void @global_singlethread_one_as_seq_cst_store(
6065; GFX6-LABEL: global_singlethread_one_as_seq_cst_store:
6066; GFX6:       ; %bb.0: ; %entry
6067; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
6068; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
6069; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
6070; GFX6-NEXT:    s_mov_b32 s2, -1
6071; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6072; GFX6-NEXT:    v_mov_b32_e32 v0, s6
6073; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
6074; GFX6-NEXT:    s_endpgm
6075;
6076; GFX7-LABEL: global_singlethread_one_as_seq_cst_store:
6077; GFX7:       ; %bb.0: ; %entry
6078; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
6079; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
6080; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6081; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6082; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6083; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6084; GFX7-NEXT:    flat_store_dword v[0:1], v2
6085; GFX7-NEXT:    s_endpgm
6086;
6087; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_store:
6088; GFX10-WGP:       ; %bb.0: ; %entry
6089; GFX10-WGP-NEXT:    s_clause 0x1
6090; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
6091; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6092; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
6093; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6094; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
6095; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
6096; GFX10-WGP-NEXT:    s_endpgm
6097;
6098; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_store:
6099; GFX10-CU:       ; %bb.0: ; %entry
6100; GFX10-CU-NEXT:    s_clause 0x1
6101; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
6102; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6103; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
6104; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6105; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
6106; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
6107; GFX10-CU-NEXT:    s_endpgm
6108;
6109; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_store:
6110; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6111; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x0
6112; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6113; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
6114; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
6115; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6116; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6117; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
6118; SKIP-CACHE-INV-NEXT:    s_endpgm
6119;
6120; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_store:
6121; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6122; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
6123; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6124; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6125; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6126; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6127; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
6128; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6129;
6130; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_store:
6131; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6132; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
6133; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6134; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6135; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6136; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6137; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
6138; GFX90A-TGSPLIT-NEXT:    s_endpgm
6139;
6140; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_store:
6141; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6142; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
6143; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
6144; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6145; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6146; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
6147; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6148; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6149;
6150; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_store:
6151; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6152; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
6153; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
6154; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6155; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6156; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
6157; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6158; GFX940-TGSPLIT-NEXT:    s_endpgm
6159    i32 %in, i32 addrspace(1)* %out) {
6160entry:
6161  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread-one-as") seq_cst, align 4
6162  ret void
6163}
6164
6165define amdgpu_kernel void @global_singlethread_one_as_monotonic_atomicrmw(
6166; GFX6-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
6167; GFX6:       ; %bb.0: ; %entry
6168; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
6169; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6170; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
6171; GFX6-NEXT:    s_mov_b32 s2, -1
6172; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6173; GFX6-NEXT:    v_mov_b32_e32 v0, s6
6174; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
6175; GFX6-NEXT:    s_endpgm
6176;
6177; GFX7-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
6178; GFX7:       ; %bb.0: ; %entry
6179; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6180; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
6181; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6182; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6183; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6184; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6185; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
6186; GFX7-NEXT:    s_endpgm
6187;
6188; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
6189; GFX10-WGP:       ; %bb.0: ; %entry
6190; GFX10-WGP-NEXT:    s_clause 0x1
6191; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
6192; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6193; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
6194; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6195; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
6196; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
6197; GFX10-WGP-NEXT:    s_endpgm
6198;
6199; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
6200; GFX10-CU:       ; %bb.0: ; %entry
6201; GFX10-CU-NEXT:    s_clause 0x1
6202; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
6203; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6204; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
6205; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6206; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
6207; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
6208; GFX10-CU-NEXT:    s_endpgm
6209;
6210; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
6211; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6212; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
6213; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6214; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
6215; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
6216; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6217; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6218; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
6219; SKIP-CACHE-INV-NEXT:    s_endpgm
6220;
6221; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
6222; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6223; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6224; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6225; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6226; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6227; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6228; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
6229; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6230;
6231; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
6232; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6233; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6234; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6235; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6236; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6237; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6238; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
6239; GFX90A-TGSPLIT-NEXT:    s_endpgm
6240;
6241; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
6242; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6243; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
6244; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6245; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6246; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6247; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
6248; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
6249; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6250;
6251; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
6252; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6253; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
6254; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6255; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6256; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6257; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
6258; GFX940-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
6259; GFX940-TGSPLIT-NEXT:    s_endpgm
6260    i32 addrspace(1)* %out, i32 %in) {
6261entry:
6262  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") monotonic
6263  ret void
6264}
6265
6266define amdgpu_kernel void @global_singlethread_one_as_acquire_atomicrmw(
6267; GFX6-LABEL: global_singlethread_one_as_acquire_atomicrmw:
6268; GFX6:       ; %bb.0: ; %entry
6269; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
6270; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6271; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
6272; GFX6-NEXT:    s_mov_b32 s2, -1
6273; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6274; GFX6-NEXT:    v_mov_b32_e32 v0, s6
6275; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
6276; GFX6-NEXT:    s_endpgm
6277;
6278; GFX7-LABEL: global_singlethread_one_as_acquire_atomicrmw:
6279; GFX7:       ; %bb.0: ; %entry
6280; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6281; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
6282; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6283; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6284; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6285; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6286; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
6287; GFX7-NEXT:    s_endpgm
6288;
6289; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_atomicrmw:
6290; GFX10-WGP:       ; %bb.0: ; %entry
6291; GFX10-WGP-NEXT:    s_clause 0x1
6292; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
6293; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6294; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
6295; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6296; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
6297; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
6298; GFX10-WGP-NEXT:    s_endpgm
6299;
6300; GFX10-CU-LABEL: global_singlethread_one_as_acquire_atomicrmw:
6301; GFX10-CU:       ; %bb.0: ; %entry
6302; GFX10-CU-NEXT:    s_clause 0x1
6303; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
6304; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6305; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
6306; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6307; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
6308; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
6309; GFX10-CU-NEXT:    s_endpgm
6310;
6311; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_atomicrmw:
6312; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6313; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
6314; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6315; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
6316; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
6317; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6318; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6319; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
6320; SKIP-CACHE-INV-NEXT:    s_endpgm
6321;
6322; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_atomicrmw:
6323; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6324; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6325; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6326; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6327; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6328; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6329; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
6330; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6331;
6332; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_atomicrmw:
6333; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6334; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6335; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6336; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6337; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6338; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6339; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
6340; GFX90A-TGSPLIT-NEXT:    s_endpgm
6341;
6342; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_atomicrmw:
6343; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6344; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
6345; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6346; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6347; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6348; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
6349; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
6350; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6351;
6352; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acquire_atomicrmw:
6353; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6354; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
6355; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6356; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6357; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6358; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
6359; GFX940-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
6360; GFX940-TGSPLIT-NEXT:    s_endpgm
6361    i32 addrspace(1)* %out, i32 %in) {
6362entry:
6363  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") acquire
6364  ret void
6365}
6366
6367define amdgpu_kernel void @global_singlethread_one_as_release_atomicrmw(
6368; GFX6-LABEL: global_singlethread_one_as_release_atomicrmw:
6369; GFX6:       ; %bb.0: ; %entry
6370; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
6371; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6372; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
6373; GFX6-NEXT:    s_mov_b32 s2, -1
6374; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6375; GFX6-NEXT:    v_mov_b32_e32 v0, s6
6376; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
6377; GFX6-NEXT:    s_endpgm
6378;
6379; GFX7-LABEL: global_singlethread_one_as_release_atomicrmw:
6380; GFX7:       ; %bb.0: ; %entry
6381; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6382; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
6383; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6384; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6385; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6386; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6387; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
6388; GFX7-NEXT:    s_endpgm
6389;
6390; GFX10-WGP-LABEL: global_singlethread_one_as_release_atomicrmw:
6391; GFX10-WGP:       ; %bb.0: ; %entry
6392; GFX10-WGP-NEXT:    s_clause 0x1
6393; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
6394; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6395; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
6396; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6397; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
6398; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
6399; GFX10-WGP-NEXT:    s_endpgm
6400;
6401; GFX10-CU-LABEL: global_singlethread_one_as_release_atomicrmw:
6402; GFX10-CU:       ; %bb.0: ; %entry
6403; GFX10-CU-NEXT:    s_clause 0x1
6404; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
6405; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6406; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
6407; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6408; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
6409; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
6410; GFX10-CU-NEXT:    s_endpgm
6411;
6412; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_atomicrmw:
6413; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6414; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
6415; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6416; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
6417; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
6418; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6419; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6420; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
6421; SKIP-CACHE-INV-NEXT:    s_endpgm
6422;
6423; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_atomicrmw:
6424; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6425; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6426; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6427; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6428; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6429; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6430; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
6431; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6432;
6433; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_release_atomicrmw:
6434; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6435; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6436; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6437; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6438; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6439; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6440; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
6441; GFX90A-TGSPLIT-NEXT:    s_endpgm
6442;
6443; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_atomicrmw:
6444; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6445; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
6446; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6447; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6448; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6449; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
6450; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
6451; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6452;
6453; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_release_atomicrmw:
6454; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6455; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
6456; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6457; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6458; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6459; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
6460; GFX940-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
6461; GFX940-TGSPLIT-NEXT:    s_endpgm
6462    i32 addrspace(1)* %out, i32 %in) {
6463entry:
6464  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") release
6465  ret void
6466}
6467
6468define amdgpu_kernel void @global_singlethread_one_as_acq_rel_atomicrmw(
6469; GFX6-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
6470; GFX6:       ; %bb.0: ; %entry
6471; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
6472; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6473; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
6474; GFX6-NEXT:    s_mov_b32 s2, -1
6475; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6476; GFX6-NEXT:    v_mov_b32_e32 v0, s6
6477; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
6478; GFX6-NEXT:    s_endpgm
6479;
6480; GFX7-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
6481; GFX7:       ; %bb.0: ; %entry
6482; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6483; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
6484; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6485; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6486; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6487; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6488; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
6489; GFX7-NEXT:    s_endpgm
6490;
6491; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
6492; GFX10-WGP:       ; %bb.0: ; %entry
6493; GFX10-WGP-NEXT:    s_clause 0x1
6494; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
6495; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6496; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
6497; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6498; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
6499; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
6500; GFX10-WGP-NEXT:    s_endpgm
6501;
6502; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
6503; GFX10-CU:       ; %bb.0: ; %entry
6504; GFX10-CU-NEXT:    s_clause 0x1
6505; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
6506; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6507; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
6508; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6509; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
6510; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
6511; GFX10-CU-NEXT:    s_endpgm
6512;
6513; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
6514; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6515; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
6516; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6517; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
6518; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
6519; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6520; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6521; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
6522; SKIP-CACHE-INV-NEXT:    s_endpgm
6523;
6524; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
6525; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6526; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6527; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6528; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6529; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6530; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6531; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
6532; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6533;
6534; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
6535; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6536; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6537; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6538; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6539; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6540; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6541; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
6542; GFX90A-TGSPLIT-NEXT:    s_endpgm
6543;
6544; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
6545; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6546; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
6547; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6548; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6549; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6550; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
6551; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
6552; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6553;
6554; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
6555; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6556; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
6557; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6558; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6559; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6560; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
6561; GFX940-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
6562; GFX940-TGSPLIT-NEXT:    s_endpgm
6563    i32 addrspace(1)* %out, i32 %in) {
6564entry:
6565  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") acq_rel
6566  ret void
6567}
6568
6569define amdgpu_kernel void @global_singlethread_one_as_seq_cst_atomicrmw(
6570; GFX6-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
6571; GFX6:       ; %bb.0: ; %entry
6572; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
6573; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6574; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
6575; GFX6-NEXT:    s_mov_b32 s2, -1
6576; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6577; GFX6-NEXT:    v_mov_b32_e32 v0, s6
6578; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
6579; GFX6-NEXT:    s_endpgm
6580;
6581; GFX7-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
6582; GFX7:       ; %bb.0: ; %entry
6583; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6584; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
6585; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6586; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6587; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6588; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6589; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
6590; GFX7-NEXT:    s_endpgm
6591;
6592; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
6593; GFX10-WGP:       ; %bb.0: ; %entry
6594; GFX10-WGP-NEXT:    s_clause 0x1
6595; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
6596; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6597; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
6598; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6599; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
6600; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
6601; GFX10-WGP-NEXT:    s_endpgm
6602;
6603; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
6604; GFX10-CU:       ; %bb.0: ; %entry
6605; GFX10-CU-NEXT:    s_clause 0x1
6606; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
6607; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6608; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
6609; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6610; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
6611; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
6612; GFX10-CU-NEXT:    s_endpgm
6613;
6614; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
6615; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6616; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
6617; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6618; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
6619; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
6620; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6621; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6622; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
6623; SKIP-CACHE-INV-NEXT:    s_endpgm
6624;
6625; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
6626; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6627; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6628; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6629; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6630; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6631; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6632; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
6633; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6634;
6635; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
6636; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6637; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6638; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6639; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6640; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6641; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6642; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
6643; GFX90A-TGSPLIT-NEXT:    s_endpgm
6644;
6645; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
6646; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6647; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
6648; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6649; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6650; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6651; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
6652; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
6653; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6654;
6655; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
6656; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6657; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
6658; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6659; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6660; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6661; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
6662; GFX940-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
6663; GFX940-TGSPLIT-NEXT:    s_endpgm
6664    i32 addrspace(1)* %out, i32 %in) {
6665entry:
6666  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") seq_cst
6667  ret void
6668}
6669
6670define amdgpu_kernel void @global_singlethread_one_as_acquire_ret_atomicrmw(
6671; GFX6-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
6672; GFX6:       ; %bb.0: ; %entry
6673; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
6674; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6675; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
6676; GFX6-NEXT:    s_mov_b32 s2, -1
6677; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6678; GFX6-NEXT:    v_mov_b32_e32 v0, s6
6679; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
6680; GFX6-NEXT:    s_waitcnt vmcnt(0)
6681; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
6682; GFX6-NEXT:    s_endpgm
6683;
6684; GFX7-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
6685; GFX7:       ; %bb.0: ; %entry
6686; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6687; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
6688; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6689; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6690; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6691; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6692; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
6693; GFX7-NEXT:    s_waitcnt vmcnt(0)
6694; GFX7-NEXT:    flat_store_dword v[0:1], v2
6695; GFX7-NEXT:    s_endpgm
6696;
6697; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
6698; GFX10-WGP:       ; %bb.0: ; %entry
6699; GFX10-WGP-NEXT:    s_clause 0x1
6700; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
6701; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6702; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
6703; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6704; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
6705; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
6706; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6707; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
6708; GFX10-WGP-NEXT:    s_endpgm
6709;
6710; GFX10-CU-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
6711; GFX10-CU:       ; %bb.0: ; %entry
6712; GFX10-CU-NEXT:    s_clause 0x1
6713; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
6714; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6715; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
6716; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6717; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
6718; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
6719; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6720; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
6721; GFX10-CU-NEXT:    s_endpgm
6722;
6723; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
6724; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6725; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
6726; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6727; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
6728; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
6729; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6730; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6731; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
6732; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6733; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
6734; SKIP-CACHE-INV-NEXT:    s_endpgm
6735;
6736; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
6737; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6738; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6739; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6740; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6741; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6742; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6743; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
6744; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6745; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
6746; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6747;
6748; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
6749; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6750; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6751; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6752; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6753; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6754; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6755; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
6756; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6757; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
6758; GFX90A-TGSPLIT-NEXT:    s_endpgm
6759;
6760; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
6761; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6762; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
6763; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6764; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6765; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6766; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
6767; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[2:3] sc0
6768; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6769; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6770; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6771;
6772; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
6773; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6774; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
6775; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6776; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6777; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6778; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
6779; GFX940-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[2:3] sc0
6780; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6781; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6782; GFX940-TGSPLIT-NEXT:    s_endpgm
6783    i32 addrspace(1)* %out, i32 %in) {
6784entry:
6785  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") acquire
6786  store i32 %val, i32 addrspace(1)* %out, align 4
6787  ret void
6788}
6789
6790define amdgpu_kernel void @global_singlethread_one_as_acq_rel_ret_atomicrmw(
6791; GFX6-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
6792; GFX6:       ; %bb.0: ; %entry
6793; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
6794; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6795; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
6796; GFX6-NEXT:    s_mov_b32 s2, -1
6797; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6798; GFX6-NEXT:    v_mov_b32_e32 v0, s6
6799; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
6800; GFX6-NEXT:    s_waitcnt vmcnt(0)
6801; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
6802; GFX6-NEXT:    s_endpgm
6803;
6804; GFX7-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
6805; GFX7:       ; %bb.0: ; %entry
6806; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6807; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
6808; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6809; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6810; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6811; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6812; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
6813; GFX7-NEXT:    s_waitcnt vmcnt(0)
6814; GFX7-NEXT:    flat_store_dword v[0:1], v2
6815; GFX7-NEXT:    s_endpgm
6816;
6817; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
6818; GFX10-WGP:       ; %bb.0: ; %entry
6819; GFX10-WGP-NEXT:    s_clause 0x1
6820; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
6821; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6822; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
6823; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6824; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
6825; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
6826; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6827; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
6828; GFX10-WGP-NEXT:    s_endpgm
6829;
6830; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
6831; GFX10-CU:       ; %bb.0: ; %entry
6832; GFX10-CU-NEXT:    s_clause 0x1
6833; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
6834; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6835; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
6836; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6837; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
6838; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
6839; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6840; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
6841; GFX10-CU-NEXT:    s_endpgm
6842;
6843; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
6844; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6845; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
6846; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6847; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
6848; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
6849; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6850; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6851; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
6852; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6853; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
6854; SKIP-CACHE-INV-NEXT:    s_endpgm
6855;
6856; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
6857; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6858; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6859; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6860; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6861; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6862; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6863; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
6864; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6865; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
6866; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6867;
6868; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
6869; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6870; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6871; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6872; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6873; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6874; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6875; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
6876; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6877; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
6878; GFX90A-TGSPLIT-NEXT:    s_endpgm
6879;
6880; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
6881; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6882; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
6883; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6884; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6885; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6886; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
6887; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[2:3] sc0
6888; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6889; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6890; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6891;
6892; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
6893; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6894; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
6895; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6896; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6897; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6898; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
6899; GFX940-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[2:3] sc0
6900; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6901; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6902; GFX940-TGSPLIT-NEXT:    s_endpgm
6903    i32 addrspace(1)* %out, i32 %in) {
6904entry:
6905  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") acq_rel
6906  store i32 %val, i32 addrspace(1)* %out, align 4
6907  ret void
6908}
6909
6910define amdgpu_kernel void @global_singlethread_one_as_seq_cst_ret_atomicrmw(
6911; GFX6-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
6912; GFX6:       ; %bb.0: ; %entry
6913; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
6914; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6915; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
6916; GFX6-NEXT:    s_mov_b32 s2, -1
6917; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6918; GFX6-NEXT:    v_mov_b32_e32 v0, s6
6919; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
6920; GFX6-NEXT:    s_waitcnt vmcnt(0)
6921; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
6922; GFX6-NEXT:    s_endpgm
6923;
6924; GFX7-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
6925; GFX7:       ; %bb.0: ; %entry
6926; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6927; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
6928; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6929; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6930; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6931; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6932; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
6933; GFX7-NEXT:    s_waitcnt vmcnt(0)
6934; GFX7-NEXT:    flat_store_dword v[0:1], v2
6935; GFX7-NEXT:    s_endpgm
6936;
6937; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
6938; GFX10-WGP:       ; %bb.0: ; %entry
6939; GFX10-WGP-NEXT:    s_clause 0x1
6940; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
6941; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6942; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
6943; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6944; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
6945; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
6946; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6947; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
6948; GFX10-WGP-NEXT:    s_endpgm
6949;
6950; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
6951; GFX10-CU:       ; %bb.0: ; %entry
6952; GFX10-CU-NEXT:    s_clause 0x1
6953; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
6954; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6955; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
6956; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6957; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
6958; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
6959; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6960; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
6961; GFX10-CU-NEXT:    s_endpgm
6962;
6963; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
6964; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6965; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
6966; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6967; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
6968; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
6969; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6970; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6971; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
6972; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6973; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
6974; SKIP-CACHE-INV-NEXT:    s_endpgm
6975;
6976; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
6977; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6978; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6979; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6980; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6981; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6982; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6983; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
6984; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6985; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
6986; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6987;
6988; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
6989; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6990; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6991; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6992; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6993; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6994; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6995; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
6996; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6997; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
6998; GFX90A-TGSPLIT-NEXT:    s_endpgm
6999;
7000; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
7001; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7002; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7003; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7004; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7005; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7006; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
7007; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[2:3] sc0
7008; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7009; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
7010; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7011;
7012; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
7013; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7014; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7015; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7016; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7017; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7018; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
7019; GFX940-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[2:3] sc0
7020; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7021; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
7022; GFX940-TGSPLIT-NEXT:    s_endpgm
7023    i32 addrspace(1)* %out, i32 %in) {
7024entry:
7025  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") seq_cst
7026  store i32 %val, i32 addrspace(1)* %out, align 4
7027  ret void
7028}
7029
7030define amdgpu_kernel void @global_singlethread_one_as_monotonic_monotonic_cmpxchg(
7031; GFX6-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
7032; GFX6:       ; %bb.0: ; %entry
7033; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
7034; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7035; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7036; GFX6-NEXT:    s_mov_b32 s2, -1
7037; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7038; GFX6-NEXT:    v_mov_b32_e32 v0, s6
7039; GFX6-NEXT:    v_mov_b32_e32 v1, s7
7040; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
7041; GFX6-NEXT:    s_endpgm
7042;
7043; GFX7-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
7044; GFX7:       ; %bb.0: ; %entry
7045; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7046; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7047; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7048; GFX7-NEXT:    s_add_u32 s0, s0, 16
7049; GFX7-NEXT:    s_addc_u32 s1, s1, 0
7050; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7051; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7052; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7053; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7054; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7055; GFX7-NEXT:    s_endpgm
7056;
7057; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
7058; GFX10-WGP:       ; %bb.0: ; %entry
7059; GFX10-WGP-NEXT:    s_clause 0x1
7060; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7061; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7062; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
7063; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7064; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7065; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7066; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7067; GFX10-WGP-NEXT:    s_endpgm
7068;
7069; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
7070; GFX10-CU:       ; %bb.0: ; %entry
7071; GFX10-CU-NEXT:    s_clause 0x1
7072; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7073; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7074; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
7075; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7076; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7077; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7078; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7079; GFX10-CU-NEXT:    s_endpgm
7080;
7081; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
7082; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7083; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
7084; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7085; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
7086; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
7087; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7088; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7089; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7090; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
7091; SKIP-CACHE-INV-NEXT:    s_endpgm
7092;
7093; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
7094; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7095; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7096; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7097; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7098; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7099; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7100; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7101; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7102;
7103; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
7104; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7105; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7106; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7107; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7108; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7109; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7110; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7111; GFX90A-TGSPLIT-NEXT:    s_endpgm
7112;
7113; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
7114; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7115; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7116; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
7117; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7118; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7119; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7120; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
7121; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7122;
7123; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
7124; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7125; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7126; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
7127; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7128; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7129; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7130; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
7131; GFX940-TGSPLIT-NEXT:    s_endpgm
7132    i32 addrspace(1)* %out, i32 %in, i32 %old) {
7133entry:
7134  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
7135  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic monotonic
7136  ret void
7137}
7138
7139define amdgpu_kernel void @global_singlethread_one_as_acquire_monotonic_cmpxchg(
7140; GFX6-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
7141; GFX6:       ; %bb.0: ; %entry
7142; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
7143; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7144; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7145; GFX6-NEXT:    s_mov_b32 s2, -1
7146; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7147; GFX6-NEXT:    v_mov_b32_e32 v0, s6
7148; GFX6-NEXT:    v_mov_b32_e32 v1, s7
7149; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
7150; GFX6-NEXT:    s_endpgm
7151;
7152; GFX7-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
7153; GFX7:       ; %bb.0: ; %entry
7154; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7155; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7156; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7157; GFX7-NEXT:    s_add_u32 s0, s0, 16
7158; GFX7-NEXT:    s_addc_u32 s1, s1, 0
7159; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7160; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7161; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7162; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7163; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7164; GFX7-NEXT:    s_endpgm
7165;
7166; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
7167; GFX10-WGP:       ; %bb.0: ; %entry
7168; GFX10-WGP-NEXT:    s_clause 0x1
7169; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7170; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7171; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
7172; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7173; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7174; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7175; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7176; GFX10-WGP-NEXT:    s_endpgm
7177;
7178; GFX10-CU-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
7179; GFX10-CU:       ; %bb.0: ; %entry
7180; GFX10-CU-NEXT:    s_clause 0x1
7181; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7182; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7183; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
7184; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7185; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7186; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7187; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7188; GFX10-CU-NEXT:    s_endpgm
7189;
7190; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
7191; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7192; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
7193; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7194; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
7195; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
7196; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7197; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7198; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7199; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
7200; SKIP-CACHE-INV-NEXT:    s_endpgm
7201;
7202; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
7203; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7204; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7205; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7206; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7207; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7208; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7209; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7210; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7211;
7212; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
7213; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7214; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7215; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7216; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7217; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7218; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7219; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7220; GFX90A-TGSPLIT-NEXT:    s_endpgm
7221;
7222; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
7223; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7224; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7225; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
7226; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7227; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7228; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7229; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
7230; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7231;
7232; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
7233; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7234; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7235; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
7236; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7237; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7238; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7239; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
7240; GFX940-TGSPLIT-NEXT:    s_endpgm
7241    i32 addrspace(1)* %out, i32 %in, i32 %old) {
7242entry:
7243  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
7244  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire monotonic
7245  ret void
7246}
7247
7248define amdgpu_kernel void @global_singlethread_one_as_release_monotonic_cmpxchg(
7249; GFX6-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
7250; GFX6:       ; %bb.0: ; %entry
7251; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
7252; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7253; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7254; GFX6-NEXT:    s_mov_b32 s2, -1
7255; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7256; GFX6-NEXT:    v_mov_b32_e32 v0, s6
7257; GFX6-NEXT:    v_mov_b32_e32 v1, s7
7258; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
7259; GFX6-NEXT:    s_endpgm
7260;
7261; GFX7-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
7262; GFX7:       ; %bb.0: ; %entry
7263; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7264; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7265; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7266; GFX7-NEXT:    s_add_u32 s0, s0, 16
7267; GFX7-NEXT:    s_addc_u32 s1, s1, 0
7268; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7269; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7270; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7271; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7272; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7273; GFX7-NEXT:    s_endpgm
7274;
7275; GFX10-WGP-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
7276; GFX10-WGP:       ; %bb.0: ; %entry
7277; GFX10-WGP-NEXT:    s_clause 0x1
7278; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7279; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7280; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
7281; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7282; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7283; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7284; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7285; GFX10-WGP-NEXT:    s_endpgm
7286;
7287; GFX10-CU-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
7288; GFX10-CU:       ; %bb.0: ; %entry
7289; GFX10-CU-NEXT:    s_clause 0x1
7290; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7291; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7292; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
7293; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7294; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7295; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7296; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7297; GFX10-CU-NEXT:    s_endpgm
7298;
7299; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
7300; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7301; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
7302; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7303; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
7304; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
7305; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7306; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7307; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7308; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
7309; SKIP-CACHE-INV-NEXT:    s_endpgm
7310;
7311; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
7312; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7313; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7314; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7315; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7316; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7317; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7318; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7319; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7320;
7321; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
7322; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7323; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7324; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7325; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7326; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7327; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7328; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7329; GFX90A-TGSPLIT-NEXT:    s_endpgm
7330;
7331; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
7332; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7333; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7334; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
7335; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7336; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7337; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7338; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
7339; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7340;
7341; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
7342; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7343; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7344; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
7345; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7346; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7347; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7348; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
7349; GFX940-TGSPLIT-NEXT:    s_endpgm
7350    i32 addrspace(1)* %out, i32 %in, i32 %old) {
7351entry:
7352  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
7353  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release monotonic
7354  ret void
7355}
7356
7357define amdgpu_kernel void @global_singlethread_one_as_acq_rel_monotonic_cmpxchg(
7358; GFX6-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
7359; GFX6:       ; %bb.0: ; %entry
7360; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
7361; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7362; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7363; GFX6-NEXT:    s_mov_b32 s2, -1
7364; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7365; GFX6-NEXT:    v_mov_b32_e32 v0, s6
7366; GFX6-NEXT:    v_mov_b32_e32 v1, s7
7367; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
7368; GFX6-NEXT:    s_endpgm
7369;
7370; GFX7-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
7371; GFX7:       ; %bb.0: ; %entry
7372; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7373; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7374; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7375; GFX7-NEXT:    s_add_u32 s0, s0, 16
7376; GFX7-NEXT:    s_addc_u32 s1, s1, 0
7377; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7378; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7379; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7380; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7381; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7382; GFX7-NEXT:    s_endpgm
7383;
7384; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
7385; GFX10-WGP:       ; %bb.0: ; %entry
7386; GFX10-WGP-NEXT:    s_clause 0x1
7387; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7388; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7389; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
7390; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7391; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7392; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7393; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7394; GFX10-WGP-NEXT:    s_endpgm
7395;
7396; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
7397; GFX10-CU:       ; %bb.0: ; %entry
7398; GFX10-CU-NEXT:    s_clause 0x1
7399; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7400; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7401; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
7402; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7403; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7404; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7405; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7406; GFX10-CU-NEXT:    s_endpgm
7407;
7408; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
7409; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7410; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
7411; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7412; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
7413; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
7414; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7415; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7416; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7417; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
7418; SKIP-CACHE-INV-NEXT:    s_endpgm
7419;
7420; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
7421; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7422; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7423; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7424; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7425; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7426; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7427; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7428; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7429;
7430; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
7431; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7432; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7433; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7434; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7435; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7436; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7437; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7438; GFX90A-TGSPLIT-NEXT:    s_endpgm
7439;
7440; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
7441; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7442; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7443; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
7444; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7445; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7446; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7447; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
7448; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7449;
7450; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
7451; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7452; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7453; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
7454; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7455; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7456; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7457; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
7458; GFX940-TGSPLIT-NEXT:    s_endpgm
7459    i32 addrspace(1)* %out, i32 %in, i32 %old) {
7460entry:
7461  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
7462  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel monotonic
7463  ret void
7464}
7465
7466define amdgpu_kernel void @global_singlethread_one_as_seq_cst_monotonic_cmpxchg(
7467; GFX6-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
7468; GFX6:       ; %bb.0: ; %entry
7469; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
7470; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7471; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7472; GFX6-NEXT:    s_mov_b32 s2, -1
7473; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7474; GFX6-NEXT:    v_mov_b32_e32 v0, s6
7475; GFX6-NEXT:    v_mov_b32_e32 v1, s7
7476; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
7477; GFX6-NEXT:    s_endpgm
7478;
7479; GFX7-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
7480; GFX7:       ; %bb.0: ; %entry
7481; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7482; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7483; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7484; GFX7-NEXT:    s_add_u32 s0, s0, 16
7485; GFX7-NEXT:    s_addc_u32 s1, s1, 0
7486; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7487; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7488; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7489; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7490; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7491; GFX7-NEXT:    s_endpgm
7492;
7493; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
7494; GFX10-WGP:       ; %bb.0: ; %entry
7495; GFX10-WGP-NEXT:    s_clause 0x1
7496; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7497; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7498; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
7499; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7500; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7501; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7502; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7503; GFX10-WGP-NEXT:    s_endpgm
7504;
7505; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
7506; GFX10-CU:       ; %bb.0: ; %entry
7507; GFX10-CU-NEXT:    s_clause 0x1
7508; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7509; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7510; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
7511; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7512; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7513; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7514; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7515; GFX10-CU-NEXT:    s_endpgm
7516;
7517; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
7518; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7519; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
7520; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7521; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
7522; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
7523; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7524; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7525; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7526; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
7527; SKIP-CACHE-INV-NEXT:    s_endpgm
7528;
7529; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
7530; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7531; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7532; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7533; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7534; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7535; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7536; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7537; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7538;
7539; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
7540; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7541; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7542; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7543; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7544; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7545; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7546; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7547; GFX90A-TGSPLIT-NEXT:    s_endpgm
7548;
7549; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
7550; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7551; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7552; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
7553; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7554; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7555; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7556; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
7557; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7558;
7559; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
7560; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7561; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7562; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
7563; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7564; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7565; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7566; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
7567; GFX940-TGSPLIT-NEXT:    s_endpgm
7568    i32 addrspace(1)* %out, i32 %in, i32 %old) {
7569entry:
7570  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
7571  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst monotonic
7572  ret void
7573}
7574
7575define amdgpu_kernel void @global_singlethread_one_as_monotonic_acquire_cmpxchg(
7576; GFX6-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
7577; GFX6:       ; %bb.0: ; %entry
7578; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
7579; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7580; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7581; GFX6-NEXT:    s_mov_b32 s2, -1
7582; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7583; GFX6-NEXT:    v_mov_b32_e32 v0, s6
7584; GFX6-NEXT:    v_mov_b32_e32 v1, s7
7585; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
7586; GFX6-NEXT:    s_endpgm
7587;
7588; GFX7-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
7589; GFX7:       ; %bb.0: ; %entry
7590; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7591; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7592; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7593; GFX7-NEXT:    s_add_u32 s0, s0, 16
7594; GFX7-NEXT:    s_addc_u32 s1, s1, 0
7595; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7596; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7597; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7598; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7599; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7600; GFX7-NEXT:    s_endpgm
7601;
7602; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
7603; GFX10-WGP:       ; %bb.0: ; %entry
7604; GFX10-WGP-NEXT:    s_clause 0x1
7605; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7606; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7607; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
7608; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7609; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7610; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7611; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7612; GFX10-WGP-NEXT:    s_endpgm
7613;
7614; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
7615; GFX10-CU:       ; %bb.0: ; %entry
7616; GFX10-CU-NEXT:    s_clause 0x1
7617; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7618; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7619; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
7620; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7621; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7622; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7623; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7624; GFX10-CU-NEXT:    s_endpgm
7625;
7626; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
7627; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7628; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
7629; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7630; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
7631; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
7632; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7633; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7634; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7635; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
7636; SKIP-CACHE-INV-NEXT:    s_endpgm
7637;
7638; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
7639; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7640; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7641; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7642; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7643; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7644; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7645; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7646; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7647;
7648; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
7649; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7650; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7651; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7652; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7653; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7654; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7655; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7656; GFX90A-TGSPLIT-NEXT:    s_endpgm
7657;
7658; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
7659; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7660; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7661; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
7662; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7663; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7664; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7665; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
7666; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7667;
7668; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
7669; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7670; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7671; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
7672; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7673; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7674; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7675; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
7676; GFX940-TGSPLIT-NEXT:    s_endpgm
7677    i32 addrspace(1)* %out, i32 %in, i32 %old) {
7678entry:
7679  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
7680  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic acquire
7681  ret void
7682}
7683
7684define amdgpu_kernel void @global_singlethread_one_as_acquire_acquire_cmpxchg(
7685; GFX6-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
7686; GFX6:       ; %bb.0: ; %entry
7687; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
7688; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7689; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7690; GFX6-NEXT:    s_mov_b32 s2, -1
7691; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7692; GFX6-NEXT:    v_mov_b32_e32 v0, s6
7693; GFX6-NEXT:    v_mov_b32_e32 v1, s7
7694; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
7695; GFX6-NEXT:    s_endpgm
7696;
7697; GFX7-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
7698; GFX7:       ; %bb.0: ; %entry
7699; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7700; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7701; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7702; GFX7-NEXT:    s_add_u32 s0, s0, 16
7703; GFX7-NEXT:    s_addc_u32 s1, s1, 0
7704; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7705; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7706; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7707; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7708; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7709; GFX7-NEXT:    s_endpgm
7710;
7711; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
7712; GFX10-WGP:       ; %bb.0: ; %entry
7713; GFX10-WGP-NEXT:    s_clause 0x1
7714; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7715; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7716; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
7717; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7718; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7719; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7720; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7721; GFX10-WGP-NEXT:    s_endpgm
7722;
7723; GFX10-CU-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
7724; GFX10-CU:       ; %bb.0: ; %entry
7725; GFX10-CU-NEXT:    s_clause 0x1
7726; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7727; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7728; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
7729; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7730; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7731; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7732; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7733; GFX10-CU-NEXT:    s_endpgm
7734;
7735; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
7736; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7737; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
7738; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7739; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
7740; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
7741; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7742; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7743; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7744; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
7745; SKIP-CACHE-INV-NEXT:    s_endpgm
7746;
7747; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
7748; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7749; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7750; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7751; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7752; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7753; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7754; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7755; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7756;
7757; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
7758; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7759; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7760; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7761; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7762; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7763; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7764; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7765; GFX90A-TGSPLIT-NEXT:    s_endpgm
7766;
7767; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
7768; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7769; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7770; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
7771; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7772; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7773; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7774; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
7775; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7776;
7777; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
7778; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7779; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7780; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
7781; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7782; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7783; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7784; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
7785; GFX940-TGSPLIT-NEXT:    s_endpgm
7786    i32 addrspace(1)* %out, i32 %in, i32 %old) {
7787entry:
7788  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
7789  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire acquire
7790  ret void
7791}
7792
7793define amdgpu_kernel void @global_singlethread_one_as_release_acquire_cmpxchg(
7794; GFX6-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
7795; GFX6:       ; %bb.0: ; %entry
7796; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
7797; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7798; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7799; GFX6-NEXT:    s_mov_b32 s2, -1
7800; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7801; GFX6-NEXT:    v_mov_b32_e32 v0, s6
7802; GFX6-NEXT:    v_mov_b32_e32 v1, s7
7803; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
7804; GFX6-NEXT:    s_endpgm
7805;
7806; GFX7-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
7807; GFX7:       ; %bb.0: ; %entry
7808; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7809; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7810; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7811; GFX7-NEXT:    s_add_u32 s0, s0, 16
7812; GFX7-NEXT:    s_addc_u32 s1, s1, 0
7813; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7814; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7815; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7816; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7817; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7818; GFX7-NEXT:    s_endpgm
7819;
7820; GFX10-WGP-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
7821; GFX10-WGP:       ; %bb.0: ; %entry
7822; GFX10-WGP-NEXT:    s_clause 0x1
7823; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7824; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7825; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
7826; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7827; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7828; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7829; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7830; GFX10-WGP-NEXT:    s_endpgm
7831;
7832; GFX10-CU-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
7833; GFX10-CU:       ; %bb.0: ; %entry
7834; GFX10-CU-NEXT:    s_clause 0x1
7835; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7836; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7837; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
7838; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7839; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7840; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7841; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7842; GFX10-CU-NEXT:    s_endpgm
7843;
7844; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
7845; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7846; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
7847; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7848; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
7849; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
7850; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7851; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7852; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7853; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
7854; SKIP-CACHE-INV-NEXT:    s_endpgm
7855;
7856; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
7857; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7858; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7859; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7860; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7861; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7862; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7863; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7864; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7865;
7866; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
7867; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7868; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7869; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7870; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7871; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7872; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7873; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7874; GFX90A-TGSPLIT-NEXT:    s_endpgm
7875;
7876; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
7877; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7878; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7879; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
7880; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7881; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7882; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7883; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
7884; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7885;
7886; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
7887; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7888; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7889; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
7890; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7891; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7892; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7893; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
7894; GFX940-TGSPLIT-NEXT:    s_endpgm
7895    i32 addrspace(1)* %out, i32 %in, i32 %old) {
7896entry:
7897  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
7898  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release acquire
7899  ret void
7900}
7901
7902define amdgpu_kernel void @global_singlethread_one_as_acq_rel_acquire_cmpxchg(
7903; GFX6-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
7904; GFX6:       ; %bb.0: ; %entry
7905; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
7906; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7907; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7908; GFX6-NEXT:    s_mov_b32 s2, -1
7909; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7910; GFX6-NEXT:    v_mov_b32_e32 v0, s6
7911; GFX6-NEXT:    v_mov_b32_e32 v1, s7
7912; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
7913; GFX6-NEXT:    s_endpgm
7914;
7915; GFX7-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
7916; GFX7:       ; %bb.0: ; %entry
7917; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7918; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7919; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7920; GFX7-NEXT:    s_add_u32 s0, s0, 16
7921; GFX7-NEXT:    s_addc_u32 s1, s1, 0
7922; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7923; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7924; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7925; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7926; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7927; GFX7-NEXT:    s_endpgm
7928;
7929; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
7930; GFX10-WGP:       ; %bb.0: ; %entry
7931; GFX10-WGP-NEXT:    s_clause 0x1
7932; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7933; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7934; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
7935; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7936; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7937; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7938; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7939; GFX10-WGP-NEXT:    s_endpgm
7940;
7941; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
7942; GFX10-CU:       ; %bb.0: ; %entry
7943; GFX10-CU-NEXT:    s_clause 0x1
7944; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7945; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7946; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
7947; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7948; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7949; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7950; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7951; GFX10-CU-NEXT:    s_endpgm
7952;
7953; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
7954; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7955; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
7956; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7957; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
7958; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
7959; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7960; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7961; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7962; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
7963; SKIP-CACHE-INV-NEXT:    s_endpgm
7964;
7965; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
7966; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7967; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7968; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7969; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7970; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7971; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7972; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7973; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7974;
7975; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
7976; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7977; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7978; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7979; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7980; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7981; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7982; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7983; GFX90A-TGSPLIT-NEXT:    s_endpgm
7984;
7985; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
7986; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7987; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7988; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
7989; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7990; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7991; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7992; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
7993; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7994;
7995; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
7996; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7997; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7998; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
7999; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8000; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8001; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8002; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
8003; GFX940-TGSPLIT-NEXT:    s_endpgm
8004    i32 addrspace(1)* %out, i32 %in, i32 %old) {
8005entry:
8006  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
8007  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel acquire
8008  ret void
8009}
8010
8011define amdgpu_kernel void @global_singlethread_one_as_seq_cst_acquire_cmpxchg(
8012; GFX6-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
8013; GFX6:       ; %bb.0: ; %entry
8014; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
8015; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8016; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
8017; GFX6-NEXT:    s_mov_b32 s2, -1
8018; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8019; GFX6-NEXT:    v_mov_b32_e32 v0, s6
8020; GFX6-NEXT:    v_mov_b32_e32 v1, s7
8021; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
8022; GFX6-NEXT:    s_endpgm
8023;
8024; GFX7-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
8025; GFX7:       ; %bb.0: ; %entry
8026; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8027; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8028; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8029; GFX7-NEXT:    s_add_u32 s0, s0, 16
8030; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8031; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8032; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8033; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8034; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8035; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8036; GFX7-NEXT:    s_endpgm
8037;
8038; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
8039; GFX10-WGP:       ; %bb.0: ; %entry
8040; GFX10-WGP-NEXT:    s_clause 0x1
8041; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8042; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8043; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
8044; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8045; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8046; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8047; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8048; GFX10-WGP-NEXT:    s_endpgm
8049;
8050; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
8051; GFX10-CU:       ; %bb.0: ; %entry
8052; GFX10-CU-NEXT:    s_clause 0x1
8053; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8054; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8055; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
8056; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8057; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8058; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8059; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8060; GFX10-CU-NEXT:    s_endpgm
8061;
8062; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
8063; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8064; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
8065; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
8066; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
8067; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
8068; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8069; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8070; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8071; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
8072; SKIP-CACHE-INV-NEXT:    s_endpgm
8073;
8074; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
8075; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8076; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8077; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8078; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8079; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8080; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8081; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8082; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8083;
8084; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
8085; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8086; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8087; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8088; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8089; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8090; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8091; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8092; GFX90A-TGSPLIT-NEXT:    s_endpgm
8093;
8094; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
8095; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8096; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8097; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
8098; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8099; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8100; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8101; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
8102; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8103;
8104; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
8105; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8106; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8107; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
8108; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8109; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8110; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8111; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
8112; GFX940-TGSPLIT-NEXT:    s_endpgm
8113    i32 addrspace(1)* %out, i32 %in, i32 %old) {
8114entry:
8115  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
8116  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst acquire
8117  ret void
8118}
8119
8120define amdgpu_kernel void @global_singlethread_one_as_monotonic_seq_cst_cmpxchg(
8121; GFX6-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
8122; GFX6:       ; %bb.0: ; %entry
8123; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
8124; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8125; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
8126; GFX6-NEXT:    s_mov_b32 s2, -1
8127; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8128; GFX6-NEXT:    v_mov_b32_e32 v0, s6
8129; GFX6-NEXT:    v_mov_b32_e32 v1, s7
8130; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
8131; GFX6-NEXT:    s_endpgm
8132;
8133; GFX7-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
8134; GFX7:       ; %bb.0: ; %entry
8135; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8136; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8137; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8138; GFX7-NEXT:    s_add_u32 s0, s0, 16
8139; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8140; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8141; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8142; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8143; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8144; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8145; GFX7-NEXT:    s_endpgm
8146;
8147; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
8148; GFX10-WGP:       ; %bb.0: ; %entry
8149; GFX10-WGP-NEXT:    s_clause 0x1
8150; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8151; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8152; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
8153; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8154; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8155; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8156; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8157; GFX10-WGP-NEXT:    s_endpgm
8158;
8159; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
8160; GFX10-CU:       ; %bb.0: ; %entry
8161; GFX10-CU-NEXT:    s_clause 0x1
8162; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8163; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8164; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
8165; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8166; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8167; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8168; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8169; GFX10-CU-NEXT:    s_endpgm
8170;
8171; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
8172; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8173; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
8174; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
8175; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
8176; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
8177; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8178; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8179; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8180; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
8181; SKIP-CACHE-INV-NEXT:    s_endpgm
8182;
8183; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
8184; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8185; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8186; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8187; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8188; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8189; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8190; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8191; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8192;
8193; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
8194; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8195; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8196; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8197; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8198; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8199; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8200; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8201; GFX90A-TGSPLIT-NEXT:    s_endpgm
8202;
8203; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
8204; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8205; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8206; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
8207; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8208; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8209; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8210; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
8211; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8212;
8213; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
8214; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8215; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8216; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
8217; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8218; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8219; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8220; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
8221; GFX940-TGSPLIT-NEXT:    s_endpgm
8222    i32 addrspace(1)* %out, i32 %in, i32 %old) {
8223entry:
8224  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
8225  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic seq_cst
8226  ret void
8227}
8228
8229define amdgpu_kernel void @global_singlethread_one_as_acquire_seq_cst_cmpxchg(
8230; GFX6-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
8231; GFX6:       ; %bb.0: ; %entry
8232; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
8233; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8234; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
8235; GFX6-NEXT:    s_mov_b32 s2, -1
8236; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8237; GFX6-NEXT:    v_mov_b32_e32 v0, s6
8238; GFX6-NEXT:    v_mov_b32_e32 v1, s7
8239; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
8240; GFX6-NEXT:    s_endpgm
8241;
8242; GFX7-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
8243; GFX7:       ; %bb.0: ; %entry
8244; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8245; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8246; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8247; GFX7-NEXT:    s_add_u32 s0, s0, 16
8248; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8249; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8250; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8251; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8252; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8253; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8254; GFX7-NEXT:    s_endpgm
8255;
8256; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
8257; GFX10-WGP:       ; %bb.0: ; %entry
8258; GFX10-WGP-NEXT:    s_clause 0x1
8259; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8260; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8261; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
8262; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8263; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8264; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8265; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8266; GFX10-WGP-NEXT:    s_endpgm
8267;
8268; GFX10-CU-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
8269; GFX10-CU:       ; %bb.0: ; %entry
8270; GFX10-CU-NEXT:    s_clause 0x1
8271; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8272; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8273; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
8274; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8275; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8276; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8277; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8278; GFX10-CU-NEXT:    s_endpgm
8279;
8280; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
8281; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8282; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
8283; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
8284; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
8285; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
8286; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8287; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8288; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8289; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
8290; SKIP-CACHE-INV-NEXT:    s_endpgm
8291;
8292; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
8293; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8294; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8295; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8296; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8297; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8298; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8299; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8300; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8301;
8302; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
8303; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8304; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8305; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8306; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8307; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8308; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8309; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8310; GFX90A-TGSPLIT-NEXT:    s_endpgm
8311;
8312; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
8313; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8314; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8315; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
8316; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8317; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8318; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8319; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
8320; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8321;
8322; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
8323; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8324; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8325; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
8326; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8327; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8328; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8329; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
8330; GFX940-TGSPLIT-NEXT:    s_endpgm
8331    i32 addrspace(1)* %out, i32 %in, i32 %old) {
8332entry:
8333  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
8334  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire seq_cst
8335  ret void
8336}
8337
8338define amdgpu_kernel void @global_singlethread_one_as_release_seq_cst_cmpxchg(
8339; GFX6-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
8340; GFX6:       ; %bb.0: ; %entry
8341; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
8342; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8343; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
8344; GFX6-NEXT:    s_mov_b32 s2, -1
8345; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8346; GFX6-NEXT:    v_mov_b32_e32 v0, s6
8347; GFX6-NEXT:    v_mov_b32_e32 v1, s7
8348; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
8349; GFX6-NEXT:    s_endpgm
8350;
8351; GFX7-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
8352; GFX7:       ; %bb.0: ; %entry
8353; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8354; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8355; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8356; GFX7-NEXT:    s_add_u32 s0, s0, 16
8357; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8358; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8359; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8360; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8361; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8362; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8363; GFX7-NEXT:    s_endpgm
8364;
8365; GFX10-WGP-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
8366; GFX10-WGP:       ; %bb.0: ; %entry
8367; GFX10-WGP-NEXT:    s_clause 0x1
8368; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8369; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8370; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
8371; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8372; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8373; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8374; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8375; GFX10-WGP-NEXT:    s_endpgm
8376;
8377; GFX10-CU-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
8378; GFX10-CU:       ; %bb.0: ; %entry
8379; GFX10-CU-NEXT:    s_clause 0x1
8380; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8381; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8382; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
8383; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8384; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8385; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8386; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8387; GFX10-CU-NEXT:    s_endpgm
8388;
8389; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
8390; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8391; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
8392; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
8393; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
8394; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
8395; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8396; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8397; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8398; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
8399; SKIP-CACHE-INV-NEXT:    s_endpgm
8400;
8401; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
8402; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8403; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8404; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8405; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8406; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8407; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8408; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8409; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8410;
8411; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
8412; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8413; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8414; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8415; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8416; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8417; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8418; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8419; GFX90A-TGSPLIT-NEXT:    s_endpgm
8420;
8421; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
8422; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8423; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8424; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
8425; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8426; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8427; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8428; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
8429; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8430;
8431; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
8432; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8433; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8434; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
8435; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8436; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8437; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8438; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
8439; GFX940-TGSPLIT-NEXT:    s_endpgm
8440    i32 addrspace(1)* %out, i32 %in, i32 %old) {
8441entry:
8442  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
8443  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release seq_cst
8444  ret void
8445}
8446
8447define amdgpu_kernel void @global_singlethread_one_as_acq_rel_seq_cst_cmpxchg(
8448; GFX6-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
8449; GFX6:       ; %bb.0: ; %entry
8450; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
8451; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8452; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
8453; GFX6-NEXT:    s_mov_b32 s2, -1
8454; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8455; GFX6-NEXT:    v_mov_b32_e32 v0, s6
8456; GFX6-NEXT:    v_mov_b32_e32 v1, s7
8457; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
8458; GFX6-NEXT:    s_endpgm
8459;
8460; GFX7-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
8461; GFX7:       ; %bb.0: ; %entry
8462; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8463; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8464; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8465; GFX7-NEXT:    s_add_u32 s0, s0, 16
8466; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8467; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8468; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8469; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8470; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8471; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8472; GFX7-NEXT:    s_endpgm
8473;
8474; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
8475; GFX10-WGP:       ; %bb.0: ; %entry
8476; GFX10-WGP-NEXT:    s_clause 0x1
8477; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8478; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8479; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
8480; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8481; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8482; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8483; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8484; GFX10-WGP-NEXT:    s_endpgm
8485;
8486; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
8487; GFX10-CU:       ; %bb.0: ; %entry
8488; GFX10-CU-NEXT:    s_clause 0x1
8489; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8490; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8491; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
8492; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8493; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8494; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8495; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8496; GFX10-CU-NEXT:    s_endpgm
8497;
8498; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
8499; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8500; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
8501; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
8502; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
8503; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
8504; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8505; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8506; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8507; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
8508; SKIP-CACHE-INV-NEXT:    s_endpgm
8509;
8510; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
8511; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8512; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8513; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8514; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8515; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8516; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8517; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8518; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8519;
8520; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
8521; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8522; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8523; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8524; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8525; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8526; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8527; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8528; GFX90A-TGSPLIT-NEXT:    s_endpgm
8529;
8530; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
8531; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8532; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8533; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
8534; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8535; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8536; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8537; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
8538; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8539;
8540; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
8541; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8542; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8543; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
8544; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8545; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8546; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8547; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
8548; GFX940-TGSPLIT-NEXT:    s_endpgm
8549    i32 addrspace(1)* %out, i32 %in, i32 %old) {
8550entry:
8551  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
8552  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel seq_cst
8553  ret void
8554}
8555
8556define amdgpu_kernel void @global_singlethread_one_as_seq_cst_seq_cst_cmpxchg(
8557; GFX6-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
8558; GFX6:       ; %bb.0: ; %entry
8559; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
8560; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8561; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
8562; GFX6-NEXT:    s_mov_b32 s2, -1
8563; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8564; GFX6-NEXT:    v_mov_b32_e32 v0, s6
8565; GFX6-NEXT:    v_mov_b32_e32 v1, s7
8566; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
8567; GFX6-NEXT:    s_endpgm
8568;
8569; GFX7-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
8570; GFX7:       ; %bb.0: ; %entry
8571; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8572; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8573; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8574; GFX7-NEXT:    s_add_u32 s0, s0, 16
8575; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8576; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8577; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8578; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8579; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8580; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8581; GFX7-NEXT:    s_endpgm
8582;
8583; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
8584; GFX10-WGP:       ; %bb.0: ; %entry
8585; GFX10-WGP-NEXT:    s_clause 0x1
8586; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8587; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8588; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
8589; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8590; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8591; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8592; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8593; GFX10-WGP-NEXT:    s_endpgm
8594;
8595; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
8596; GFX10-CU:       ; %bb.0: ; %entry
8597; GFX10-CU-NEXT:    s_clause 0x1
8598; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8599; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8600; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
8601; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8602; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8603; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8604; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8605; GFX10-CU-NEXT:    s_endpgm
8606;
8607; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
8608; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8609; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
8610; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
8611; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
8612; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
8613; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8614; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8615; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8616; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
8617; SKIP-CACHE-INV-NEXT:    s_endpgm
8618;
8619; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
8620; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8621; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8622; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8623; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8624; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8625; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8626; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8627; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8628;
8629; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
8630; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8631; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8632; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8633; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8634; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8635; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8636; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8637; GFX90A-TGSPLIT-NEXT:    s_endpgm
8638;
8639; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
8640; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8641; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8642; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
8643; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8644; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8645; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8646; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
8647; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8648;
8649; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
8650; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8651; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8652; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
8653; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8654; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8655; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8656; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
8657; GFX940-TGSPLIT-NEXT:    s_endpgm
8658    i32 addrspace(1)* %out, i32 %in, i32 %old) {
8659entry:
8660  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
8661  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst seq_cst
8662  ret void
8663}
8664
8665define amdgpu_kernel void @global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg(
8666; GFX6-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
8667; GFX6:       ; %bb.0: ; %entry
8668; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
8669; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8670; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
8671; GFX6-NEXT:    s_mov_b32 s2, -1
8672; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8673; GFX6-NEXT:    v_mov_b32_e32 v0, s6
8674; GFX6-NEXT:    v_mov_b32_e32 v1, s7
8675; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
8676; GFX6-NEXT:    s_waitcnt vmcnt(0)
8677; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
8678; GFX6-NEXT:    s_endpgm
8679;
8680; GFX7-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
8681; GFX7:       ; %bb.0: ; %entry
8682; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8683; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8684; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8685; GFX7-NEXT:    s_add_u32 s4, s0, 16
8686; GFX7-NEXT:    s_addc_u32 s5, s1, 0
8687; GFX7-NEXT:    v_mov_b32_e32 v0, s4
8688; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8689; GFX7-NEXT:    v_mov_b32_e32 v1, s5
8690; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8691; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8692; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8693; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8694; GFX7-NEXT:    s_waitcnt vmcnt(0)
8695; GFX7-NEXT:    flat_store_dword v[0:1], v2
8696; GFX7-NEXT:    s_endpgm
8697;
8698; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
8699; GFX10-WGP:       ; %bb.0: ; %entry
8700; GFX10-WGP-NEXT:    s_clause 0x1
8701; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8702; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8703; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
8704; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8705; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8706; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8707; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8708; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8709; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
8710; GFX10-WGP-NEXT:    s_endpgm
8711;
8712; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
8713; GFX10-CU:       ; %bb.0: ; %entry
8714; GFX10-CU-NEXT:    s_clause 0x1
8715; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8716; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8717; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
8718; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8719; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8720; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8721; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8722; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8723; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
8724; GFX10-CU-NEXT:    s_endpgm
8725;
8726; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
8727; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8728; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
8729; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
8730; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
8731; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
8732; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8733; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8734; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8735; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
8736; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8737; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
8738; SKIP-CACHE-INV-NEXT:    s_endpgm
8739;
8740; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
8741; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8742; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8743; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8744; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8745; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8746; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8747; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8748; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8749; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
8750; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8751;
8752; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
8753; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8754; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8755; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8756; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8757; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8758; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8759; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8760; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8761; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
8762; GFX90A-TGSPLIT-NEXT:    s_endpgm
8763;
8764; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
8765; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8766; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8767; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
8768; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8769; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8770; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8771; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
8772; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8773; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
8774; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8775;
8776; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
8777; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8778; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8779; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
8780; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8781; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8782; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8783; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
8784; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8785; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
8786; GFX940-TGSPLIT-NEXT:    s_endpgm
8787    i32 addrspace(1)* %out, i32 %in, i32 %old) {
8788entry:
8789  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
8790  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic monotonic
8791  %val0 = extractvalue { i32, i1 } %val, 0
8792  store i32 %val0, i32 addrspace(1)* %out, align 4
8793  ret void
8794}
8795
8796define amdgpu_kernel void @global_singlethread_one_as_acquire_monotonic_ret_cmpxchg(
8797; GFX6-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
8798; GFX6:       ; %bb.0: ; %entry
8799; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
8800; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8801; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
8802; GFX6-NEXT:    s_mov_b32 s2, -1
8803; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8804; GFX6-NEXT:    v_mov_b32_e32 v0, s6
8805; GFX6-NEXT:    v_mov_b32_e32 v1, s7
8806; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
8807; GFX6-NEXT:    s_waitcnt vmcnt(0)
8808; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
8809; GFX6-NEXT:    s_endpgm
8810;
8811; GFX7-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
8812; GFX7:       ; %bb.0: ; %entry
8813; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8814; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8815; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8816; GFX7-NEXT:    s_add_u32 s4, s0, 16
8817; GFX7-NEXT:    s_addc_u32 s5, s1, 0
8818; GFX7-NEXT:    v_mov_b32_e32 v0, s4
8819; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8820; GFX7-NEXT:    v_mov_b32_e32 v1, s5
8821; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8822; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8823; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8824; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8825; GFX7-NEXT:    s_waitcnt vmcnt(0)
8826; GFX7-NEXT:    flat_store_dword v[0:1], v2
8827; GFX7-NEXT:    s_endpgm
8828;
8829; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
8830; GFX10-WGP:       ; %bb.0: ; %entry
8831; GFX10-WGP-NEXT:    s_clause 0x1
8832; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8833; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8834; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
8835; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8836; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8837; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8838; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8839; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8840; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
8841; GFX10-WGP-NEXT:    s_endpgm
8842;
8843; GFX10-CU-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
8844; GFX10-CU:       ; %bb.0: ; %entry
8845; GFX10-CU-NEXT:    s_clause 0x1
8846; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8847; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8848; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
8849; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8850; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8851; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8852; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8853; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8854; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
8855; GFX10-CU-NEXT:    s_endpgm
8856;
8857; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
8858; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8859; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
8860; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
8861; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
8862; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
8863; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8864; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8865; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8866; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
8867; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8868; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
8869; SKIP-CACHE-INV-NEXT:    s_endpgm
8870;
8871; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
8872; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8873; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8874; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8875; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8876; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8877; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8878; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8879; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8880; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
8881; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8882;
8883; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
8884; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8885; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8886; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8887; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8888; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8889; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8890; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8891; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8892; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
8893; GFX90A-TGSPLIT-NEXT:    s_endpgm
8894;
8895; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
8896; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8897; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8898; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
8899; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8900; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8901; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8902; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
8903; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8904; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
8905; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8906;
8907; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
8908; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8909; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8910; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
8911; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8912; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8913; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8914; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
8915; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8916; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
8917; GFX940-TGSPLIT-NEXT:    s_endpgm
8918    i32 addrspace(1)* %out, i32 %in, i32 %old) {
8919entry:
8920  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
8921  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire monotonic
8922  %val0 = extractvalue { i32, i1 } %val, 0
8923  store i32 %val0, i32 addrspace(1)* %out, align 4
8924  ret void
8925}
8926
8927define amdgpu_kernel void @global_singlethread_one_as_release_monotonic_ret_cmpxchg(
8928; GFX6-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
8929; GFX6:       ; %bb.0: ; %entry
8930; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
8931; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8932; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
8933; GFX6-NEXT:    s_mov_b32 s2, -1
8934; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8935; GFX6-NEXT:    v_mov_b32_e32 v0, s6
8936; GFX6-NEXT:    v_mov_b32_e32 v1, s7
8937; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
8938; GFX6-NEXT:    s_waitcnt vmcnt(0)
8939; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
8940; GFX6-NEXT:    s_endpgm
8941;
8942; GFX7-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
8943; GFX7:       ; %bb.0: ; %entry
8944; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8945; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8946; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8947; GFX7-NEXT:    s_add_u32 s4, s0, 16
8948; GFX7-NEXT:    s_addc_u32 s5, s1, 0
8949; GFX7-NEXT:    v_mov_b32_e32 v0, s4
8950; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8951; GFX7-NEXT:    v_mov_b32_e32 v1, s5
8952; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8953; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8954; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8955; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8956; GFX7-NEXT:    s_waitcnt vmcnt(0)
8957; GFX7-NEXT:    flat_store_dword v[0:1], v2
8958; GFX7-NEXT:    s_endpgm
8959;
8960; GFX10-WGP-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
8961; GFX10-WGP:       ; %bb.0: ; %entry
8962; GFX10-WGP-NEXT:    s_clause 0x1
8963; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8964; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8965; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
8966; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8967; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8968; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8969; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8970; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8971; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
8972; GFX10-WGP-NEXT:    s_endpgm
8973;
8974; GFX10-CU-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
8975; GFX10-CU:       ; %bb.0: ; %entry
8976; GFX10-CU-NEXT:    s_clause 0x1
8977; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8978; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8979; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
8980; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8981; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8982; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8983; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8984; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8985; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
8986; GFX10-CU-NEXT:    s_endpgm
8987;
8988; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
8989; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8990; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
8991; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
8992; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
8993; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
8994; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8995; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8996; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8997; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
8998; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8999; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
9000; SKIP-CACHE-INV-NEXT:    s_endpgm
9001;
9002; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
9003; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9004; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9005; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9006; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9007; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9008; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9009; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9010; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9011; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
9012; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9013;
9014; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
9015; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9016; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9017; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9018; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9019; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9020; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9021; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9022; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9023; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
9024; GFX90A-TGSPLIT-NEXT:    s_endpgm
9025;
9026; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
9027; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9028; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9029; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9030; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9031; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9032; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9033; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
9034; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9035; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
9036; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9037;
9038; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
9039; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9040; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9041; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9042; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9043; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9044; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9045; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
9046; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9047; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
9048; GFX940-TGSPLIT-NEXT:    s_endpgm
9049    i32 addrspace(1)* %out, i32 %in, i32 %old) {
9050entry:
9051  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
9052  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release monotonic
9053  %val0 = extractvalue { i32, i1 } %val, 0
9054  store i32 %val0, i32 addrspace(1)* %out, align 4
9055  ret void
9056}
9057
9058define amdgpu_kernel void @global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg(
9059; GFX6-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
9060; GFX6:       ; %bb.0: ; %entry
9061; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
9062; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9063; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
9064; GFX6-NEXT:    s_mov_b32 s2, -1
9065; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9066; GFX6-NEXT:    v_mov_b32_e32 v0, s6
9067; GFX6-NEXT:    v_mov_b32_e32 v1, s7
9068; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
9069; GFX6-NEXT:    s_waitcnt vmcnt(0)
9070; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
9071; GFX6-NEXT:    s_endpgm
9072;
9073; GFX7-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
9074; GFX7:       ; %bb.0: ; %entry
9075; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9076; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9077; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9078; GFX7-NEXT:    s_add_u32 s4, s0, 16
9079; GFX7-NEXT:    s_addc_u32 s5, s1, 0
9080; GFX7-NEXT:    v_mov_b32_e32 v0, s4
9081; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9082; GFX7-NEXT:    v_mov_b32_e32 v1, s5
9083; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9084; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9085; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9086; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9087; GFX7-NEXT:    s_waitcnt vmcnt(0)
9088; GFX7-NEXT:    flat_store_dword v[0:1], v2
9089; GFX7-NEXT:    s_endpgm
9090;
9091; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
9092; GFX10-WGP:       ; %bb.0: ; %entry
9093; GFX10-WGP-NEXT:    s_clause 0x1
9094; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9095; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9096; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
9097; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9098; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9099; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9100; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9101; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9102; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
9103; GFX10-WGP-NEXT:    s_endpgm
9104;
9105; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
9106; GFX10-CU:       ; %bb.0: ; %entry
9107; GFX10-CU-NEXT:    s_clause 0x1
9108; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9109; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9110; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
9111; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9112; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9113; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9114; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9115; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9116; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
9117; GFX10-CU-NEXT:    s_endpgm
9118;
9119; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
9120; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9121; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
9122; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
9123; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
9124; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
9125; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9126; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
9127; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
9128; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
9129; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9130; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
9131; SKIP-CACHE-INV-NEXT:    s_endpgm
9132;
9133; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
9134; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9135; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9136; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9137; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9138; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9139; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9140; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9141; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9142; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
9143; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9144;
9145; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
9146; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9147; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9148; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9149; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9150; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9151; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9152; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9153; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9154; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
9155; GFX90A-TGSPLIT-NEXT:    s_endpgm
9156;
9157; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
9158; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9159; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9160; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9161; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9162; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9163; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9164; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
9165; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9166; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
9167; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9168;
9169; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
9170; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9171; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9172; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9173; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9174; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9175; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9176; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
9177; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9178; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
9179; GFX940-TGSPLIT-NEXT:    s_endpgm
9180    i32 addrspace(1)* %out, i32 %in, i32 %old) {
9181entry:
9182  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
9183  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel monotonic
9184  %val0 = extractvalue { i32, i1 } %val, 0
9185  store i32 %val0, i32 addrspace(1)* %out, align 4
9186  ret void
9187}
9188
9189define amdgpu_kernel void @global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg(
9190; GFX6-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
9191; GFX6:       ; %bb.0: ; %entry
9192; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
9193; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9194; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
9195; GFX6-NEXT:    s_mov_b32 s2, -1
9196; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9197; GFX6-NEXT:    v_mov_b32_e32 v0, s6
9198; GFX6-NEXT:    v_mov_b32_e32 v1, s7
9199; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
9200; GFX6-NEXT:    s_waitcnt vmcnt(0)
9201; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
9202; GFX6-NEXT:    s_endpgm
9203;
9204; GFX7-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
9205; GFX7:       ; %bb.0: ; %entry
9206; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9207; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9208; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9209; GFX7-NEXT:    s_add_u32 s4, s0, 16
9210; GFX7-NEXT:    s_addc_u32 s5, s1, 0
9211; GFX7-NEXT:    v_mov_b32_e32 v0, s4
9212; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9213; GFX7-NEXT:    v_mov_b32_e32 v1, s5
9214; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9215; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9216; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9217; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9218; GFX7-NEXT:    s_waitcnt vmcnt(0)
9219; GFX7-NEXT:    flat_store_dword v[0:1], v2
9220; GFX7-NEXT:    s_endpgm
9221;
9222; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
9223; GFX10-WGP:       ; %bb.0: ; %entry
9224; GFX10-WGP-NEXT:    s_clause 0x1
9225; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9226; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9227; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
9228; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9229; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9230; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9231; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9232; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9233; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
9234; GFX10-WGP-NEXT:    s_endpgm
9235;
9236; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
9237; GFX10-CU:       ; %bb.0: ; %entry
9238; GFX10-CU-NEXT:    s_clause 0x1
9239; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9240; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9241; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
9242; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9243; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9244; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9245; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9246; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9247; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
9248; GFX10-CU-NEXT:    s_endpgm
9249;
9250; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
9251; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9252; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
9253; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
9254; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
9255; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
9256; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9257; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
9258; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
9259; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
9260; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9261; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
9262; SKIP-CACHE-INV-NEXT:    s_endpgm
9263;
9264; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
9265; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9266; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9267; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9268; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9269; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9270; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9271; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9272; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9273; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
9274; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9275;
9276; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
9277; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9278; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9279; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9280; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9281; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9282; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9283; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9284; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9285; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
9286; GFX90A-TGSPLIT-NEXT:    s_endpgm
9287;
9288; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
9289; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9290; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9291; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9292; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9293; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9294; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9295; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
9296; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9297; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
9298; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9299;
9300; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
9301; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9302; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9303; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9304; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9305; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9306; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9307; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
9308; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9309; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
9310; GFX940-TGSPLIT-NEXT:    s_endpgm
9311    i32 addrspace(1)* %out, i32 %in, i32 %old) {
9312entry:
9313  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
9314  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst monotonic
9315  %val0 = extractvalue { i32, i1 } %val, 0
9316  store i32 %val0, i32 addrspace(1)* %out, align 4
9317  ret void
9318}
9319
9320define amdgpu_kernel void @global_singlethread_one_as_monotonic_acquire_ret_cmpxchg(
9321; GFX6-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
9322; GFX6:       ; %bb.0: ; %entry
9323; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
9324; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9325; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
9326; GFX6-NEXT:    s_mov_b32 s2, -1
9327; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9328; GFX6-NEXT:    v_mov_b32_e32 v0, s6
9329; GFX6-NEXT:    v_mov_b32_e32 v1, s7
9330; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
9331; GFX6-NEXT:    s_waitcnt vmcnt(0)
9332; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
9333; GFX6-NEXT:    s_endpgm
9334;
9335; GFX7-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
9336; GFX7:       ; %bb.0: ; %entry
9337; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9338; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9339; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9340; GFX7-NEXT:    s_add_u32 s4, s0, 16
9341; GFX7-NEXT:    s_addc_u32 s5, s1, 0
9342; GFX7-NEXT:    v_mov_b32_e32 v0, s4
9343; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9344; GFX7-NEXT:    v_mov_b32_e32 v1, s5
9345; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9346; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9347; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9348; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9349; GFX7-NEXT:    s_waitcnt vmcnt(0)
9350; GFX7-NEXT:    flat_store_dword v[0:1], v2
9351; GFX7-NEXT:    s_endpgm
9352;
9353; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
9354; GFX10-WGP:       ; %bb.0: ; %entry
9355; GFX10-WGP-NEXT:    s_clause 0x1
9356; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9357; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9358; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
9359; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9360; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9361; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9362; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9363; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9364; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
9365; GFX10-WGP-NEXT:    s_endpgm
9366;
9367; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
9368; GFX10-CU:       ; %bb.0: ; %entry
9369; GFX10-CU-NEXT:    s_clause 0x1
9370; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9371; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9372; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
9373; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9374; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9375; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9376; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9377; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9378; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
9379; GFX10-CU-NEXT:    s_endpgm
9380;
9381; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
9382; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9383; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
9384; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
9385; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
9386; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
9387; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9388; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
9389; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
9390; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
9391; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9392; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
9393; SKIP-CACHE-INV-NEXT:    s_endpgm
9394;
9395; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
9396; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9397; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9398; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9399; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9400; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9401; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9402; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9403; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9404; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
9405; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9406;
9407; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
9408; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9409; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9410; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9411; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9412; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9413; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9414; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9415; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9416; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
9417; GFX90A-TGSPLIT-NEXT:    s_endpgm
9418;
9419; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
9420; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9421; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9422; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9423; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9424; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9425; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9426; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
9427; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9428; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
9429; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9430;
9431; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
9432; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9433; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9434; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9435; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9436; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9437; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9438; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
9439; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9440; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
9441; GFX940-TGSPLIT-NEXT:    s_endpgm
9442    i32 addrspace(1)* %out, i32 %in, i32 %old) {
9443entry:
9444  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
9445  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic acquire
9446  %val0 = extractvalue { i32, i1 } %val, 0
9447  store i32 %val0, i32 addrspace(1)* %out, align 4
9448  ret void
9449}
9450
9451define amdgpu_kernel void @global_singlethread_one_as_acquire_acquire_ret_cmpxchg(
9452; GFX6-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
9453; GFX6:       ; %bb.0: ; %entry
9454; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
9455; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9456; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
9457; GFX6-NEXT:    s_mov_b32 s2, -1
9458; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9459; GFX6-NEXT:    v_mov_b32_e32 v0, s6
9460; GFX6-NEXT:    v_mov_b32_e32 v1, s7
9461; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
9462; GFX6-NEXT:    s_waitcnt vmcnt(0)
9463; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
9464; GFX6-NEXT:    s_endpgm
9465;
9466; GFX7-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
9467; GFX7:       ; %bb.0: ; %entry
9468; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9469; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9470; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9471; GFX7-NEXT:    s_add_u32 s4, s0, 16
9472; GFX7-NEXT:    s_addc_u32 s5, s1, 0
9473; GFX7-NEXT:    v_mov_b32_e32 v0, s4
9474; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9475; GFX7-NEXT:    v_mov_b32_e32 v1, s5
9476; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9477; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9478; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9479; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9480; GFX7-NEXT:    s_waitcnt vmcnt(0)
9481; GFX7-NEXT:    flat_store_dword v[0:1], v2
9482; GFX7-NEXT:    s_endpgm
9483;
9484; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
9485; GFX10-WGP:       ; %bb.0: ; %entry
9486; GFX10-WGP-NEXT:    s_clause 0x1
9487; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9488; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9489; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
9490; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9491; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9492; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9493; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9494; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9495; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
9496; GFX10-WGP-NEXT:    s_endpgm
9497;
9498; GFX10-CU-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
9499; GFX10-CU:       ; %bb.0: ; %entry
9500; GFX10-CU-NEXT:    s_clause 0x1
9501; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9502; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9503; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
9504; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9505; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9506; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9507; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9508; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9509; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
9510; GFX10-CU-NEXT:    s_endpgm
9511;
9512; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
9513; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9514; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
9515; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
9516; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
9517; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
9518; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9519; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
9520; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
9521; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
9522; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9523; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
9524; SKIP-CACHE-INV-NEXT:    s_endpgm
9525;
9526; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
9527; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9528; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9529; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9530; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9531; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9532; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9533; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9534; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9535; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
9536; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9537;
9538; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
9539; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9540; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9541; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9542; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9543; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9544; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9545; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9546; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9547; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
9548; GFX90A-TGSPLIT-NEXT:    s_endpgm
9549;
9550; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
9551; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9552; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9553; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9554; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9555; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9556; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9557; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
9558; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9559; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
9560; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9561;
9562; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
9563; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9564; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9565; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9566; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9567; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9568; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9569; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
9570; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9571; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
9572; GFX940-TGSPLIT-NEXT:    s_endpgm
9573    i32 addrspace(1)* %out, i32 %in, i32 %old) {
9574entry:
9575  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
9576  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire acquire
9577  %val0 = extractvalue { i32, i1 } %val, 0
9578  store i32 %val0, i32 addrspace(1)* %out, align 4
9579  ret void
9580}
9581
9582define amdgpu_kernel void @global_singlethread_one_as_release_acquire_ret_cmpxchg(
9583; GFX6-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
9584; GFX6:       ; %bb.0: ; %entry
9585; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
9586; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9587; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
9588; GFX6-NEXT:    s_mov_b32 s2, -1
9589; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9590; GFX6-NEXT:    v_mov_b32_e32 v0, s6
9591; GFX6-NEXT:    v_mov_b32_e32 v1, s7
9592; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
9593; GFX6-NEXT:    s_waitcnt vmcnt(0)
9594; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
9595; GFX6-NEXT:    s_endpgm
9596;
9597; GFX7-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
9598; GFX7:       ; %bb.0: ; %entry
9599; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9600; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9601; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9602; GFX7-NEXT:    s_add_u32 s4, s0, 16
9603; GFX7-NEXT:    s_addc_u32 s5, s1, 0
9604; GFX7-NEXT:    v_mov_b32_e32 v0, s4
9605; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9606; GFX7-NEXT:    v_mov_b32_e32 v1, s5
9607; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9608; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9609; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9610; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9611; GFX7-NEXT:    s_waitcnt vmcnt(0)
9612; GFX7-NEXT:    flat_store_dword v[0:1], v2
9613; GFX7-NEXT:    s_endpgm
9614;
9615; GFX10-WGP-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
9616; GFX10-WGP:       ; %bb.0: ; %entry
9617; GFX10-WGP-NEXT:    s_clause 0x1
9618; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9619; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9620; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
9621; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9622; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9623; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9624; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9625; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9626; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
9627; GFX10-WGP-NEXT:    s_endpgm
9628;
9629; GFX10-CU-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
9630; GFX10-CU:       ; %bb.0: ; %entry
9631; GFX10-CU-NEXT:    s_clause 0x1
9632; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9633; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9634; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
9635; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9636; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9637; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9638; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9639; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9640; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
9641; GFX10-CU-NEXT:    s_endpgm
9642;
9643; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
9644; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9645; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
9646; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
9647; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
9648; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
9649; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9650; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
9651; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
9652; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
9653; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9654; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
9655; SKIP-CACHE-INV-NEXT:    s_endpgm
9656;
9657; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
9658; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9659; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9660; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9661; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9662; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9663; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9664; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9665; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9666; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
9667; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9668;
9669; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
9670; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9671; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9672; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9673; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9674; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9675; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9676; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9677; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9678; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
9679; GFX90A-TGSPLIT-NEXT:    s_endpgm
9680;
9681; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
9682; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9683; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9684; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9685; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9686; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9687; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9688; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
9689; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9690; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
9691; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9692;
9693; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
9694; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9695; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9696; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9697; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9698; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9699; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9700; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
9701; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9702; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
9703; GFX940-TGSPLIT-NEXT:    s_endpgm
9704    i32 addrspace(1)* %out, i32 %in, i32 %old) {
9705entry:
9706  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
9707  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release acquire
9708  %val0 = extractvalue { i32, i1 } %val, 0
9709  store i32 %val0, i32 addrspace(1)* %out, align 4
9710  ret void
9711}
9712
9713define amdgpu_kernel void @global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg(
9714; GFX6-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
9715; GFX6:       ; %bb.0: ; %entry
9716; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
9717; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9718; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
9719; GFX6-NEXT:    s_mov_b32 s2, -1
9720; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9721; GFX6-NEXT:    v_mov_b32_e32 v0, s6
9722; GFX6-NEXT:    v_mov_b32_e32 v1, s7
9723; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
9724; GFX6-NEXT:    s_waitcnt vmcnt(0)
9725; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
9726; GFX6-NEXT:    s_endpgm
9727;
9728; GFX7-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
9729; GFX7:       ; %bb.0: ; %entry
9730; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9731; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9732; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9733; GFX7-NEXT:    s_add_u32 s4, s0, 16
9734; GFX7-NEXT:    s_addc_u32 s5, s1, 0
9735; GFX7-NEXT:    v_mov_b32_e32 v0, s4
9736; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9737; GFX7-NEXT:    v_mov_b32_e32 v1, s5
9738; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9739; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9740; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9741; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9742; GFX7-NEXT:    s_waitcnt vmcnt(0)
9743; GFX7-NEXT:    flat_store_dword v[0:1], v2
9744; GFX7-NEXT:    s_endpgm
9745;
9746; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
9747; GFX10-WGP:       ; %bb.0: ; %entry
9748; GFX10-WGP-NEXT:    s_clause 0x1
9749; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9750; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9751; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
9752; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9753; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9754; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9755; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9756; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9757; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
9758; GFX10-WGP-NEXT:    s_endpgm
9759;
9760; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
9761; GFX10-CU:       ; %bb.0: ; %entry
9762; GFX10-CU-NEXT:    s_clause 0x1
9763; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9764; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9765; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
9766; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9767; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9768; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9769; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9770; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9771; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
9772; GFX10-CU-NEXT:    s_endpgm
9773;
9774; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
9775; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9776; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
9777; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
9778; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
9779; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
9780; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9781; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
9782; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
9783; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
9784; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9785; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
9786; SKIP-CACHE-INV-NEXT:    s_endpgm
9787;
9788; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
9789; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9790; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9791; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9792; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9793; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9794; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9795; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9796; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9797; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
9798; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9799;
9800; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
9801; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9802; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9803; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9804; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9805; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9806; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9807; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9808; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9809; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
9810; GFX90A-TGSPLIT-NEXT:    s_endpgm
9811;
9812; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
9813; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9814; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9815; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9816; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9817; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9818; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9819; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
9820; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9821; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
9822; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9823;
9824; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
9825; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9826; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9827; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9828; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9829; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9830; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9831; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
9832; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9833; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
9834; GFX940-TGSPLIT-NEXT:    s_endpgm
9835    i32 addrspace(1)* %out, i32 %in, i32 %old) {
9836entry:
9837  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
9838  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel acquire
9839  %val0 = extractvalue { i32, i1 } %val, 0
9840  store i32 %val0, i32 addrspace(1)* %out, align 4
9841  ret void
9842}
9843
9844define amdgpu_kernel void @global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg(
9845; GFX6-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
9846; GFX6:       ; %bb.0: ; %entry
9847; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
9848; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9849; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
9850; GFX6-NEXT:    s_mov_b32 s2, -1
9851; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9852; GFX6-NEXT:    v_mov_b32_e32 v0, s6
9853; GFX6-NEXT:    v_mov_b32_e32 v1, s7
9854; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
9855; GFX6-NEXT:    s_waitcnt vmcnt(0)
9856; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
9857; GFX6-NEXT:    s_endpgm
9858;
9859; GFX7-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
9860; GFX7:       ; %bb.0: ; %entry
9861; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9862; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9863; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9864; GFX7-NEXT:    s_add_u32 s4, s0, 16
9865; GFX7-NEXT:    s_addc_u32 s5, s1, 0
9866; GFX7-NEXT:    v_mov_b32_e32 v0, s4
9867; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9868; GFX7-NEXT:    v_mov_b32_e32 v1, s5
9869; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9870; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9871; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9872; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9873; GFX7-NEXT:    s_waitcnt vmcnt(0)
9874; GFX7-NEXT:    flat_store_dword v[0:1], v2
9875; GFX7-NEXT:    s_endpgm
9876;
9877; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
9878; GFX10-WGP:       ; %bb.0: ; %entry
9879; GFX10-WGP-NEXT:    s_clause 0x1
9880; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9881; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9882; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
9883; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9884; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9885; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9886; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9887; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9888; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
9889; GFX10-WGP-NEXT:    s_endpgm
9890;
9891; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
9892; GFX10-CU:       ; %bb.0: ; %entry
9893; GFX10-CU-NEXT:    s_clause 0x1
9894; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9895; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9896; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
9897; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9898; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9899; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9900; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9901; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9902; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
9903; GFX10-CU-NEXT:    s_endpgm
9904;
9905; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
9906; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9907; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
9908; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
9909; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
9910; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
9911; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9912; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
9913; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
9914; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
9915; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9916; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
9917; SKIP-CACHE-INV-NEXT:    s_endpgm
9918;
9919; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
9920; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9921; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9922; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9923; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9924; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9925; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9926; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9927; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9928; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
9929; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9930;
9931; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
9932; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9933; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9934; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9935; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9936; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9937; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9938; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9939; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9940; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
9941; GFX90A-TGSPLIT-NEXT:    s_endpgm
9942;
9943; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
9944; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9945; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9946; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9947; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9948; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9949; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9950; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
9951; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9952; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
9953; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9954;
9955; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
9956; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9957; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9958; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9959; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9960; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9961; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9962; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
9963; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9964; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
9965; GFX940-TGSPLIT-NEXT:    s_endpgm
9966    i32 addrspace(1)* %out, i32 %in, i32 %old) {
9967entry:
9968  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
9969  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst acquire
9970  %val0 = extractvalue { i32, i1 } %val, 0
9971  store i32 %val0, i32 addrspace(1)* %out, align 4
9972  ret void
9973}
9974
9975define amdgpu_kernel void @global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg(
9976; GFX6-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
9977; GFX6:       ; %bb.0: ; %entry
9978; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
9979; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9980; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
9981; GFX6-NEXT:    s_mov_b32 s2, -1
9982; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9983; GFX6-NEXT:    v_mov_b32_e32 v0, s6
9984; GFX6-NEXT:    v_mov_b32_e32 v1, s7
9985; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
9986; GFX6-NEXT:    s_waitcnt vmcnt(0)
9987; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
9988; GFX6-NEXT:    s_endpgm
9989;
9990; GFX7-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
9991; GFX7:       ; %bb.0: ; %entry
9992; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9993; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9994; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9995; GFX7-NEXT:    s_add_u32 s4, s0, 16
9996; GFX7-NEXT:    s_addc_u32 s5, s1, 0
9997; GFX7-NEXT:    v_mov_b32_e32 v0, s4
9998; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9999; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10000; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10001; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10002; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10003; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10004; GFX7-NEXT:    s_waitcnt vmcnt(0)
10005; GFX7-NEXT:    flat_store_dword v[0:1], v2
10006; GFX7-NEXT:    s_endpgm
10007;
10008; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
10009; GFX10-WGP:       ; %bb.0: ; %entry
10010; GFX10-WGP-NEXT:    s_clause 0x1
10011; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10012; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10013; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
10014; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10015; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10016; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10017; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10018; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10019; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
10020; GFX10-WGP-NEXT:    s_endpgm
10021;
10022; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
10023; GFX10-CU:       ; %bb.0: ; %entry
10024; GFX10-CU-NEXT:    s_clause 0x1
10025; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10026; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10027; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
10028; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10029; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10030; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10031; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10032; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10033; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
10034; GFX10-CU-NEXT:    s_endpgm
10035;
10036; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
10037; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10038; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
10039; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
10040; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
10041; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
10042; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10043; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10044; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10045; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
10046; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10047; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
10048; SKIP-CACHE-INV-NEXT:    s_endpgm
10049;
10050; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
10051; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10052; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10053; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10054; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10055; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10056; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10057; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10058; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10059; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
10060; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10061;
10062; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
10063; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10064; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10065; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10066; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10067; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10068; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10069; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10070; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10071; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
10072; GFX90A-TGSPLIT-NEXT:    s_endpgm
10073;
10074; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
10075; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10076; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10077; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10078; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10079; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10080; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10081; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
10082; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10083; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
10084; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10085;
10086; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
10087; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10088; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10089; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10090; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10091; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10092; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10093; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
10094; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10095; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
10096; GFX940-TGSPLIT-NEXT:    s_endpgm
10097    i32 addrspace(1)* %out, i32 %in, i32 %old) {
10098entry:
10099  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
10100  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic seq_cst
10101  %val0 = extractvalue { i32, i1 } %val, 0
10102  store i32 %val0, i32 addrspace(1)* %out, align 4
10103  ret void
10104}
10105
10106define amdgpu_kernel void @global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg(
10107; GFX6-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
10108; GFX6:       ; %bb.0: ; %entry
10109; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
10110; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10111; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
10112; GFX6-NEXT:    s_mov_b32 s2, -1
10113; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10114; GFX6-NEXT:    v_mov_b32_e32 v0, s6
10115; GFX6-NEXT:    v_mov_b32_e32 v1, s7
10116; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
10117; GFX6-NEXT:    s_waitcnt vmcnt(0)
10118; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
10119; GFX6-NEXT:    s_endpgm
10120;
10121; GFX7-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
10122; GFX7:       ; %bb.0: ; %entry
10123; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10124; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10125; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10126; GFX7-NEXT:    s_add_u32 s4, s0, 16
10127; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10128; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10129; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10130; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10131; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10132; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10133; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10134; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10135; GFX7-NEXT:    s_waitcnt vmcnt(0)
10136; GFX7-NEXT:    flat_store_dword v[0:1], v2
10137; GFX7-NEXT:    s_endpgm
10138;
10139; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
10140; GFX10-WGP:       ; %bb.0: ; %entry
10141; GFX10-WGP-NEXT:    s_clause 0x1
10142; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10143; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10144; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
10145; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10146; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10147; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10148; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10149; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10150; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
10151; GFX10-WGP-NEXT:    s_endpgm
10152;
10153; GFX10-CU-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
10154; GFX10-CU:       ; %bb.0: ; %entry
10155; GFX10-CU-NEXT:    s_clause 0x1
10156; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10157; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10158; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
10159; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10160; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10161; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10162; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10163; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10164; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
10165; GFX10-CU-NEXT:    s_endpgm
10166;
10167; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
10168; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10169; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
10170; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
10171; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
10172; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
10173; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10174; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10175; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10176; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
10177; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10178; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
10179; SKIP-CACHE-INV-NEXT:    s_endpgm
10180;
10181; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
10182; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10183; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10184; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10185; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10186; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10187; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10188; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10189; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10190; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
10191; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10192;
10193; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
10194; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10195; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10196; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10197; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10198; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10199; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10200; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10201; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10202; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
10203; GFX90A-TGSPLIT-NEXT:    s_endpgm
10204;
10205; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
10206; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10207; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10208; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10209; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10210; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10211; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10212; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
10213; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10214; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
10215; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10216;
10217; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
10218; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10219; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10220; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10221; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10222; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10223; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10224; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
10225; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10226; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
10227; GFX940-TGSPLIT-NEXT:    s_endpgm
10228    i32 addrspace(1)* %out, i32 %in, i32 %old) {
10229entry:
10230  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
10231  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire seq_cst
10232  %val0 = extractvalue { i32, i1 } %val, 0
10233  store i32 %val0, i32 addrspace(1)* %out, align 4
10234  ret void
10235}
10236
10237define amdgpu_kernel void @global_singlethread_one_as_release_seq_cst_ret_cmpxchg(
10238; GFX6-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
10239; GFX6:       ; %bb.0: ; %entry
10240; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
10241; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10242; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
10243; GFX6-NEXT:    s_mov_b32 s2, -1
10244; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10245; GFX6-NEXT:    v_mov_b32_e32 v0, s6
10246; GFX6-NEXT:    v_mov_b32_e32 v1, s7
10247; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
10248; GFX6-NEXT:    s_waitcnt vmcnt(0)
10249; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
10250; GFX6-NEXT:    s_endpgm
10251;
10252; GFX7-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
10253; GFX7:       ; %bb.0: ; %entry
10254; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10255; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10256; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10257; GFX7-NEXT:    s_add_u32 s4, s0, 16
10258; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10259; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10260; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10261; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10262; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10263; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10264; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10265; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10266; GFX7-NEXT:    s_waitcnt vmcnt(0)
10267; GFX7-NEXT:    flat_store_dword v[0:1], v2
10268; GFX7-NEXT:    s_endpgm
10269;
10270; GFX10-WGP-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
10271; GFX10-WGP:       ; %bb.0: ; %entry
10272; GFX10-WGP-NEXT:    s_clause 0x1
10273; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10274; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10275; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
10276; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10277; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10278; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10279; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10280; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10281; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
10282; GFX10-WGP-NEXT:    s_endpgm
10283;
10284; GFX10-CU-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
10285; GFX10-CU:       ; %bb.0: ; %entry
10286; GFX10-CU-NEXT:    s_clause 0x1
10287; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10288; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10289; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
10290; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10291; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10292; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10293; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10294; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10295; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
10296; GFX10-CU-NEXT:    s_endpgm
10297;
10298; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
10299; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10300; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
10301; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
10302; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
10303; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
10304; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10305; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10306; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10307; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
10308; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10309; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
10310; SKIP-CACHE-INV-NEXT:    s_endpgm
10311;
10312; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
10313; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10314; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10315; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10316; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10317; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10318; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10319; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10320; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10321; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
10322; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10323;
10324; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
10325; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10326; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10327; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10328; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10329; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10330; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10331; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10332; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10333; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
10334; GFX90A-TGSPLIT-NEXT:    s_endpgm
10335;
10336; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
10337; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10338; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10339; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10340; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10341; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10342; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10343; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
10344; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10345; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
10346; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10347;
10348; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
10349; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10350; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10351; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10352; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10353; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10354; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10355; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
10356; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10357; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
10358; GFX940-TGSPLIT-NEXT:    s_endpgm
10359    i32 addrspace(1)* %out, i32 %in, i32 %old) {
10360entry:
10361  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
10362  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release seq_cst
10363  %val0 = extractvalue { i32, i1 } %val, 0
10364  store i32 %val0, i32 addrspace(1)* %out, align 4
10365  ret void
10366}
10367
10368define amdgpu_kernel void @global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg(
10369; GFX6-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
10370; GFX6:       ; %bb.0: ; %entry
10371; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
10372; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10373; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
10374; GFX6-NEXT:    s_mov_b32 s2, -1
10375; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10376; GFX6-NEXT:    v_mov_b32_e32 v0, s6
10377; GFX6-NEXT:    v_mov_b32_e32 v1, s7
10378; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
10379; GFX6-NEXT:    s_waitcnt vmcnt(0)
10380; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
10381; GFX6-NEXT:    s_endpgm
10382;
10383; GFX7-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
10384; GFX7:       ; %bb.0: ; %entry
10385; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10386; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10387; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10388; GFX7-NEXT:    s_add_u32 s4, s0, 16
10389; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10390; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10391; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10392; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10393; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10394; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10395; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10396; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10397; GFX7-NEXT:    s_waitcnt vmcnt(0)
10398; GFX7-NEXT:    flat_store_dword v[0:1], v2
10399; GFX7-NEXT:    s_endpgm
10400;
10401; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
10402; GFX10-WGP:       ; %bb.0: ; %entry
10403; GFX10-WGP-NEXT:    s_clause 0x1
10404; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10405; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10406; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
10407; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10408; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10409; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10410; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10411; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10412; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
10413; GFX10-WGP-NEXT:    s_endpgm
10414;
10415; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
10416; GFX10-CU:       ; %bb.0: ; %entry
10417; GFX10-CU-NEXT:    s_clause 0x1
10418; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10419; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10420; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
10421; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10422; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10423; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10424; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10425; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10426; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
10427; GFX10-CU-NEXT:    s_endpgm
10428;
10429; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
10430; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10431; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
10432; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
10433; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
10434; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
10435; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10436; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10437; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10438; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
10439; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10440; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
10441; SKIP-CACHE-INV-NEXT:    s_endpgm
10442;
10443; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
10444; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10445; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10446; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10447; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10448; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10449; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10450; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10451; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10452; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
10453; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10454;
10455; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
10456; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10457; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10458; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10459; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10460; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10461; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10462; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10463; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10464; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
10465; GFX90A-TGSPLIT-NEXT:    s_endpgm
10466;
10467; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
10468; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10469; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10470; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10471; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10472; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10473; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10474; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
10475; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10476; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
10477; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10478;
10479; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
10480; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10481; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10482; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10483; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10484; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10485; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10486; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
10487; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10488; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
10489; GFX940-TGSPLIT-NEXT:    s_endpgm
10490    i32 addrspace(1)* %out, i32 %in, i32 %old) {
10491entry:
10492  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
10493  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel seq_cst
10494  %val0 = extractvalue { i32, i1 } %val, 0
10495  store i32 %val0, i32 addrspace(1)* %out, align 4
10496  ret void
10497}
10498
10499define amdgpu_kernel void @global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg(
10500; GFX6-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
10501; GFX6:       ; %bb.0: ; %entry
10502; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
10503; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10504; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
10505; GFX6-NEXT:    s_mov_b32 s2, -1
10506; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10507; GFX6-NEXT:    v_mov_b32_e32 v0, s6
10508; GFX6-NEXT:    v_mov_b32_e32 v1, s7
10509; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
10510; GFX6-NEXT:    s_waitcnt vmcnt(0)
10511; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
10512; GFX6-NEXT:    s_endpgm
10513;
10514; GFX7-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
10515; GFX7:       ; %bb.0: ; %entry
10516; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10517; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10518; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10519; GFX7-NEXT:    s_add_u32 s4, s0, 16
10520; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10521; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10522; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10523; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10524; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10525; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10526; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10527; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10528; GFX7-NEXT:    s_waitcnt vmcnt(0)
10529; GFX7-NEXT:    flat_store_dword v[0:1], v2
10530; GFX7-NEXT:    s_endpgm
10531;
10532; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
10533; GFX10-WGP:       ; %bb.0: ; %entry
10534; GFX10-WGP-NEXT:    s_clause 0x1
10535; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10536; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10537; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
10538; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10539; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10540; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10541; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10542; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10543; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
10544; GFX10-WGP-NEXT:    s_endpgm
10545;
10546; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
10547; GFX10-CU:       ; %bb.0: ; %entry
10548; GFX10-CU-NEXT:    s_clause 0x1
10549; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10550; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10551; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
10552; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10553; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10554; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10555; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10556; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10557; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
10558; GFX10-CU-NEXT:    s_endpgm
10559;
10560; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
10561; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10562; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
10563; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
10564; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
10565; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
10566; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10567; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10568; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10569; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
10570; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10571; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
10572; SKIP-CACHE-INV-NEXT:    s_endpgm
10573;
10574; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
10575; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10576; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10577; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10578; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10579; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10580; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10581; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10582; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10583; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
10584; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10585;
10586; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
10587; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10588; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10589; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10590; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10591; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10592; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10593; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10594; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10595; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
10596; GFX90A-TGSPLIT-NEXT:    s_endpgm
10597;
10598; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
10599; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10600; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10601; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10602; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10603; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10604; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10605; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
10606; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10607; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
10608; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10609;
10610; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
10611; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10612; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10613; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10614; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10615; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10616; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10617; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
10618; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10619; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
10620; GFX940-TGSPLIT-NEXT:    s_endpgm
10621    i32 addrspace(1)* %out, i32 %in, i32 %old) {
10622entry:
10623  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
10624  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst seq_cst
10625  %val0 = extractvalue { i32, i1 } %val, 0
10626  store i32 %val0, i32 addrspace(1)* %out, align 4
10627  ret void
10628}
10629
10630