1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx600 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX6 %s
3; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
4; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
5; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
6; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
7; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-NOTTGSPLIT %s
8; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-TGSPLIT %s
9; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-NOTTGSPLIT %s
10; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
11; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11-WGP %s
12; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11-CU %s
13
14define amdgpu_kernel void @global_singlethread_unordered_load(
15; GFX6-LABEL: global_singlethread_unordered_load:
16; GFX6:       ; %bb.0: ; %entry
17; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
18; GFX6-NEXT:    s_mov_b32 s7, 0x100f000
19; GFX6-NEXT:    s_mov_b32 s6, -1
20; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
21; GFX6-NEXT:    s_mov_b32 s4, s0
22; GFX6-NEXT:    s_mov_b32 s5, s1
23; GFX6-NEXT:    buffer_load_dword v0, off, s[4:7], 0
24; GFX6-NEXT:    s_mov_b32 s4, s2
25; GFX6-NEXT:    s_mov_b32 s5, s3
26; GFX6-NEXT:    s_waitcnt vmcnt(0)
27; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
28; GFX6-NEXT:    s_endpgm
29;
30; GFX7-LABEL: global_singlethread_unordered_load:
31; GFX7:       ; %bb.0: ; %entry
32; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
33; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
34; GFX7-NEXT:    v_mov_b32_e32 v0, s0
35; GFX7-NEXT:    v_mov_b32_e32 v1, s1
36; GFX7-NEXT:    flat_load_dword v2, v[0:1]
37; GFX7-NEXT:    v_mov_b32_e32 v0, s2
38; GFX7-NEXT:    v_mov_b32_e32 v1, s3
39; GFX7-NEXT:    s_waitcnt vmcnt(0)
40; GFX7-NEXT:    flat_store_dword v[0:1], v2
41; GFX7-NEXT:    s_endpgm
42;
43; GFX10-WGP-LABEL: global_singlethread_unordered_load:
44; GFX10-WGP:       ; %bb.0: ; %entry
45; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
46; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
47; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
48; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
49; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
50; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
51; GFX10-WGP-NEXT:    s_endpgm
52;
53; GFX10-CU-LABEL: global_singlethread_unordered_load:
54; GFX10-CU:       ; %bb.0: ; %entry
55; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
56; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
57; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
58; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
59; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
60; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
61; GFX10-CU-NEXT:    s_endpgm
62;
63; SKIP-CACHE-INV-LABEL: global_singlethread_unordered_load:
64; SKIP-CACHE-INV:       ; %bb.0: ; %entry
65; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
66; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
67; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
68; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
69; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s0
70; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s1
71; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[4:7], 0
72; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s2
73; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s3
74; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
75; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
76; SKIP-CACHE-INV-NEXT:    s_endpgm
77;
78; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_unordered_load:
79; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
80; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
81; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
82; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
83; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
84; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
85; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
86; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
87;
88; GFX90A-TGSPLIT-LABEL: global_singlethread_unordered_load:
89; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
90; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
91; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
92; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
93; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
94; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
95; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
96; GFX90A-TGSPLIT-NEXT:    s_endpgm
97;
98; GFX940-NOTTGSPLIT-LABEL: global_singlethread_unordered_load:
99; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
100; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
101; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
102; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
103; GFX940-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
104; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
105; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
106; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
107;
108; GFX940-TGSPLIT-LABEL: global_singlethread_unordered_load:
109; GFX940-TGSPLIT:       ; %bb.0: ; %entry
110; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
111; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
112; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
113; GFX940-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
114; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
115; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
116; GFX940-TGSPLIT-NEXT:    s_endpgm
117;
118; GFX11-WGP-LABEL: global_singlethread_unordered_load:
119; GFX11-WGP:       ; %bb.0: ; %entry
120; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
121; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
122; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
123; GFX11-WGP-NEXT:    global_load_b32 v1, v0, s[0:1]
124; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
125; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[2:3]
126; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
127; GFX11-WGP-NEXT:    s_endpgm
128;
129; GFX11-CU-LABEL: global_singlethread_unordered_load:
130; GFX11-CU:       ; %bb.0: ; %entry
131; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
132; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
133; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
134; GFX11-CU-NEXT:    global_load_b32 v1, v0, s[0:1]
135; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
136; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[2:3]
137; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
138; GFX11-CU-NEXT:    s_endpgm
139    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
140entry:
141  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread") unordered, align 4
142  store i32 %val, i32 addrspace(1)* %out
143  ret void
144}
145
146define amdgpu_kernel void @global_singlethread_monotonic_load(
147; GFX6-LABEL: global_singlethread_monotonic_load:
148; GFX6:       ; %bb.0: ; %entry
149; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
150; GFX6-NEXT:    s_mov_b32 s7, 0x100f000
151; GFX6-NEXT:    s_mov_b32 s6, -1
152; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
153; GFX6-NEXT:    s_mov_b32 s4, s0
154; GFX6-NEXT:    s_mov_b32 s5, s1
155; GFX6-NEXT:    buffer_load_dword v0, off, s[4:7], 0
156; GFX6-NEXT:    s_mov_b32 s4, s2
157; GFX6-NEXT:    s_mov_b32 s5, s3
158; GFX6-NEXT:    s_waitcnt vmcnt(0)
159; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
160; GFX6-NEXT:    s_endpgm
161;
162; GFX7-LABEL: global_singlethread_monotonic_load:
163; GFX7:       ; %bb.0: ; %entry
164; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
165; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
166; GFX7-NEXT:    v_mov_b32_e32 v0, s0
167; GFX7-NEXT:    v_mov_b32_e32 v1, s1
168; GFX7-NEXT:    flat_load_dword v2, v[0:1]
169; GFX7-NEXT:    v_mov_b32_e32 v0, s2
170; GFX7-NEXT:    v_mov_b32_e32 v1, s3
171; GFX7-NEXT:    s_waitcnt vmcnt(0)
172; GFX7-NEXT:    flat_store_dword v[0:1], v2
173; GFX7-NEXT:    s_endpgm
174;
175; GFX10-WGP-LABEL: global_singlethread_monotonic_load:
176; GFX10-WGP:       ; %bb.0: ; %entry
177; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
178; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
179; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
180; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
181; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
182; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
183; GFX10-WGP-NEXT:    s_endpgm
184;
185; GFX10-CU-LABEL: global_singlethread_monotonic_load:
186; GFX10-CU:       ; %bb.0: ; %entry
187; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
188; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
189; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
190; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
191; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
192; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
193; GFX10-CU-NEXT:    s_endpgm
194;
195; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_load:
196; SKIP-CACHE-INV:       ; %bb.0: ; %entry
197; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
198; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
199; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
200; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
201; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s0
202; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s1
203; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[4:7], 0
204; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s2
205; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s3
206; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
207; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
208; SKIP-CACHE-INV-NEXT:    s_endpgm
209;
210; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_load:
211; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
212; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
213; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
214; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
215; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
216; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
217; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
218; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
219;
220; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_load:
221; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
222; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
223; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
224; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
225; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
226; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
227; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
228; GFX90A-TGSPLIT-NEXT:    s_endpgm
229;
230; GFX940-NOTTGSPLIT-LABEL: global_singlethread_monotonic_load:
231; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
232; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
233; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
234; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
235; GFX940-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
236; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
237; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
238; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
239;
240; GFX940-TGSPLIT-LABEL: global_singlethread_monotonic_load:
241; GFX940-TGSPLIT:       ; %bb.0: ; %entry
242; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
243; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
244; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
245; GFX940-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
246; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
247; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
248; GFX940-TGSPLIT-NEXT:    s_endpgm
249;
250; GFX11-WGP-LABEL: global_singlethread_monotonic_load:
251; GFX11-WGP:       ; %bb.0: ; %entry
252; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
253; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
254; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
255; GFX11-WGP-NEXT:    global_load_b32 v1, v0, s[0:1]
256; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
257; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[2:3]
258; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
259; GFX11-WGP-NEXT:    s_endpgm
260;
261; GFX11-CU-LABEL: global_singlethread_monotonic_load:
262; GFX11-CU:       ; %bb.0: ; %entry
263; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
264; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
265; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
266; GFX11-CU-NEXT:    global_load_b32 v1, v0, s[0:1]
267; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
268; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[2:3]
269; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
270; GFX11-CU-NEXT:    s_endpgm
271    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
272entry:
273  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread") monotonic, align 4
274  store i32 %val, i32 addrspace(1)* %out
275  ret void
276}
277
278define amdgpu_kernel void @global_singlethread_acquire_load(
279; GFX6-LABEL: global_singlethread_acquire_load:
280; GFX6:       ; %bb.0: ; %entry
281; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
282; GFX6-NEXT:    s_mov_b32 s7, 0x100f000
283; GFX6-NEXT:    s_mov_b32 s6, -1
284; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
285; GFX6-NEXT:    s_mov_b32 s4, s0
286; GFX6-NEXT:    s_mov_b32 s5, s1
287; GFX6-NEXT:    buffer_load_dword v0, off, s[4:7], 0
288; GFX6-NEXT:    s_mov_b32 s4, s2
289; GFX6-NEXT:    s_mov_b32 s5, s3
290; GFX6-NEXT:    s_waitcnt vmcnt(0)
291; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
292; GFX6-NEXT:    s_endpgm
293;
294; GFX7-LABEL: global_singlethread_acquire_load:
295; GFX7:       ; %bb.0: ; %entry
296; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
297; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
298; GFX7-NEXT:    v_mov_b32_e32 v0, s0
299; GFX7-NEXT:    v_mov_b32_e32 v1, s1
300; GFX7-NEXT:    flat_load_dword v2, v[0:1]
301; GFX7-NEXT:    v_mov_b32_e32 v0, s2
302; GFX7-NEXT:    v_mov_b32_e32 v1, s3
303; GFX7-NEXT:    s_waitcnt vmcnt(0)
304; GFX7-NEXT:    flat_store_dword v[0:1], v2
305; GFX7-NEXT:    s_endpgm
306;
307; GFX10-WGP-LABEL: global_singlethread_acquire_load:
308; GFX10-WGP:       ; %bb.0: ; %entry
309; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
310; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
311; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
312; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
313; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
314; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
315; GFX10-WGP-NEXT:    s_endpgm
316;
317; GFX10-CU-LABEL: global_singlethread_acquire_load:
318; GFX10-CU:       ; %bb.0: ; %entry
319; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
320; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
321; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
322; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
323; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
324; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
325; GFX10-CU-NEXT:    s_endpgm
326;
327; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_load:
328; SKIP-CACHE-INV:       ; %bb.0: ; %entry
329; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
330; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
331; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
332; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
333; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s0
334; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s1
335; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[4:7], 0
336; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s2
337; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s3
338; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
339; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
340; SKIP-CACHE-INV-NEXT:    s_endpgm
341;
342; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_load:
343; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
344; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
345; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
346; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
347; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
348; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
349; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
350; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
351;
352; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_load:
353; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
354; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
355; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
356; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
357; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
358; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
359; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
360; GFX90A-TGSPLIT-NEXT:    s_endpgm
361;
362; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acquire_load:
363; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
364; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
365; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
366; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
367; GFX940-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
368; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
369; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
370; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
371;
372; GFX940-TGSPLIT-LABEL: global_singlethread_acquire_load:
373; GFX940-TGSPLIT:       ; %bb.0: ; %entry
374; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
375; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
376; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
377; GFX940-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
378; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
379; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
380; GFX940-TGSPLIT-NEXT:    s_endpgm
381;
382; GFX11-WGP-LABEL: global_singlethread_acquire_load:
383; GFX11-WGP:       ; %bb.0: ; %entry
384; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
385; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
386; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
387; GFX11-WGP-NEXT:    global_load_b32 v1, v0, s[0:1]
388; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
389; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[2:3]
390; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
391; GFX11-WGP-NEXT:    s_endpgm
392;
393; GFX11-CU-LABEL: global_singlethread_acquire_load:
394; GFX11-CU:       ; %bb.0: ; %entry
395; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
396; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
397; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
398; GFX11-CU-NEXT:    global_load_b32 v1, v0, s[0:1]
399; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
400; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[2:3]
401; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
402; GFX11-CU-NEXT:    s_endpgm
403    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
404entry:
405  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread") acquire, align 4
406  store i32 %val, i32 addrspace(1)* %out
407  ret void
408}
409
410define amdgpu_kernel void @global_singlethread_seq_cst_load(
411; GFX6-LABEL: global_singlethread_seq_cst_load:
412; GFX6:       ; %bb.0: ; %entry
413; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
414; GFX6-NEXT:    s_mov_b32 s7, 0x100f000
415; GFX6-NEXT:    s_mov_b32 s6, -1
416; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
417; GFX6-NEXT:    s_mov_b32 s4, s0
418; GFX6-NEXT:    s_mov_b32 s5, s1
419; GFX6-NEXT:    buffer_load_dword v0, off, s[4:7], 0
420; GFX6-NEXT:    s_mov_b32 s4, s2
421; GFX6-NEXT:    s_mov_b32 s5, s3
422; GFX6-NEXT:    s_waitcnt vmcnt(0)
423; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
424; GFX6-NEXT:    s_endpgm
425;
426; GFX7-LABEL: global_singlethread_seq_cst_load:
427; GFX7:       ; %bb.0: ; %entry
428; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
429; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
430; GFX7-NEXT:    v_mov_b32_e32 v0, s0
431; GFX7-NEXT:    v_mov_b32_e32 v1, s1
432; GFX7-NEXT:    flat_load_dword v2, v[0:1]
433; GFX7-NEXT:    v_mov_b32_e32 v0, s2
434; GFX7-NEXT:    v_mov_b32_e32 v1, s3
435; GFX7-NEXT:    s_waitcnt vmcnt(0)
436; GFX7-NEXT:    flat_store_dword v[0:1], v2
437; GFX7-NEXT:    s_endpgm
438;
439; GFX10-WGP-LABEL: global_singlethread_seq_cst_load:
440; GFX10-WGP:       ; %bb.0: ; %entry
441; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
442; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
443; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
444; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
445; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
446; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
447; GFX10-WGP-NEXT:    s_endpgm
448;
449; GFX10-CU-LABEL: global_singlethread_seq_cst_load:
450; GFX10-CU:       ; %bb.0: ; %entry
451; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
452; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
453; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
454; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
455; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
456; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
457; GFX10-CU-NEXT:    s_endpgm
458;
459; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_load:
460; SKIP-CACHE-INV:       ; %bb.0: ; %entry
461; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
462; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
463; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
464; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
465; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s0
466; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s1
467; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[4:7], 0
468; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s2
469; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s3
470; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
471; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
472; SKIP-CACHE-INV-NEXT:    s_endpgm
473;
474; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_load:
475; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
476; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
477; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
478; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
479; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
480; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
481; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
482; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
483;
484; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_load:
485; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
486; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
487; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
488; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
489; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
490; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
491; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
492; GFX90A-TGSPLIT-NEXT:    s_endpgm
493;
494; GFX940-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_load:
495; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
496; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
497; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
498; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
499; GFX940-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
500; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
501; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
502; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
503;
504; GFX940-TGSPLIT-LABEL: global_singlethread_seq_cst_load:
505; GFX940-TGSPLIT:       ; %bb.0: ; %entry
506; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
507; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
508; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
509; GFX940-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
510; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
511; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
512; GFX940-TGSPLIT-NEXT:    s_endpgm
513;
514; GFX11-WGP-LABEL: global_singlethread_seq_cst_load:
515; GFX11-WGP:       ; %bb.0: ; %entry
516; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
517; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
518; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
519; GFX11-WGP-NEXT:    global_load_b32 v1, v0, s[0:1]
520; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
521; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[2:3]
522; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
523; GFX11-WGP-NEXT:    s_endpgm
524;
525; GFX11-CU-LABEL: global_singlethread_seq_cst_load:
526; GFX11-CU:       ; %bb.0: ; %entry
527; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
528; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
529; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
530; GFX11-CU-NEXT:    global_load_b32 v1, v0, s[0:1]
531; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
532; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[2:3]
533; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
534; GFX11-CU-NEXT:    s_endpgm
535    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
536entry:
537  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread") seq_cst, align 4
538  store i32 %val, i32 addrspace(1)* %out
539  ret void
540}
541
542define amdgpu_kernel void @global_singlethread_unordered_store(
543; GFX6-LABEL: global_singlethread_unordered_store:
544; GFX6:       ; %bb.0: ; %entry
545; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
546; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
547; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
548; GFX6-NEXT:    s_mov_b32 s2, -1
549; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
550; GFX6-NEXT:    v_mov_b32_e32 v0, s6
551; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
552; GFX6-NEXT:    s_endpgm
553;
554; GFX7-LABEL: global_singlethread_unordered_store:
555; GFX7:       ; %bb.0: ; %entry
556; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
557; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
558; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
559; GFX7-NEXT:    v_mov_b32_e32 v0, s0
560; GFX7-NEXT:    v_mov_b32_e32 v1, s1
561; GFX7-NEXT:    v_mov_b32_e32 v2, s2
562; GFX7-NEXT:    flat_store_dword v[0:1], v2
563; GFX7-NEXT:    s_endpgm
564;
565; GFX10-WGP-LABEL: global_singlethread_unordered_store:
566; GFX10-WGP:       ; %bb.0: ; %entry
567; GFX10-WGP-NEXT:    s_clause 0x1
568; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
569; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
570; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
571; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
572; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
573; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
574; GFX10-WGP-NEXT:    s_endpgm
575;
576; GFX10-CU-LABEL: global_singlethread_unordered_store:
577; GFX10-CU:       ; %bb.0: ; %entry
578; GFX10-CU-NEXT:    s_clause 0x1
579; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
580; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
581; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
582; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
583; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
584; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
585; GFX10-CU-NEXT:    s_endpgm
586;
587; SKIP-CACHE-INV-LABEL: global_singlethread_unordered_store:
588; SKIP-CACHE-INV:       ; %bb.0: ; %entry
589; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x0
590; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
591; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
592; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
593; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
594; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
595; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
596; SKIP-CACHE-INV-NEXT:    s_endpgm
597;
598; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_unordered_store:
599; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
600; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
601; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
602; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
603; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
604; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
605; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
606; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
607;
608; GFX90A-TGSPLIT-LABEL: global_singlethread_unordered_store:
609; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
610; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
611; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
612; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
613; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
614; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
615; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
616; GFX90A-TGSPLIT-NEXT:    s_endpgm
617;
618; GFX940-NOTTGSPLIT-LABEL: global_singlethread_unordered_store:
619; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
620; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
621; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
622; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
623; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
624; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
625; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
626; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
627;
628; GFX940-TGSPLIT-LABEL: global_singlethread_unordered_store:
629; GFX940-TGSPLIT:       ; %bb.0: ; %entry
630; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
631; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
632; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
633; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
634; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
635; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
636; GFX940-TGSPLIT-NEXT:    s_endpgm
637;
638; GFX11-WGP-LABEL: global_singlethread_unordered_store:
639; GFX11-WGP:       ; %bb.0: ; %entry
640; GFX11-WGP-NEXT:    s_clause 0x1
641; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x0
642; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
643; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
644; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
645; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[0:1]
646; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
647; GFX11-WGP-NEXT:    s_endpgm
648;
649; GFX11-CU-LABEL: global_singlethread_unordered_store:
650; GFX11-CU:       ; %bb.0: ; %entry
651; GFX11-CU-NEXT:    s_clause 0x1
652; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x0
653; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
654; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
655; GFX11-CU-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
656; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
657; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
658; GFX11-CU-NEXT:    s_endpgm
659    i32 %in, i32 addrspace(1)* %out) {
660entry:
661  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread") unordered, align 4
662  ret void
663}
664
665define amdgpu_kernel void @global_singlethread_monotonic_store(
666; GFX6-LABEL: global_singlethread_monotonic_store:
667; GFX6:       ; %bb.0: ; %entry
668; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
669; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
670; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
671; GFX6-NEXT:    s_mov_b32 s2, -1
672; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
673; GFX6-NEXT:    v_mov_b32_e32 v0, s6
674; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
675; GFX6-NEXT:    s_endpgm
676;
677; GFX7-LABEL: global_singlethread_monotonic_store:
678; GFX7:       ; %bb.0: ; %entry
679; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
680; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
681; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
682; GFX7-NEXT:    v_mov_b32_e32 v0, s0
683; GFX7-NEXT:    v_mov_b32_e32 v1, s1
684; GFX7-NEXT:    v_mov_b32_e32 v2, s2
685; GFX7-NEXT:    flat_store_dword v[0:1], v2
686; GFX7-NEXT:    s_endpgm
687;
688; GFX10-WGP-LABEL: global_singlethread_monotonic_store:
689; GFX10-WGP:       ; %bb.0: ; %entry
690; GFX10-WGP-NEXT:    s_clause 0x1
691; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
692; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
693; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
694; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
695; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
696; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
697; GFX10-WGP-NEXT:    s_endpgm
698;
699; GFX10-CU-LABEL: global_singlethread_monotonic_store:
700; GFX10-CU:       ; %bb.0: ; %entry
701; GFX10-CU-NEXT:    s_clause 0x1
702; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
703; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
704; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
705; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
706; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
707; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
708; GFX10-CU-NEXT:    s_endpgm
709;
710; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_store:
711; SKIP-CACHE-INV:       ; %bb.0: ; %entry
712; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x0
713; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
714; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
715; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
716; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
717; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
718; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
719; SKIP-CACHE-INV-NEXT:    s_endpgm
720;
721; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_store:
722; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
723; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
724; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
725; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
726; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
727; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
728; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
729; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
730;
731; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_store:
732; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
733; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
734; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
735; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
736; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
737; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
738; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
739; GFX90A-TGSPLIT-NEXT:    s_endpgm
740;
741; GFX940-NOTTGSPLIT-LABEL: global_singlethread_monotonic_store:
742; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
743; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
744; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
745; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
746; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
747; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
748; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
749; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
750;
751; GFX940-TGSPLIT-LABEL: global_singlethread_monotonic_store:
752; GFX940-TGSPLIT:       ; %bb.0: ; %entry
753; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
754; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
755; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
756; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
757; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
758; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
759; GFX940-TGSPLIT-NEXT:    s_endpgm
760;
761; GFX11-WGP-LABEL: global_singlethread_monotonic_store:
762; GFX11-WGP:       ; %bb.0: ; %entry
763; GFX11-WGP-NEXT:    s_clause 0x1
764; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x0
765; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
766; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
767; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
768; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[0:1]
769; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
770; GFX11-WGP-NEXT:    s_endpgm
771;
772; GFX11-CU-LABEL: global_singlethread_monotonic_store:
773; GFX11-CU:       ; %bb.0: ; %entry
774; GFX11-CU-NEXT:    s_clause 0x1
775; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x0
776; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
777; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
778; GFX11-CU-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
779; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
780; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
781; GFX11-CU-NEXT:    s_endpgm
782    i32 %in, i32 addrspace(1)* %out) {
783entry:
784  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread") monotonic, align 4
785  ret void
786}
787
788define amdgpu_kernel void @global_singlethread_release_store(
789; GFX6-LABEL: global_singlethread_release_store:
790; GFX6:       ; %bb.0: ; %entry
791; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
792; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
793; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
794; GFX6-NEXT:    s_mov_b32 s2, -1
795; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
796; GFX6-NEXT:    v_mov_b32_e32 v0, s6
797; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
798; GFX6-NEXT:    s_endpgm
799;
800; GFX7-LABEL: global_singlethread_release_store:
801; GFX7:       ; %bb.0: ; %entry
802; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
803; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
804; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
805; GFX7-NEXT:    v_mov_b32_e32 v0, s0
806; GFX7-NEXT:    v_mov_b32_e32 v1, s1
807; GFX7-NEXT:    v_mov_b32_e32 v2, s2
808; GFX7-NEXT:    flat_store_dword v[0:1], v2
809; GFX7-NEXT:    s_endpgm
810;
811; GFX10-WGP-LABEL: global_singlethread_release_store:
812; GFX10-WGP:       ; %bb.0: ; %entry
813; GFX10-WGP-NEXT:    s_clause 0x1
814; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
815; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
816; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
817; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
818; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
819; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
820; GFX10-WGP-NEXT:    s_endpgm
821;
822; GFX10-CU-LABEL: global_singlethread_release_store:
823; GFX10-CU:       ; %bb.0: ; %entry
824; GFX10-CU-NEXT:    s_clause 0x1
825; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
826; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
827; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
828; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
829; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
830; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
831; GFX10-CU-NEXT:    s_endpgm
832;
833; SKIP-CACHE-INV-LABEL: global_singlethread_release_store:
834; SKIP-CACHE-INV:       ; %bb.0: ; %entry
835; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x0
836; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
837; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
838; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
839; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
840; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
841; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
842; SKIP-CACHE-INV-NEXT:    s_endpgm
843;
844; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_release_store:
845; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
846; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
847; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
848; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
849; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
850; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
851; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
852; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
853;
854; GFX90A-TGSPLIT-LABEL: global_singlethread_release_store:
855; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
856; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
857; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
858; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
859; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
860; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
861; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
862; GFX90A-TGSPLIT-NEXT:    s_endpgm
863;
864; GFX940-NOTTGSPLIT-LABEL: global_singlethread_release_store:
865; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
866; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
867; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
868; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
869; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
870; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
871; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
872; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
873;
874; GFX940-TGSPLIT-LABEL: global_singlethread_release_store:
875; GFX940-TGSPLIT:       ; %bb.0: ; %entry
876; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
877; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
878; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
879; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
880; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
881; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
882; GFX940-TGSPLIT-NEXT:    s_endpgm
883;
884; GFX11-WGP-LABEL: global_singlethread_release_store:
885; GFX11-WGP:       ; %bb.0: ; %entry
886; GFX11-WGP-NEXT:    s_clause 0x1
887; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x0
888; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
889; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
890; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
891; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[0:1]
892; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
893; GFX11-WGP-NEXT:    s_endpgm
894;
895; GFX11-CU-LABEL: global_singlethread_release_store:
896; GFX11-CU:       ; %bb.0: ; %entry
897; GFX11-CU-NEXT:    s_clause 0x1
898; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x0
899; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
900; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
901; GFX11-CU-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
902; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
903; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
904; GFX11-CU-NEXT:    s_endpgm
905    i32 %in, i32 addrspace(1)* %out) {
906entry:
907  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread") release, align 4
908  ret void
909}
910
911define amdgpu_kernel void @global_singlethread_seq_cst_store(
912; GFX6-LABEL: global_singlethread_seq_cst_store:
913; GFX6:       ; %bb.0: ; %entry
914; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
915; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
916; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
917; GFX6-NEXT:    s_mov_b32 s2, -1
918; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
919; GFX6-NEXT:    v_mov_b32_e32 v0, s6
920; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
921; GFX6-NEXT:    s_endpgm
922;
923; GFX7-LABEL: global_singlethread_seq_cst_store:
924; GFX7:       ; %bb.0: ; %entry
925; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
926; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
927; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
928; GFX7-NEXT:    v_mov_b32_e32 v0, s0
929; GFX7-NEXT:    v_mov_b32_e32 v1, s1
930; GFX7-NEXT:    v_mov_b32_e32 v2, s2
931; GFX7-NEXT:    flat_store_dword v[0:1], v2
932; GFX7-NEXT:    s_endpgm
933;
934; GFX10-WGP-LABEL: global_singlethread_seq_cst_store:
935; GFX10-WGP:       ; %bb.0: ; %entry
936; GFX10-WGP-NEXT:    s_clause 0x1
937; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
938; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
939; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
940; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
941; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
942; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
943; GFX10-WGP-NEXT:    s_endpgm
944;
945; GFX10-CU-LABEL: global_singlethread_seq_cst_store:
946; GFX10-CU:       ; %bb.0: ; %entry
947; GFX10-CU-NEXT:    s_clause 0x1
948; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
949; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
950; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
951; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
952; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
953; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
954; GFX10-CU-NEXT:    s_endpgm
955;
956; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_store:
957; SKIP-CACHE-INV:       ; %bb.0: ; %entry
958; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x0
959; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
960; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
961; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
962; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
963; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
964; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
965; SKIP-CACHE-INV-NEXT:    s_endpgm
966;
967; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_store:
968; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
969; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
970; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
971; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
972; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
973; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
974; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
975; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
976;
977; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_store:
978; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
979; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
980; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
981; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
982; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
983; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
984; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
985; GFX90A-TGSPLIT-NEXT:    s_endpgm
986;
987; GFX940-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_store:
988; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
989; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
990; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
991; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
992; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
993; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
994; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
995; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
996;
997; GFX940-TGSPLIT-LABEL: global_singlethread_seq_cst_store:
998; GFX940-TGSPLIT:       ; %bb.0: ; %entry
999; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
1000; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
1001; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1002; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1003; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1004; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
1005; GFX940-TGSPLIT-NEXT:    s_endpgm
1006;
1007; GFX11-WGP-LABEL: global_singlethread_seq_cst_store:
1008; GFX11-WGP:       ; %bb.0: ; %entry
1009; GFX11-WGP-NEXT:    s_clause 0x1
1010; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x0
1011; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
1012; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1013; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
1014; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[0:1]
1015; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1016; GFX11-WGP-NEXT:    s_endpgm
1017;
1018; GFX11-CU-LABEL: global_singlethread_seq_cst_store:
1019; GFX11-CU:       ; %bb.0: ; %entry
1020; GFX11-CU-NEXT:    s_clause 0x1
1021; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x0
1022; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
1023; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1024; GFX11-CU-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
1025; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
1026; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1027; GFX11-CU-NEXT:    s_endpgm
1028    i32 %in, i32 addrspace(1)* %out) {
1029entry:
1030  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread") seq_cst, align 4
1031  ret void
1032}
1033
1034define amdgpu_kernel void @global_singlethread_monotonic_atomicrmw(
1035; GFX6-LABEL: global_singlethread_monotonic_atomicrmw:
1036; GFX6:       ; %bb.0: ; %entry
1037; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
1038; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1039; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1040; GFX6-NEXT:    s_mov_b32 s2, -1
1041; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1042; GFX6-NEXT:    v_mov_b32_e32 v0, s6
1043; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
1044; GFX6-NEXT:    s_endpgm
1045;
1046; GFX7-LABEL: global_singlethread_monotonic_atomicrmw:
1047; GFX7:       ; %bb.0: ; %entry
1048; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1049; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1050; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1051; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1052; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1053; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1054; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1055; GFX7-NEXT:    s_endpgm
1056;
1057; GFX10-WGP-LABEL: global_singlethread_monotonic_atomicrmw:
1058; GFX10-WGP:       ; %bb.0: ; %entry
1059; GFX10-WGP-NEXT:    s_clause 0x1
1060; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1061; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1062; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
1063; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1064; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1065; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
1066; GFX10-WGP-NEXT:    s_endpgm
1067;
1068; GFX10-CU-LABEL: global_singlethread_monotonic_atomicrmw:
1069; GFX10-CU:       ; %bb.0: ; %entry
1070; GFX10-CU-NEXT:    s_clause 0x1
1071; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1072; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1073; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
1074; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1075; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1076; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
1077; GFX10-CU-NEXT:    s_endpgm
1078;
1079; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_atomicrmw:
1080; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1081; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
1082; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1083; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
1084; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
1085; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1086; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
1087; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
1088; SKIP-CACHE-INV-NEXT:    s_endpgm
1089;
1090; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_atomicrmw:
1091; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1092; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1093; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1094; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1095; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1096; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1097; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1098; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1099;
1100; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_atomicrmw:
1101; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1102; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1103; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1104; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1105; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1106; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1107; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1108; GFX90A-TGSPLIT-NEXT:    s_endpgm
1109;
1110; GFX940-NOTTGSPLIT-LABEL: global_singlethread_monotonic_atomicrmw:
1111; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1112; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1113; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1114; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1115; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1116; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1117; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
1118; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1119;
1120; GFX940-TGSPLIT-LABEL: global_singlethread_monotonic_atomicrmw:
1121; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1122; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1123; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1124; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1125; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1126; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1127; GFX940-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
1128; GFX940-TGSPLIT-NEXT:    s_endpgm
1129;
1130; GFX11-WGP-LABEL: global_singlethread_monotonic_atomicrmw:
1131; GFX11-WGP:       ; %bb.0: ; %entry
1132; GFX11-WGP-NEXT:    s_clause 0x1
1133; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x8
1134; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1135; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1136; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
1137; GFX11-WGP-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
1138; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1139; GFX11-WGP-NEXT:    s_endpgm
1140;
1141; GFX11-CU-LABEL: global_singlethread_monotonic_atomicrmw:
1142; GFX11-CU:       ; %bb.0: ; %entry
1143; GFX11-CU-NEXT:    s_clause 0x1
1144; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x8
1145; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1146; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1147; GFX11-CU-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
1148; GFX11-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
1149; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1150; GFX11-CU-NEXT:    s_endpgm
1151    i32 addrspace(1)* %out, i32 %in) {
1152entry:
1153  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") monotonic
1154  ret void
1155}
1156
1157define amdgpu_kernel void @global_singlethread_acquire_atomicrmw(
1158; GFX6-LABEL: global_singlethread_acquire_atomicrmw:
1159; GFX6:       ; %bb.0: ; %entry
1160; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
1161; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1162; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1163; GFX6-NEXT:    s_mov_b32 s2, -1
1164; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1165; GFX6-NEXT:    v_mov_b32_e32 v0, s6
1166; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
1167; GFX6-NEXT:    s_endpgm
1168;
1169; GFX7-LABEL: global_singlethread_acquire_atomicrmw:
1170; GFX7:       ; %bb.0: ; %entry
1171; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1172; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1173; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1174; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1175; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1176; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1177; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1178; GFX7-NEXT:    s_endpgm
1179;
1180; GFX10-WGP-LABEL: global_singlethread_acquire_atomicrmw:
1181; GFX10-WGP:       ; %bb.0: ; %entry
1182; GFX10-WGP-NEXT:    s_clause 0x1
1183; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1184; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1185; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
1186; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1187; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1188; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
1189; GFX10-WGP-NEXT:    s_endpgm
1190;
1191; GFX10-CU-LABEL: global_singlethread_acquire_atomicrmw:
1192; GFX10-CU:       ; %bb.0: ; %entry
1193; GFX10-CU-NEXT:    s_clause 0x1
1194; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1195; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1196; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
1197; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1198; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1199; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
1200; GFX10-CU-NEXT:    s_endpgm
1201;
1202; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_atomicrmw:
1203; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1204; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
1205; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1206; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
1207; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
1208; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1209; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
1210; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
1211; SKIP-CACHE-INV-NEXT:    s_endpgm
1212;
1213; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_atomicrmw:
1214; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1215; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1216; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1217; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1218; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1219; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1220; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1221; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1222;
1223; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_atomicrmw:
1224; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1225; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1226; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1227; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1228; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1229; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1230; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1231; GFX90A-TGSPLIT-NEXT:    s_endpgm
1232;
1233; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acquire_atomicrmw:
1234; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1235; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1236; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1237; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1238; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1239; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1240; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
1241; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1242;
1243; GFX940-TGSPLIT-LABEL: global_singlethread_acquire_atomicrmw:
1244; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1245; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1246; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1247; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1248; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1249; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1250; GFX940-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
1251; GFX940-TGSPLIT-NEXT:    s_endpgm
1252;
1253; GFX11-WGP-LABEL: global_singlethread_acquire_atomicrmw:
1254; GFX11-WGP:       ; %bb.0: ; %entry
1255; GFX11-WGP-NEXT:    s_clause 0x1
1256; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x8
1257; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1258; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1259; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
1260; GFX11-WGP-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
1261; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1262; GFX11-WGP-NEXT:    s_endpgm
1263;
1264; GFX11-CU-LABEL: global_singlethread_acquire_atomicrmw:
1265; GFX11-CU:       ; %bb.0: ; %entry
1266; GFX11-CU-NEXT:    s_clause 0x1
1267; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x8
1268; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1269; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1270; GFX11-CU-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
1271; GFX11-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
1272; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1273; GFX11-CU-NEXT:    s_endpgm
1274    i32 addrspace(1)* %out, i32 %in) {
1275entry:
1276  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") acquire
1277  ret void
1278}
1279
1280define amdgpu_kernel void @global_singlethread_release_atomicrmw(
1281; GFX6-LABEL: global_singlethread_release_atomicrmw:
1282; GFX6:       ; %bb.0: ; %entry
1283; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
1284; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1285; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1286; GFX6-NEXT:    s_mov_b32 s2, -1
1287; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1288; GFX6-NEXT:    v_mov_b32_e32 v0, s6
1289; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
1290; GFX6-NEXT:    s_endpgm
1291;
1292; GFX7-LABEL: global_singlethread_release_atomicrmw:
1293; GFX7:       ; %bb.0: ; %entry
1294; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1295; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1296; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1297; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1298; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1299; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1300; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1301; GFX7-NEXT:    s_endpgm
1302;
1303; GFX10-WGP-LABEL: global_singlethread_release_atomicrmw:
1304; GFX10-WGP:       ; %bb.0: ; %entry
1305; GFX10-WGP-NEXT:    s_clause 0x1
1306; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1307; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1308; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
1309; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1310; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1311; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
1312; GFX10-WGP-NEXT:    s_endpgm
1313;
1314; GFX10-CU-LABEL: global_singlethread_release_atomicrmw:
1315; GFX10-CU:       ; %bb.0: ; %entry
1316; GFX10-CU-NEXT:    s_clause 0x1
1317; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1318; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1319; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
1320; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1321; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1322; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
1323; GFX10-CU-NEXT:    s_endpgm
1324;
1325; SKIP-CACHE-INV-LABEL: global_singlethread_release_atomicrmw:
1326; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1327; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
1328; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1329; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
1330; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
1331; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1332; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
1333; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
1334; SKIP-CACHE-INV-NEXT:    s_endpgm
1335;
1336; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_release_atomicrmw:
1337; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1338; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1339; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1340; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1341; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1342; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1343; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1344; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1345;
1346; GFX90A-TGSPLIT-LABEL: global_singlethread_release_atomicrmw:
1347; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1348; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1349; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1350; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1351; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1352; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1353; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1354; GFX90A-TGSPLIT-NEXT:    s_endpgm
1355;
1356; GFX940-NOTTGSPLIT-LABEL: global_singlethread_release_atomicrmw:
1357; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1358; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1359; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1360; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1361; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1362; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1363; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
1364; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1365;
1366; GFX940-TGSPLIT-LABEL: global_singlethread_release_atomicrmw:
1367; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1368; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1369; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1370; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1371; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1372; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1373; GFX940-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
1374; GFX940-TGSPLIT-NEXT:    s_endpgm
1375;
1376; GFX11-WGP-LABEL: global_singlethread_release_atomicrmw:
1377; GFX11-WGP:       ; %bb.0: ; %entry
1378; GFX11-WGP-NEXT:    s_clause 0x1
1379; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x8
1380; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1381; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1382; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
1383; GFX11-WGP-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
1384; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1385; GFX11-WGP-NEXT:    s_endpgm
1386;
1387; GFX11-CU-LABEL: global_singlethread_release_atomicrmw:
1388; GFX11-CU:       ; %bb.0: ; %entry
1389; GFX11-CU-NEXT:    s_clause 0x1
1390; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x8
1391; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1392; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1393; GFX11-CU-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
1394; GFX11-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
1395; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1396; GFX11-CU-NEXT:    s_endpgm
1397    i32 addrspace(1)* %out, i32 %in) {
1398entry:
1399  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") release
1400  ret void
1401}
1402
1403define amdgpu_kernel void @global_singlethread_acq_rel_atomicrmw(
1404; GFX6-LABEL: global_singlethread_acq_rel_atomicrmw:
1405; GFX6:       ; %bb.0: ; %entry
1406; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
1407; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1408; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1409; GFX6-NEXT:    s_mov_b32 s2, -1
1410; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1411; GFX6-NEXT:    v_mov_b32_e32 v0, s6
1412; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
1413; GFX6-NEXT:    s_endpgm
1414;
1415; GFX7-LABEL: global_singlethread_acq_rel_atomicrmw:
1416; GFX7:       ; %bb.0: ; %entry
1417; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1418; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1419; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1420; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1421; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1422; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1423; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1424; GFX7-NEXT:    s_endpgm
1425;
1426; GFX10-WGP-LABEL: global_singlethread_acq_rel_atomicrmw:
1427; GFX10-WGP:       ; %bb.0: ; %entry
1428; GFX10-WGP-NEXT:    s_clause 0x1
1429; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1430; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1431; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
1432; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1433; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1434; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
1435; GFX10-WGP-NEXT:    s_endpgm
1436;
1437; GFX10-CU-LABEL: global_singlethread_acq_rel_atomicrmw:
1438; GFX10-CU:       ; %bb.0: ; %entry
1439; GFX10-CU-NEXT:    s_clause 0x1
1440; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1441; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1442; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
1443; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1444; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1445; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
1446; GFX10-CU-NEXT:    s_endpgm
1447;
1448; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_atomicrmw:
1449; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1450; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
1451; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1452; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
1453; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
1454; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1455; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
1456; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
1457; SKIP-CACHE-INV-NEXT:    s_endpgm
1458;
1459; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_atomicrmw:
1460; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1461; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1462; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1463; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1464; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1465; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1466; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1467; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1468;
1469; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_atomicrmw:
1470; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1471; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1472; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1473; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1474; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1475; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1476; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1477; GFX90A-TGSPLIT-NEXT:    s_endpgm
1478;
1479; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_atomicrmw:
1480; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1481; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1482; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1483; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1484; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1485; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1486; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
1487; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1488;
1489; GFX940-TGSPLIT-LABEL: global_singlethread_acq_rel_atomicrmw:
1490; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1491; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1492; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1493; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1494; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1495; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1496; GFX940-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
1497; GFX940-TGSPLIT-NEXT:    s_endpgm
1498;
1499; GFX11-WGP-LABEL: global_singlethread_acq_rel_atomicrmw:
1500; GFX11-WGP:       ; %bb.0: ; %entry
1501; GFX11-WGP-NEXT:    s_clause 0x1
1502; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x8
1503; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1504; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1505; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
1506; GFX11-WGP-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
1507; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1508; GFX11-WGP-NEXT:    s_endpgm
1509;
1510; GFX11-CU-LABEL: global_singlethread_acq_rel_atomicrmw:
1511; GFX11-CU:       ; %bb.0: ; %entry
1512; GFX11-CU-NEXT:    s_clause 0x1
1513; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x8
1514; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1515; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1516; GFX11-CU-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
1517; GFX11-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
1518; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1519; GFX11-CU-NEXT:    s_endpgm
1520    i32 addrspace(1)* %out, i32 %in) {
1521entry:
1522  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") acq_rel
1523  ret void
1524}
1525
1526define amdgpu_kernel void @global_singlethread_seq_cst_atomicrmw(
1527; GFX6-LABEL: global_singlethread_seq_cst_atomicrmw:
1528; GFX6:       ; %bb.0: ; %entry
1529; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
1530; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1531; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1532; GFX6-NEXT:    s_mov_b32 s2, -1
1533; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1534; GFX6-NEXT:    v_mov_b32_e32 v0, s6
1535; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
1536; GFX6-NEXT:    s_endpgm
1537;
1538; GFX7-LABEL: global_singlethread_seq_cst_atomicrmw:
1539; GFX7:       ; %bb.0: ; %entry
1540; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1541; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1542; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1543; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1544; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1545; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1546; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1547; GFX7-NEXT:    s_endpgm
1548;
1549; GFX10-WGP-LABEL: global_singlethread_seq_cst_atomicrmw:
1550; GFX10-WGP:       ; %bb.0: ; %entry
1551; GFX10-WGP-NEXT:    s_clause 0x1
1552; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1553; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1554; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
1555; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1556; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1557; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
1558; GFX10-WGP-NEXT:    s_endpgm
1559;
1560; GFX10-CU-LABEL: global_singlethread_seq_cst_atomicrmw:
1561; GFX10-CU:       ; %bb.0: ; %entry
1562; GFX10-CU-NEXT:    s_clause 0x1
1563; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1564; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1565; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
1566; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1567; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1568; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
1569; GFX10-CU-NEXT:    s_endpgm
1570;
1571; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_atomicrmw:
1572; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1573; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
1574; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1575; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
1576; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
1577; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1578; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
1579; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
1580; SKIP-CACHE-INV-NEXT:    s_endpgm
1581;
1582; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_atomicrmw:
1583; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1584; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1585; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1586; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1587; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1588; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1589; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1590; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1591;
1592; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_atomicrmw:
1593; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1594; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1595; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1596; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1597; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1598; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1599; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1600; GFX90A-TGSPLIT-NEXT:    s_endpgm
1601;
1602; GFX940-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_atomicrmw:
1603; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1604; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1605; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1606; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1607; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1608; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1609; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
1610; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1611;
1612; GFX940-TGSPLIT-LABEL: global_singlethread_seq_cst_atomicrmw:
1613; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1614; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1615; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1616; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1617; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1618; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1619; GFX940-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
1620; GFX940-TGSPLIT-NEXT:    s_endpgm
1621;
1622; GFX11-WGP-LABEL: global_singlethread_seq_cst_atomicrmw:
1623; GFX11-WGP:       ; %bb.0: ; %entry
1624; GFX11-WGP-NEXT:    s_clause 0x1
1625; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x8
1626; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1627; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1628; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
1629; GFX11-WGP-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
1630; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1631; GFX11-WGP-NEXT:    s_endpgm
1632;
1633; GFX11-CU-LABEL: global_singlethread_seq_cst_atomicrmw:
1634; GFX11-CU:       ; %bb.0: ; %entry
1635; GFX11-CU-NEXT:    s_clause 0x1
1636; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x8
1637; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1638; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1639; GFX11-CU-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
1640; GFX11-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
1641; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1642; GFX11-CU-NEXT:    s_endpgm
1643    i32 addrspace(1)* %out, i32 %in) {
1644entry:
1645  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") seq_cst
1646  ret void
1647}
1648
1649define amdgpu_kernel void @global_singlethread_acquire_ret_atomicrmw(
1650; GFX6-LABEL: global_singlethread_acquire_ret_atomicrmw:
1651; GFX6:       ; %bb.0: ; %entry
1652; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
1653; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1654; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1655; GFX6-NEXT:    s_mov_b32 s2, -1
1656; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1657; GFX6-NEXT:    v_mov_b32_e32 v0, s6
1658; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
1659; GFX6-NEXT:    s_waitcnt vmcnt(0)
1660; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1661; GFX6-NEXT:    s_endpgm
1662;
1663; GFX7-LABEL: global_singlethread_acquire_ret_atomicrmw:
1664; GFX7:       ; %bb.0: ; %entry
1665; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1666; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1667; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1668; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1669; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1670; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1671; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1672; GFX7-NEXT:    s_waitcnt vmcnt(0)
1673; GFX7-NEXT:    flat_store_dword v[0:1], v2
1674; GFX7-NEXT:    s_endpgm
1675;
1676; GFX10-WGP-LABEL: global_singlethread_acquire_ret_atomicrmw:
1677; GFX10-WGP:       ; %bb.0: ; %entry
1678; GFX10-WGP-NEXT:    s_clause 0x1
1679; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1680; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1681; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
1682; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1683; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1684; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1685; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
1686; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
1687; GFX10-WGP-NEXT:    s_endpgm
1688;
1689; GFX10-CU-LABEL: global_singlethread_acquire_ret_atomicrmw:
1690; GFX10-CU:       ; %bb.0: ; %entry
1691; GFX10-CU-NEXT:    s_clause 0x1
1692; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1693; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1694; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
1695; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1696; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1697; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1698; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
1699; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
1700; GFX10-CU-NEXT:    s_endpgm
1701;
1702; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_ret_atomicrmw:
1703; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1704; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
1705; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1706; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
1707; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
1708; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1709; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
1710; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
1711; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
1712; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1713; SKIP-CACHE-INV-NEXT:    s_endpgm
1714;
1715; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_ret_atomicrmw:
1716; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1717; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1718; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1719; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1720; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1721; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1722; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1723; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1724; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
1725; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1726;
1727; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_ret_atomicrmw:
1728; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1729; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1730; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1731; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1732; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1733; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1734; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1735; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1736; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
1737; GFX90A-TGSPLIT-NEXT:    s_endpgm
1738;
1739; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acquire_ret_atomicrmw:
1740; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1741; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1742; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1743; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1744; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1745; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1746; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[2:3] sc0
1747; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1748; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
1749; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1750;
1751; GFX940-TGSPLIT-LABEL: global_singlethread_acquire_ret_atomicrmw:
1752; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1753; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1754; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1755; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1756; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1757; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1758; GFX940-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[2:3] sc0
1759; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1760; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
1761; GFX940-TGSPLIT-NEXT:    s_endpgm
1762;
1763; GFX11-WGP-LABEL: global_singlethread_acquire_ret_atomicrmw:
1764; GFX11-WGP:       ; %bb.0: ; %entry
1765; GFX11-WGP-NEXT:    s_clause 0x1
1766; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x8
1767; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1768; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1769; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
1770; GFX11-WGP-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
1771; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
1772; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[0:1]
1773; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1774; GFX11-WGP-NEXT:    s_endpgm
1775;
1776; GFX11-CU-LABEL: global_singlethread_acquire_ret_atomicrmw:
1777; GFX11-CU:       ; %bb.0: ; %entry
1778; GFX11-CU-NEXT:    s_clause 0x1
1779; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x8
1780; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1781; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1782; GFX11-CU-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
1783; GFX11-CU-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
1784; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
1785; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
1786; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1787; GFX11-CU-NEXT:    s_endpgm
1788    i32 addrspace(1)* %out, i32 %in) {
1789entry:
1790  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") acquire
1791  store i32 %val, i32 addrspace(1)* %out, align 4
1792  ret void
1793}
1794
1795define amdgpu_kernel void @global_singlethread_acq_rel_ret_atomicrmw(
1796; GFX6-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1797; GFX6:       ; %bb.0: ; %entry
1798; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
1799; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1800; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1801; GFX6-NEXT:    s_mov_b32 s2, -1
1802; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1803; GFX6-NEXT:    v_mov_b32_e32 v0, s6
1804; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
1805; GFX6-NEXT:    s_waitcnt vmcnt(0)
1806; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1807; GFX6-NEXT:    s_endpgm
1808;
1809; GFX7-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1810; GFX7:       ; %bb.0: ; %entry
1811; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1812; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1813; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1814; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1815; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1816; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1817; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1818; GFX7-NEXT:    s_waitcnt vmcnt(0)
1819; GFX7-NEXT:    flat_store_dword v[0:1], v2
1820; GFX7-NEXT:    s_endpgm
1821;
1822; GFX10-WGP-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1823; GFX10-WGP:       ; %bb.0: ; %entry
1824; GFX10-WGP-NEXT:    s_clause 0x1
1825; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1826; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1827; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
1828; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1829; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1830; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1831; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
1832; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
1833; GFX10-WGP-NEXT:    s_endpgm
1834;
1835; GFX10-CU-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1836; GFX10-CU:       ; %bb.0: ; %entry
1837; GFX10-CU-NEXT:    s_clause 0x1
1838; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1839; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1840; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
1841; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1842; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1843; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1844; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
1845; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
1846; GFX10-CU-NEXT:    s_endpgm
1847;
1848; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1849; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1850; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
1851; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1852; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
1853; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
1854; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1855; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
1856; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
1857; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
1858; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1859; SKIP-CACHE-INV-NEXT:    s_endpgm
1860;
1861; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1862; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1863; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1864; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1865; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1866; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1867; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1868; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1869; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1870; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
1871; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1872;
1873; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1874; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1875; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1876; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1877; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1878; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1879; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1880; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1881; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1882; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
1883; GFX90A-TGSPLIT-NEXT:    s_endpgm
1884;
1885; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1886; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1887; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1888; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1889; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1890; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1891; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1892; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[2:3] sc0
1893; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1894; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
1895; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1896;
1897; GFX940-TGSPLIT-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1898; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1899; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1900; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1901; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1902; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1903; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1904; GFX940-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[2:3] sc0
1905; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1906; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
1907; GFX940-TGSPLIT-NEXT:    s_endpgm
1908;
1909; GFX11-WGP-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1910; GFX11-WGP:       ; %bb.0: ; %entry
1911; GFX11-WGP-NEXT:    s_clause 0x1
1912; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x8
1913; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1914; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1915; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
1916; GFX11-WGP-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
1917; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
1918; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[0:1]
1919; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1920; GFX11-WGP-NEXT:    s_endpgm
1921;
1922; GFX11-CU-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1923; GFX11-CU:       ; %bb.0: ; %entry
1924; GFX11-CU-NEXT:    s_clause 0x1
1925; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x8
1926; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1927; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1928; GFX11-CU-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
1929; GFX11-CU-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
1930; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
1931; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
1932; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1933; GFX11-CU-NEXT:    s_endpgm
1934    i32 addrspace(1)* %out, i32 %in) {
1935entry:
1936  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") acq_rel
1937  store i32 %val, i32 addrspace(1)* %out, align 4
1938  ret void
1939}
1940
1941define amdgpu_kernel void @global_singlethread_seq_cst_ret_atomicrmw(
1942; GFX6-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
1943; GFX6:       ; %bb.0: ; %entry
1944; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
1945; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1946; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1947; GFX6-NEXT:    s_mov_b32 s2, -1
1948; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1949; GFX6-NEXT:    v_mov_b32_e32 v0, s6
1950; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
1951; GFX6-NEXT:    s_waitcnt vmcnt(0)
1952; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1953; GFX6-NEXT:    s_endpgm
1954;
1955; GFX7-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
1956; GFX7:       ; %bb.0: ; %entry
1957; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1958; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1959; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1960; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1961; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1962; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1963; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1964; GFX7-NEXT:    s_waitcnt vmcnt(0)
1965; GFX7-NEXT:    flat_store_dword v[0:1], v2
1966; GFX7-NEXT:    s_endpgm
1967;
1968; GFX10-WGP-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
1969; GFX10-WGP:       ; %bb.0: ; %entry
1970; GFX10-WGP-NEXT:    s_clause 0x1
1971; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1972; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1973; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
1974; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1975; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1976; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1977; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
1978; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
1979; GFX10-WGP-NEXT:    s_endpgm
1980;
1981; GFX10-CU-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
1982; GFX10-CU:       ; %bb.0: ; %entry
1983; GFX10-CU-NEXT:    s_clause 0x1
1984; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1985; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1986; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
1987; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1988; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1989; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1990; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
1991; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
1992; GFX10-CU-NEXT:    s_endpgm
1993;
1994; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
1995; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1996; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
1997; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1998; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
1999; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
2000; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2001; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2002; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
2003; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
2004; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
2005; SKIP-CACHE-INV-NEXT:    s_endpgm
2006;
2007; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
2008; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2009; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
2010; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2011; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
2012; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2013; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2014; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
2015; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2016; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
2017; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2018;
2019; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
2020; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2021; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
2022; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2023; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
2024; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2025; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2026; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
2027; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2028; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
2029; GFX90A-TGSPLIT-NEXT:    s_endpgm
2030;
2031; GFX940-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
2032; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2033; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
2034; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2035; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
2036; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2037; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
2038; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[2:3] sc0
2039; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2040; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
2041; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2042;
2043; GFX940-TGSPLIT-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
2044; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2045; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
2046; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2047; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
2048; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2049; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
2050; GFX940-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[2:3] sc0
2051; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2052; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
2053; GFX940-TGSPLIT-NEXT:    s_endpgm
2054;
2055; GFX11-WGP-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
2056; GFX11-WGP:       ; %bb.0: ; %entry
2057; GFX11-WGP-NEXT:    s_clause 0x1
2058; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x8
2059; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
2060; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2061; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
2062; GFX11-WGP-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
2063; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
2064; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[0:1]
2065; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2066; GFX11-WGP-NEXT:    s_endpgm
2067;
2068; GFX11-CU-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
2069; GFX11-CU:       ; %bb.0: ; %entry
2070; GFX11-CU-NEXT:    s_clause 0x1
2071; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x8
2072; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
2073; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2074; GFX11-CU-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
2075; GFX11-CU-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
2076; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
2077; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
2078; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2079; GFX11-CU-NEXT:    s_endpgm
2080    i32 addrspace(1)* %out, i32 %in) {
2081entry:
2082  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") seq_cst
2083  store i32 %val, i32 addrspace(1)* %out, align 4
2084  ret void
2085}
2086
2087define amdgpu_kernel void @global_singlethread_monotonic_monotonic_cmpxchg(
2088; GFX6-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
2089; GFX6:       ; %bb.0: ; %entry
2090; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
2091; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2092; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2093; GFX6-NEXT:    s_mov_b32 s2, -1
2094; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2095; GFX6-NEXT:    v_mov_b32_e32 v0, s6
2096; GFX6-NEXT:    v_mov_b32_e32 v1, s7
2097; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2098; GFX6-NEXT:    s_endpgm
2099;
2100; GFX7-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
2101; GFX7:       ; %bb.0: ; %entry
2102; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2103; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2104; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2105; GFX7-NEXT:    s_add_u32 s0, s0, 16
2106; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2107; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2108; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2109; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2110; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2111; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2112; GFX7-NEXT:    s_endpgm
2113;
2114; GFX10-WGP-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
2115; GFX10-WGP:       ; %bb.0: ; %entry
2116; GFX10-WGP-NEXT:    s_clause 0x1
2117; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2118; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2119; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2120; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2121; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2122; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2123; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2124; GFX10-WGP-NEXT:    s_endpgm
2125;
2126; GFX10-CU-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
2127; GFX10-CU:       ; %bb.0: ; %entry
2128; GFX10-CU-NEXT:    s_clause 0x1
2129; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2130; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2131; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2132; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2133; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2134; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2135; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2136; GFX10-CU-NEXT:    s_endpgm
2137;
2138; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
2139; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2140; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
2141; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
2142; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
2143; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
2144; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2145; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2146; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2147; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2148; SKIP-CACHE-INV-NEXT:    s_endpgm
2149;
2150; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
2151; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2152; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2153; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2154; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2155; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2156; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2157; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2158; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2159;
2160; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
2161; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2162; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2163; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2164; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2165; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2166; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2167; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2168; GFX90A-TGSPLIT-NEXT:    s_endpgm
2169;
2170; GFX940-NOTTGSPLIT-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
2171; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2172; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2173; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2174; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2175; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2176; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2177; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2178; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2179;
2180; GFX940-TGSPLIT-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
2181; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2182; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2183; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2184; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2185; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2186; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2187; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2188; GFX940-TGSPLIT-NEXT:    s_endpgm
2189;
2190; GFX11-WGP-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
2191; GFX11-WGP:       ; %bb.0: ; %entry
2192; GFX11-WGP-NEXT:    s_clause 0x1
2193; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
2194; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
2195; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
2196; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2197; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2198; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
2199; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2200; GFX11-WGP-NEXT:    s_endpgm
2201;
2202; GFX11-CU-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
2203; GFX11-CU:       ; %bb.0: ; %entry
2204; GFX11-CU-NEXT:    s_clause 0x1
2205; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
2206; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
2207; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
2208; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2209; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2210; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
2211; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2212; GFX11-CU-NEXT:    s_endpgm
2213    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2214entry:
2215  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2216  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic monotonic
2217  ret void
2218}
2219
2220define amdgpu_kernel void @global_singlethread_acquire_monotonic_cmpxchg(
2221; GFX6-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
2222; GFX6:       ; %bb.0: ; %entry
2223; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
2224; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2225; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2226; GFX6-NEXT:    s_mov_b32 s2, -1
2227; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2228; GFX6-NEXT:    v_mov_b32_e32 v0, s6
2229; GFX6-NEXT:    v_mov_b32_e32 v1, s7
2230; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2231; GFX6-NEXT:    s_endpgm
2232;
2233; GFX7-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
2234; GFX7:       ; %bb.0: ; %entry
2235; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2236; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2237; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2238; GFX7-NEXT:    s_add_u32 s0, s0, 16
2239; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2240; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2241; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2242; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2243; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2244; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2245; GFX7-NEXT:    s_endpgm
2246;
2247; GFX10-WGP-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
2248; GFX10-WGP:       ; %bb.0: ; %entry
2249; GFX10-WGP-NEXT:    s_clause 0x1
2250; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2251; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2252; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2253; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2254; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2255; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2256; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2257; GFX10-WGP-NEXT:    s_endpgm
2258;
2259; GFX10-CU-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
2260; GFX10-CU:       ; %bb.0: ; %entry
2261; GFX10-CU-NEXT:    s_clause 0x1
2262; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2263; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2264; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2265; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2266; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2267; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2268; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2269; GFX10-CU-NEXT:    s_endpgm
2270;
2271; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
2272; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2273; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
2274; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
2275; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
2276; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
2277; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2278; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2279; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2280; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2281; SKIP-CACHE-INV-NEXT:    s_endpgm
2282;
2283; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
2284; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2285; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2286; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2287; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2288; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2289; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2290; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2291; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2292;
2293; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
2294; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2295; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2296; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2297; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2298; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2299; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2300; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2301; GFX90A-TGSPLIT-NEXT:    s_endpgm
2302;
2303; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
2304; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2305; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2306; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2307; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2308; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2309; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2310; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2311; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2312;
2313; GFX940-TGSPLIT-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
2314; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2315; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2316; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2317; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2318; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2319; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2320; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2321; GFX940-TGSPLIT-NEXT:    s_endpgm
2322;
2323; GFX11-WGP-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
2324; GFX11-WGP:       ; %bb.0: ; %entry
2325; GFX11-WGP-NEXT:    s_clause 0x1
2326; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
2327; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
2328; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
2329; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2330; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2331; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
2332; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2333; GFX11-WGP-NEXT:    s_endpgm
2334;
2335; GFX11-CU-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
2336; GFX11-CU:       ; %bb.0: ; %entry
2337; GFX11-CU-NEXT:    s_clause 0x1
2338; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
2339; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
2340; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
2341; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2342; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2343; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
2344; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2345; GFX11-CU-NEXT:    s_endpgm
2346    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2347entry:
2348  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2349  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire monotonic
2350  ret void
2351}
2352
2353define amdgpu_kernel void @global_singlethread_release_monotonic_cmpxchg(
2354; GFX6-LABEL: global_singlethread_release_monotonic_cmpxchg:
2355; GFX6:       ; %bb.0: ; %entry
2356; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
2357; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2358; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2359; GFX6-NEXT:    s_mov_b32 s2, -1
2360; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2361; GFX6-NEXT:    v_mov_b32_e32 v0, s6
2362; GFX6-NEXT:    v_mov_b32_e32 v1, s7
2363; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2364; GFX6-NEXT:    s_endpgm
2365;
2366; GFX7-LABEL: global_singlethread_release_monotonic_cmpxchg:
2367; GFX7:       ; %bb.0: ; %entry
2368; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2369; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2370; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2371; GFX7-NEXT:    s_add_u32 s0, s0, 16
2372; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2373; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2374; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2375; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2376; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2377; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2378; GFX7-NEXT:    s_endpgm
2379;
2380; GFX10-WGP-LABEL: global_singlethread_release_monotonic_cmpxchg:
2381; GFX10-WGP:       ; %bb.0: ; %entry
2382; GFX10-WGP-NEXT:    s_clause 0x1
2383; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2384; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2385; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2386; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2387; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2388; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2389; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2390; GFX10-WGP-NEXT:    s_endpgm
2391;
2392; GFX10-CU-LABEL: global_singlethread_release_monotonic_cmpxchg:
2393; GFX10-CU:       ; %bb.0: ; %entry
2394; GFX10-CU-NEXT:    s_clause 0x1
2395; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2396; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2397; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2398; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2399; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2400; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2401; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2402; GFX10-CU-NEXT:    s_endpgm
2403;
2404; SKIP-CACHE-INV-LABEL: global_singlethread_release_monotonic_cmpxchg:
2405; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2406; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
2407; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
2408; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
2409; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
2410; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2411; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2412; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2413; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2414; SKIP-CACHE-INV-NEXT:    s_endpgm
2415;
2416; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_release_monotonic_cmpxchg:
2417; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2418; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2419; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2420; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2421; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2422; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2423; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2424; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2425;
2426; GFX90A-TGSPLIT-LABEL: global_singlethread_release_monotonic_cmpxchg:
2427; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2428; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2429; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2430; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2431; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2432; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2433; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2434; GFX90A-TGSPLIT-NEXT:    s_endpgm
2435;
2436; GFX940-NOTTGSPLIT-LABEL: global_singlethread_release_monotonic_cmpxchg:
2437; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2438; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2439; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2440; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2441; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2442; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2443; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2444; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2445;
2446; GFX940-TGSPLIT-LABEL: global_singlethread_release_monotonic_cmpxchg:
2447; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2448; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2449; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2450; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2451; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2452; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2453; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2454; GFX940-TGSPLIT-NEXT:    s_endpgm
2455;
2456; GFX11-WGP-LABEL: global_singlethread_release_monotonic_cmpxchg:
2457; GFX11-WGP:       ; %bb.0: ; %entry
2458; GFX11-WGP-NEXT:    s_clause 0x1
2459; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
2460; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
2461; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
2462; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2463; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2464; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
2465; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2466; GFX11-WGP-NEXT:    s_endpgm
2467;
2468; GFX11-CU-LABEL: global_singlethread_release_monotonic_cmpxchg:
2469; GFX11-CU:       ; %bb.0: ; %entry
2470; GFX11-CU-NEXT:    s_clause 0x1
2471; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
2472; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
2473; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
2474; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2475; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2476; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
2477; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2478; GFX11-CU-NEXT:    s_endpgm
2479    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2480entry:
2481  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2482  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") release monotonic
2483  ret void
2484}
2485
2486define amdgpu_kernel void @global_singlethread_acq_rel_monotonic_cmpxchg(
2487; GFX6-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
2488; GFX6:       ; %bb.0: ; %entry
2489; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
2490; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2491; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2492; GFX6-NEXT:    s_mov_b32 s2, -1
2493; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2494; GFX6-NEXT:    v_mov_b32_e32 v0, s6
2495; GFX6-NEXT:    v_mov_b32_e32 v1, s7
2496; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2497; GFX6-NEXT:    s_endpgm
2498;
2499; GFX7-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
2500; GFX7:       ; %bb.0: ; %entry
2501; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2502; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2503; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2504; GFX7-NEXT:    s_add_u32 s0, s0, 16
2505; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2506; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2507; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2508; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2509; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2510; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2511; GFX7-NEXT:    s_endpgm
2512;
2513; GFX10-WGP-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
2514; GFX10-WGP:       ; %bb.0: ; %entry
2515; GFX10-WGP-NEXT:    s_clause 0x1
2516; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2517; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2518; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2519; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2520; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2521; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2522; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2523; GFX10-WGP-NEXT:    s_endpgm
2524;
2525; GFX10-CU-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
2526; GFX10-CU:       ; %bb.0: ; %entry
2527; GFX10-CU-NEXT:    s_clause 0x1
2528; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2529; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2530; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2531; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2532; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2533; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2534; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2535; GFX10-CU-NEXT:    s_endpgm
2536;
2537; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
2538; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2539; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
2540; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
2541; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
2542; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
2543; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2544; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2545; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2546; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2547; SKIP-CACHE-INV-NEXT:    s_endpgm
2548;
2549; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
2550; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2551; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2552; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2553; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2554; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2555; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2556; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2557; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2558;
2559; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
2560; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2561; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2562; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2563; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2564; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2565; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2566; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2567; GFX90A-TGSPLIT-NEXT:    s_endpgm
2568;
2569; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
2570; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2571; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2572; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2573; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2574; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2575; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2576; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2577; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2578;
2579; GFX940-TGSPLIT-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
2580; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2581; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2582; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2583; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2584; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2585; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2586; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2587; GFX940-TGSPLIT-NEXT:    s_endpgm
2588;
2589; GFX11-WGP-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
2590; GFX11-WGP:       ; %bb.0: ; %entry
2591; GFX11-WGP-NEXT:    s_clause 0x1
2592; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
2593; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
2594; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
2595; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2596; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2597; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
2598; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2599; GFX11-WGP-NEXT:    s_endpgm
2600;
2601; GFX11-CU-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
2602; GFX11-CU:       ; %bb.0: ; %entry
2603; GFX11-CU-NEXT:    s_clause 0x1
2604; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
2605; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
2606; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
2607; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2608; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2609; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
2610; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2611; GFX11-CU-NEXT:    s_endpgm
2612    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2613entry:
2614  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2615  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel monotonic
2616  ret void
2617}
2618
2619define amdgpu_kernel void @global_singlethread_seq_cst_monotonic_cmpxchg(
2620; GFX6-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
2621; GFX6:       ; %bb.0: ; %entry
2622; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
2623; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2624; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2625; GFX6-NEXT:    s_mov_b32 s2, -1
2626; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2627; GFX6-NEXT:    v_mov_b32_e32 v0, s6
2628; GFX6-NEXT:    v_mov_b32_e32 v1, s7
2629; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2630; GFX6-NEXT:    s_endpgm
2631;
2632; GFX7-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
2633; GFX7:       ; %bb.0: ; %entry
2634; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2635; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2636; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2637; GFX7-NEXT:    s_add_u32 s0, s0, 16
2638; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2639; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2640; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2641; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2642; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2643; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2644; GFX7-NEXT:    s_endpgm
2645;
2646; GFX10-WGP-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
2647; GFX10-WGP:       ; %bb.0: ; %entry
2648; GFX10-WGP-NEXT:    s_clause 0x1
2649; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2650; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2651; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2652; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2653; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2654; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2655; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2656; GFX10-WGP-NEXT:    s_endpgm
2657;
2658; GFX10-CU-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
2659; GFX10-CU:       ; %bb.0: ; %entry
2660; GFX10-CU-NEXT:    s_clause 0x1
2661; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2662; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2663; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2664; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2665; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2666; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2667; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2668; GFX10-CU-NEXT:    s_endpgm
2669;
2670; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
2671; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2672; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
2673; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
2674; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
2675; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
2676; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2677; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2678; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2679; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2680; SKIP-CACHE-INV-NEXT:    s_endpgm
2681;
2682; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
2683; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2684; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2685; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2686; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2687; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2688; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2689; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2690; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2691;
2692; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
2693; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2694; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2695; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2696; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2697; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2698; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2699; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2700; GFX90A-TGSPLIT-NEXT:    s_endpgm
2701;
2702; GFX940-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
2703; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2704; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2705; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2706; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2707; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2708; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2709; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2710; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2711;
2712; GFX940-TGSPLIT-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
2713; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2714; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2715; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2716; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2717; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2718; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2719; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2720; GFX940-TGSPLIT-NEXT:    s_endpgm
2721;
2722; GFX11-WGP-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
2723; GFX11-WGP:       ; %bb.0: ; %entry
2724; GFX11-WGP-NEXT:    s_clause 0x1
2725; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
2726; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
2727; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
2728; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2729; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2730; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
2731; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2732; GFX11-WGP-NEXT:    s_endpgm
2733;
2734; GFX11-CU-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
2735; GFX11-CU:       ; %bb.0: ; %entry
2736; GFX11-CU-NEXT:    s_clause 0x1
2737; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
2738; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
2739; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
2740; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2741; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2742; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
2743; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2744; GFX11-CU-NEXT:    s_endpgm
2745    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2746entry:
2747  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2748  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst monotonic
2749  ret void
2750}
2751
2752define amdgpu_kernel void @global_singlethread_monotonic_acquire_cmpxchg(
2753; GFX6-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
2754; GFX6:       ; %bb.0: ; %entry
2755; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
2756; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2757; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2758; GFX6-NEXT:    s_mov_b32 s2, -1
2759; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2760; GFX6-NEXT:    v_mov_b32_e32 v0, s6
2761; GFX6-NEXT:    v_mov_b32_e32 v1, s7
2762; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2763; GFX6-NEXT:    s_endpgm
2764;
2765; GFX7-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
2766; GFX7:       ; %bb.0: ; %entry
2767; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2768; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2769; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2770; GFX7-NEXT:    s_add_u32 s0, s0, 16
2771; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2772; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2773; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2774; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2775; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2776; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2777; GFX7-NEXT:    s_endpgm
2778;
2779; GFX10-WGP-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
2780; GFX10-WGP:       ; %bb.0: ; %entry
2781; GFX10-WGP-NEXT:    s_clause 0x1
2782; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2783; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2784; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2785; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2786; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2787; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2788; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2789; GFX10-WGP-NEXT:    s_endpgm
2790;
2791; GFX10-CU-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
2792; GFX10-CU:       ; %bb.0: ; %entry
2793; GFX10-CU-NEXT:    s_clause 0x1
2794; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2795; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2796; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2797; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2798; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2799; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2800; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2801; GFX10-CU-NEXT:    s_endpgm
2802;
2803; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
2804; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2805; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
2806; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
2807; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
2808; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
2809; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2810; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2811; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2812; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2813; SKIP-CACHE-INV-NEXT:    s_endpgm
2814;
2815; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
2816; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2817; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2818; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2819; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2820; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2821; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2822; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2823; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2824;
2825; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
2826; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2827; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2828; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2829; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2830; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2831; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2832; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2833; GFX90A-TGSPLIT-NEXT:    s_endpgm
2834;
2835; GFX940-NOTTGSPLIT-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
2836; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2837; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2838; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2839; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2840; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2841; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2842; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2843; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2844;
2845; GFX940-TGSPLIT-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
2846; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2847; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2848; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2849; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2850; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2851; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2852; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2853; GFX940-TGSPLIT-NEXT:    s_endpgm
2854;
2855; GFX11-WGP-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
2856; GFX11-WGP:       ; %bb.0: ; %entry
2857; GFX11-WGP-NEXT:    s_clause 0x1
2858; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
2859; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
2860; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
2861; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2862; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2863; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
2864; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2865; GFX11-WGP-NEXT:    s_endpgm
2866;
2867; GFX11-CU-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
2868; GFX11-CU:       ; %bb.0: ; %entry
2869; GFX11-CU-NEXT:    s_clause 0x1
2870; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
2871; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
2872; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
2873; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2874; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2875; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
2876; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2877; GFX11-CU-NEXT:    s_endpgm
2878    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2879entry:
2880  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2881  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic acquire
2882  ret void
2883}
2884
2885define amdgpu_kernel void @global_singlethread_acquire_acquire_cmpxchg(
2886; GFX6-LABEL: global_singlethread_acquire_acquire_cmpxchg:
2887; GFX6:       ; %bb.0: ; %entry
2888; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
2889; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2890; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2891; GFX6-NEXT:    s_mov_b32 s2, -1
2892; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2893; GFX6-NEXT:    v_mov_b32_e32 v0, s6
2894; GFX6-NEXT:    v_mov_b32_e32 v1, s7
2895; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2896; GFX6-NEXT:    s_endpgm
2897;
2898; GFX7-LABEL: global_singlethread_acquire_acquire_cmpxchg:
2899; GFX7:       ; %bb.0: ; %entry
2900; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2901; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2902; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2903; GFX7-NEXT:    s_add_u32 s0, s0, 16
2904; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2905; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2906; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2907; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2908; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2909; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2910; GFX7-NEXT:    s_endpgm
2911;
2912; GFX10-WGP-LABEL: global_singlethread_acquire_acquire_cmpxchg:
2913; GFX10-WGP:       ; %bb.0: ; %entry
2914; GFX10-WGP-NEXT:    s_clause 0x1
2915; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2916; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2917; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2918; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2919; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2920; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2921; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2922; GFX10-WGP-NEXT:    s_endpgm
2923;
2924; GFX10-CU-LABEL: global_singlethread_acquire_acquire_cmpxchg:
2925; GFX10-CU:       ; %bb.0: ; %entry
2926; GFX10-CU-NEXT:    s_clause 0x1
2927; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2928; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2929; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2930; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2931; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2932; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2933; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2934; GFX10-CU-NEXT:    s_endpgm
2935;
2936; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_acquire_cmpxchg:
2937; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2938; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
2939; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
2940; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
2941; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
2942; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2943; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2944; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2945; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2946; SKIP-CACHE-INV-NEXT:    s_endpgm
2947;
2948; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_acquire_cmpxchg:
2949; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2950; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2951; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2952; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2953; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2954; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2955; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2956; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2957;
2958; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_acquire_cmpxchg:
2959; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2960; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2961; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2962; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2963; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2964; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2965; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2966; GFX90A-TGSPLIT-NEXT:    s_endpgm
2967;
2968; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acquire_acquire_cmpxchg:
2969; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2970; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2971; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2972; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2973; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2974; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2975; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2976; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2977;
2978; GFX940-TGSPLIT-LABEL: global_singlethread_acquire_acquire_cmpxchg:
2979; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2980; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2981; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2982; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2983; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2984; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2985; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2986; GFX940-TGSPLIT-NEXT:    s_endpgm
2987;
2988; GFX11-WGP-LABEL: global_singlethread_acquire_acquire_cmpxchg:
2989; GFX11-WGP:       ; %bb.0: ; %entry
2990; GFX11-WGP-NEXT:    s_clause 0x1
2991; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
2992; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
2993; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
2994; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2995; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2996; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
2997; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2998; GFX11-WGP-NEXT:    s_endpgm
2999;
3000; GFX11-CU-LABEL: global_singlethread_acquire_acquire_cmpxchg:
3001; GFX11-CU:       ; %bb.0: ; %entry
3002; GFX11-CU-NEXT:    s_clause 0x1
3003; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
3004; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
3005; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
3006; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3007; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3008; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
3009; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3010; GFX11-CU-NEXT:    s_endpgm
3011    i32 addrspace(1)* %out, i32 %in, i32 %old) {
3012entry:
3013  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3014  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire acquire
3015  ret void
3016}
3017
3018define amdgpu_kernel void @global_singlethread_release_acquire_cmpxchg(
3019; GFX6-LABEL: global_singlethread_release_acquire_cmpxchg:
3020; GFX6:       ; %bb.0: ; %entry
3021; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
3022; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3023; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3024; GFX6-NEXT:    s_mov_b32 s2, -1
3025; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3026; GFX6-NEXT:    v_mov_b32_e32 v0, s6
3027; GFX6-NEXT:    v_mov_b32_e32 v1, s7
3028; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3029; GFX6-NEXT:    s_endpgm
3030;
3031; GFX7-LABEL: global_singlethread_release_acquire_cmpxchg:
3032; GFX7:       ; %bb.0: ; %entry
3033; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3034; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3035; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3036; GFX7-NEXT:    s_add_u32 s0, s0, 16
3037; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3038; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3039; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3040; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3041; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3042; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3043; GFX7-NEXT:    s_endpgm
3044;
3045; GFX10-WGP-LABEL: global_singlethread_release_acquire_cmpxchg:
3046; GFX10-WGP:       ; %bb.0: ; %entry
3047; GFX10-WGP-NEXT:    s_clause 0x1
3048; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3049; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3050; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3051; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3052; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3053; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3054; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3055; GFX10-WGP-NEXT:    s_endpgm
3056;
3057; GFX10-CU-LABEL: global_singlethread_release_acquire_cmpxchg:
3058; GFX10-CU:       ; %bb.0: ; %entry
3059; GFX10-CU-NEXT:    s_clause 0x1
3060; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3061; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3062; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3063; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3064; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3065; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3066; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3067; GFX10-CU-NEXT:    s_endpgm
3068;
3069; SKIP-CACHE-INV-LABEL: global_singlethread_release_acquire_cmpxchg:
3070; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3071; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
3072; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
3073; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3074; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3075; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3076; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3077; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3078; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3079; SKIP-CACHE-INV-NEXT:    s_endpgm
3080;
3081; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_release_acquire_cmpxchg:
3082; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3083; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3084; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3085; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3086; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3087; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3088; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3089; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3090;
3091; GFX90A-TGSPLIT-LABEL: global_singlethread_release_acquire_cmpxchg:
3092; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3093; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3094; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3095; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3096; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3097; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3098; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3099; GFX90A-TGSPLIT-NEXT:    s_endpgm
3100;
3101; GFX940-NOTTGSPLIT-LABEL: global_singlethread_release_acquire_cmpxchg:
3102; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3103; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3104; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3105; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3106; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3107; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3108; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
3109; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3110;
3111; GFX940-TGSPLIT-LABEL: global_singlethread_release_acquire_cmpxchg:
3112; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3113; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3114; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3115; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3116; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3117; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3118; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
3119; GFX940-TGSPLIT-NEXT:    s_endpgm
3120;
3121; GFX11-WGP-LABEL: global_singlethread_release_acquire_cmpxchg:
3122; GFX11-WGP:       ; %bb.0: ; %entry
3123; GFX11-WGP-NEXT:    s_clause 0x1
3124; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
3125; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
3126; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
3127; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3128; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3129; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
3130; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3131; GFX11-WGP-NEXT:    s_endpgm
3132;
3133; GFX11-CU-LABEL: global_singlethread_release_acquire_cmpxchg:
3134; GFX11-CU:       ; %bb.0: ; %entry
3135; GFX11-CU-NEXT:    s_clause 0x1
3136; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
3137; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
3138; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
3139; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3140; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3141; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
3142; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3143; GFX11-CU-NEXT:    s_endpgm
3144    i32 addrspace(1)* %out, i32 %in, i32 %old) {
3145entry:
3146  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3147  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") release acquire
3148  ret void
3149}
3150
3151define amdgpu_kernel void @global_singlethread_acq_rel_acquire_cmpxchg(
3152; GFX6-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
3153; GFX6:       ; %bb.0: ; %entry
3154; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
3155; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3156; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3157; GFX6-NEXT:    s_mov_b32 s2, -1
3158; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3159; GFX6-NEXT:    v_mov_b32_e32 v0, s6
3160; GFX6-NEXT:    v_mov_b32_e32 v1, s7
3161; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3162; GFX6-NEXT:    s_endpgm
3163;
3164; GFX7-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
3165; GFX7:       ; %bb.0: ; %entry
3166; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3167; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3168; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3169; GFX7-NEXT:    s_add_u32 s0, s0, 16
3170; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3171; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3172; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3173; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3174; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3175; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3176; GFX7-NEXT:    s_endpgm
3177;
3178; GFX10-WGP-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
3179; GFX10-WGP:       ; %bb.0: ; %entry
3180; GFX10-WGP-NEXT:    s_clause 0x1
3181; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3182; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3183; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3184; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3185; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3186; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3187; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3188; GFX10-WGP-NEXT:    s_endpgm
3189;
3190; GFX10-CU-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
3191; GFX10-CU:       ; %bb.0: ; %entry
3192; GFX10-CU-NEXT:    s_clause 0x1
3193; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3194; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3195; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3196; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3197; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3198; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3199; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3200; GFX10-CU-NEXT:    s_endpgm
3201;
3202; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
3203; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3204; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
3205; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
3206; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3207; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3208; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3209; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3210; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3211; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3212; SKIP-CACHE-INV-NEXT:    s_endpgm
3213;
3214; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
3215; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3216; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3217; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3218; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3219; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3220; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3221; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3222; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3223;
3224; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
3225; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3226; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3227; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3228; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3229; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3230; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3231; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3232; GFX90A-TGSPLIT-NEXT:    s_endpgm
3233;
3234; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
3235; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3236; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3237; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3238; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3239; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3240; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3241; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
3242; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3243;
3244; GFX940-TGSPLIT-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
3245; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3246; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3247; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3248; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3249; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3250; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3251; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
3252; GFX940-TGSPLIT-NEXT:    s_endpgm
3253;
3254; GFX11-WGP-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
3255; GFX11-WGP:       ; %bb.0: ; %entry
3256; GFX11-WGP-NEXT:    s_clause 0x1
3257; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
3258; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
3259; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
3260; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3261; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3262; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
3263; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3264; GFX11-WGP-NEXT:    s_endpgm
3265;
3266; GFX11-CU-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
3267; GFX11-CU:       ; %bb.0: ; %entry
3268; GFX11-CU-NEXT:    s_clause 0x1
3269; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
3270; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
3271; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
3272; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3273; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3274; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
3275; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3276; GFX11-CU-NEXT:    s_endpgm
3277    i32 addrspace(1)* %out, i32 %in, i32 %old) {
3278entry:
3279  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3280  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel acquire
3281  ret void
3282}
3283
3284define amdgpu_kernel void @global_singlethread_seq_cst_acquire_cmpxchg(
3285; GFX6-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
3286; GFX6:       ; %bb.0: ; %entry
3287; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
3288; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3289; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3290; GFX6-NEXT:    s_mov_b32 s2, -1
3291; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3292; GFX6-NEXT:    v_mov_b32_e32 v0, s6
3293; GFX6-NEXT:    v_mov_b32_e32 v1, s7
3294; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3295; GFX6-NEXT:    s_endpgm
3296;
3297; GFX7-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
3298; GFX7:       ; %bb.0: ; %entry
3299; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3300; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3301; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3302; GFX7-NEXT:    s_add_u32 s0, s0, 16
3303; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3304; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3305; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3306; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3307; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3308; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3309; GFX7-NEXT:    s_endpgm
3310;
3311; GFX10-WGP-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
3312; GFX10-WGP:       ; %bb.0: ; %entry
3313; GFX10-WGP-NEXT:    s_clause 0x1
3314; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3315; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3316; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3317; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3318; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3319; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3320; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3321; GFX10-WGP-NEXT:    s_endpgm
3322;
3323; GFX10-CU-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
3324; GFX10-CU:       ; %bb.0: ; %entry
3325; GFX10-CU-NEXT:    s_clause 0x1
3326; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3327; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3328; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3329; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3330; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3331; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3332; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3333; GFX10-CU-NEXT:    s_endpgm
3334;
3335; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
3336; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3337; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
3338; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
3339; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3340; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3341; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3342; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3343; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3344; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3345; SKIP-CACHE-INV-NEXT:    s_endpgm
3346;
3347; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
3348; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3349; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3350; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3351; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3352; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3353; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3354; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3355; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3356;
3357; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
3358; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3359; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3360; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3361; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3362; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3363; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3364; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3365; GFX90A-TGSPLIT-NEXT:    s_endpgm
3366;
3367; GFX940-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
3368; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3369; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3370; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3371; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3372; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3373; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3374; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
3375; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3376;
3377; GFX940-TGSPLIT-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
3378; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3379; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3380; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3381; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3382; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3383; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3384; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
3385; GFX940-TGSPLIT-NEXT:    s_endpgm
3386;
3387; GFX11-WGP-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
3388; GFX11-WGP:       ; %bb.0: ; %entry
3389; GFX11-WGP-NEXT:    s_clause 0x1
3390; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
3391; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
3392; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
3393; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3394; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3395; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
3396; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3397; GFX11-WGP-NEXT:    s_endpgm
3398;
3399; GFX11-CU-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
3400; GFX11-CU:       ; %bb.0: ; %entry
3401; GFX11-CU-NEXT:    s_clause 0x1
3402; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
3403; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
3404; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
3405; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3406; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3407; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
3408; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3409; GFX11-CU-NEXT:    s_endpgm
3410    i32 addrspace(1)* %out, i32 %in, i32 %old) {
3411entry:
3412  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3413  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst acquire
3414  ret void
3415}
3416
3417define amdgpu_kernel void @global_singlethread_monotonic_seq_cst_cmpxchg(
3418; GFX6-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
3419; GFX6:       ; %bb.0: ; %entry
3420; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
3421; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3422; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3423; GFX6-NEXT:    s_mov_b32 s2, -1
3424; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3425; GFX6-NEXT:    v_mov_b32_e32 v0, s6
3426; GFX6-NEXT:    v_mov_b32_e32 v1, s7
3427; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3428; GFX6-NEXT:    s_endpgm
3429;
3430; GFX7-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
3431; GFX7:       ; %bb.0: ; %entry
3432; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3433; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3434; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3435; GFX7-NEXT:    s_add_u32 s0, s0, 16
3436; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3437; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3438; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3439; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3440; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3441; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3442; GFX7-NEXT:    s_endpgm
3443;
3444; GFX10-WGP-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
3445; GFX10-WGP:       ; %bb.0: ; %entry
3446; GFX10-WGP-NEXT:    s_clause 0x1
3447; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3448; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3449; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3450; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3451; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3452; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3453; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3454; GFX10-WGP-NEXT:    s_endpgm
3455;
3456; GFX10-CU-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
3457; GFX10-CU:       ; %bb.0: ; %entry
3458; GFX10-CU-NEXT:    s_clause 0x1
3459; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3460; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3461; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3462; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3463; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3464; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3465; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3466; GFX10-CU-NEXT:    s_endpgm
3467;
3468; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
3469; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3470; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
3471; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
3472; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3473; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3474; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3475; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3476; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3477; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3478; SKIP-CACHE-INV-NEXT:    s_endpgm
3479;
3480; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
3481; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3482; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3483; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3484; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3485; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3486; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3487; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3488; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3489;
3490; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
3491; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3492; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3493; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3494; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3495; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3496; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3497; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3498; GFX90A-TGSPLIT-NEXT:    s_endpgm
3499;
3500; GFX940-NOTTGSPLIT-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
3501; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3502; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3503; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3504; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3505; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3506; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3507; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
3508; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3509;
3510; GFX940-TGSPLIT-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
3511; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3512; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3513; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3514; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3515; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3516; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3517; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
3518; GFX940-TGSPLIT-NEXT:    s_endpgm
3519;
3520; GFX11-WGP-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
3521; GFX11-WGP:       ; %bb.0: ; %entry
3522; GFX11-WGP-NEXT:    s_clause 0x1
3523; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
3524; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
3525; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
3526; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3527; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3528; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
3529; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3530; GFX11-WGP-NEXT:    s_endpgm
3531;
3532; GFX11-CU-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
3533; GFX11-CU:       ; %bb.0: ; %entry
3534; GFX11-CU-NEXT:    s_clause 0x1
3535; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
3536; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
3537; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
3538; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3539; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3540; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
3541; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3542; GFX11-CU-NEXT:    s_endpgm
3543    i32 addrspace(1)* %out, i32 %in, i32 %old) {
3544entry:
3545  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3546  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic seq_cst
3547  ret void
3548}
3549
3550define amdgpu_kernel void @global_singlethread_acquire_seq_cst_cmpxchg(
3551; GFX6-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
3552; GFX6:       ; %bb.0: ; %entry
3553; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
3554; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3555; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3556; GFX6-NEXT:    s_mov_b32 s2, -1
3557; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3558; GFX6-NEXT:    v_mov_b32_e32 v0, s6
3559; GFX6-NEXT:    v_mov_b32_e32 v1, s7
3560; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3561; GFX6-NEXT:    s_endpgm
3562;
3563; GFX7-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
3564; GFX7:       ; %bb.0: ; %entry
3565; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3566; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3567; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3568; GFX7-NEXT:    s_add_u32 s0, s0, 16
3569; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3570; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3571; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3572; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3573; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3574; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3575; GFX7-NEXT:    s_endpgm
3576;
3577; GFX10-WGP-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
3578; GFX10-WGP:       ; %bb.0: ; %entry
3579; GFX10-WGP-NEXT:    s_clause 0x1
3580; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3581; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3582; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3583; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3584; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3585; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3586; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3587; GFX10-WGP-NEXT:    s_endpgm
3588;
3589; GFX10-CU-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
3590; GFX10-CU:       ; %bb.0: ; %entry
3591; GFX10-CU-NEXT:    s_clause 0x1
3592; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3593; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3594; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3595; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3596; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3597; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3598; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3599; GFX10-CU-NEXT:    s_endpgm
3600;
3601; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
3602; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3603; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
3604; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
3605; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3606; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3607; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3608; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3609; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3610; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3611; SKIP-CACHE-INV-NEXT:    s_endpgm
3612;
3613; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
3614; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3615; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3616; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3617; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3618; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3619; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3620; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3621; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3622;
3623; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
3624; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3625; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3626; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3627; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3628; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3629; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3630; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3631; GFX90A-TGSPLIT-NEXT:    s_endpgm
3632;
3633; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
3634; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3635; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3636; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3637; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3638; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3639; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3640; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
3641; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3642;
3643; GFX940-TGSPLIT-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
3644; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3645; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3646; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3647; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3648; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3649; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3650; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
3651; GFX940-TGSPLIT-NEXT:    s_endpgm
3652;
3653; GFX11-WGP-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
3654; GFX11-WGP:       ; %bb.0: ; %entry
3655; GFX11-WGP-NEXT:    s_clause 0x1
3656; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
3657; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
3658; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
3659; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3660; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3661; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
3662; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3663; GFX11-WGP-NEXT:    s_endpgm
3664;
3665; GFX11-CU-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
3666; GFX11-CU:       ; %bb.0: ; %entry
3667; GFX11-CU-NEXT:    s_clause 0x1
3668; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
3669; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
3670; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
3671; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3672; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3673; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
3674; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3675; GFX11-CU-NEXT:    s_endpgm
3676    i32 addrspace(1)* %out, i32 %in, i32 %old) {
3677entry:
3678  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3679  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire seq_cst
3680  ret void
3681}
3682
3683define amdgpu_kernel void @global_singlethread_release_seq_cst_cmpxchg(
3684; GFX6-LABEL: global_singlethread_release_seq_cst_cmpxchg:
3685; GFX6:       ; %bb.0: ; %entry
3686; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
3687; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3688; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3689; GFX6-NEXT:    s_mov_b32 s2, -1
3690; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3691; GFX6-NEXT:    v_mov_b32_e32 v0, s6
3692; GFX6-NEXT:    v_mov_b32_e32 v1, s7
3693; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3694; GFX6-NEXT:    s_endpgm
3695;
3696; GFX7-LABEL: global_singlethread_release_seq_cst_cmpxchg:
3697; GFX7:       ; %bb.0: ; %entry
3698; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3699; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3700; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3701; GFX7-NEXT:    s_add_u32 s0, s0, 16
3702; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3703; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3704; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3705; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3706; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3707; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3708; GFX7-NEXT:    s_endpgm
3709;
3710; GFX10-WGP-LABEL: global_singlethread_release_seq_cst_cmpxchg:
3711; GFX10-WGP:       ; %bb.0: ; %entry
3712; GFX10-WGP-NEXT:    s_clause 0x1
3713; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3714; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3715; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3716; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3717; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3718; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3719; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3720; GFX10-WGP-NEXT:    s_endpgm
3721;
3722; GFX10-CU-LABEL: global_singlethread_release_seq_cst_cmpxchg:
3723; GFX10-CU:       ; %bb.0: ; %entry
3724; GFX10-CU-NEXT:    s_clause 0x1
3725; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3726; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3727; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3728; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3729; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3730; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3731; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3732; GFX10-CU-NEXT:    s_endpgm
3733;
3734; SKIP-CACHE-INV-LABEL: global_singlethread_release_seq_cst_cmpxchg:
3735; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3736; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
3737; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
3738; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3739; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3740; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3741; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3742; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3743; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3744; SKIP-CACHE-INV-NEXT:    s_endpgm
3745;
3746; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_release_seq_cst_cmpxchg:
3747; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3748; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3749; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3750; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3751; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3752; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3753; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3754; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3755;
3756; GFX90A-TGSPLIT-LABEL: global_singlethread_release_seq_cst_cmpxchg:
3757; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3758; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3759; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3760; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3761; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3762; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3763; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3764; GFX90A-TGSPLIT-NEXT:    s_endpgm
3765;
3766; GFX940-NOTTGSPLIT-LABEL: global_singlethread_release_seq_cst_cmpxchg:
3767; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3768; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3769; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3770; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3771; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3772; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3773; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
3774; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3775;
3776; GFX940-TGSPLIT-LABEL: global_singlethread_release_seq_cst_cmpxchg:
3777; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3778; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3779; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3780; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3781; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3782; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3783; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
3784; GFX940-TGSPLIT-NEXT:    s_endpgm
3785;
3786; GFX11-WGP-LABEL: global_singlethread_release_seq_cst_cmpxchg:
3787; GFX11-WGP:       ; %bb.0: ; %entry
3788; GFX11-WGP-NEXT:    s_clause 0x1
3789; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
3790; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
3791; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
3792; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3793; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3794; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
3795; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3796; GFX11-WGP-NEXT:    s_endpgm
3797;
3798; GFX11-CU-LABEL: global_singlethread_release_seq_cst_cmpxchg:
3799; GFX11-CU:       ; %bb.0: ; %entry
3800; GFX11-CU-NEXT:    s_clause 0x1
3801; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
3802; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
3803; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
3804; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3805; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3806; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
3807; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3808; GFX11-CU-NEXT:    s_endpgm
3809    i32 addrspace(1)* %out, i32 %in, i32 %old) {
3810entry:
3811  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3812  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") release seq_cst
3813  ret void
3814}
3815
3816define amdgpu_kernel void @global_singlethread_acq_rel_seq_cst_cmpxchg(
3817; GFX6-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
3818; GFX6:       ; %bb.0: ; %entry
3819; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
3820; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3821; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3822; GFX6-NEXT:    s_mov_b32 s2, -1
3823; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3824; GFX6-NEXT:    v_mov_b32_e32 v0, s6
3825; GFX6-NEXT:    v_mov_b32_e32 v1, s7
3826; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3827; GFX6-NEXT:    s_endpgm
3828;
3829; GFX7-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
3830; GFX7:       ; %bb.0: ; %entry
3831; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3832; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3833; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3834; GFX7-NEXT:    s_add_u32 s0, s0, 16
3835; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3836; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3837; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3838; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3839; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3840; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3841; GFX7-NEXT:    s_endpgm
3842;
3843; GFX10-WGP-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
3844; GFX10-WGP:       ; %bb.0: ; %entry
3845; GFX10-WGP-NEXT:    s_clause 0x1
3846; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3847; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3848; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3849; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3850; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3851; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3852; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3853; GFX10-WGP-NEXT:    s_endpgm
3854;
3855; GFX10-CU-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
3856; GFX10-CU:       ; %bb.0: ; %entry
3857; GFX10-CU-NEXT:    s_clause 0x1
3858; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3859; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3860; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3861; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3862; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3863; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3864; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3865; GFX10-CU-NEXT:    s_endpgm
3866;
3867; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
3868; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3869; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
3870; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
3871; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3872; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3873; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3874; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3875; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3876; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3877; SKIP-CACHE-INV-NEXT:    s_endpgm
3878;
3879; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
3880; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3881; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3882; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3883; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3884; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3885; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3886; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3887; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3888;
3889; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
3890; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3891; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3892; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3893; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3894; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3895; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3896; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3897; GFX90A-TGSPLIT-NEXT:    s_endpgm
3898;
3899; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
3900; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3901; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3902; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3903; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3904; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3905; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3906; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
3907; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3908;
3909; GFX940-TGSPLIT-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
3910; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3911; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3912; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3913; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3914; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3915; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3916; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
3917; GFX940-TGSPLIT-NEXT:    s_endpgm
3918;
3919; GFX11-WGP-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
3920; GFX11-WGP:       ; %bb.0: ; %entry
3921; GFX11-WGP-NEXT:    s_clause 0x1
3922; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
3923; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
3924; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
3925; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3926; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3927; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
3928; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3929; GFX11-WGP-NEXT:    s_endpgm
3930;
3931; GFX11-CU-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
3932; GFX11-CU:       ; %bb.0: ; %entry
3933; GFX11-CU-NEXT:    s_clause 0x1
3934; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
3935; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
3936; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
3937; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3938; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3939; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
3940; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3941; GFX11-CU-NEXT:    s_endpgm
3942    i32 addrspace(1)* %out, i32 %in, i32 %old) {
3943entry:
3944  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3945  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel seq_cst
3946  ret void
3947}
3948
3949define amdgpu_kernel void @global_singlethread_seq_cst_seq_cst_cmpxchg(
3950; GFX6-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
3951; GFX6:       ; %bb.0: ; %entry
3952; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
3953; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3954; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3955; GFX6-NEXT:    s_mov_b32 s2, -1
3956; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3957; GFX6-NEXT:    v_mov_b32_e32 v0, s6
3958; GFX6-NEXT:    v_mov_b32_e32 v1, s7
3959; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3960; GFX6-NEXT:    s_endpgm
3961;
3962; GFX7-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
3963; GFX7:       ; %bb.0: ; %entry
3964; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3965; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3966; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3967; GFX7-NEXT:    s_add_u32 s0, s0, 16
3968; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3969; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3970; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3971; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3972; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3973; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3974; GFX7-NEXT:    s_endpgm
3975;
3976; GFX10-WGP-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
3977; GFX10-WGP:       ; %bb.0: ; %entry
3978; GFX10-WGP-NEXT:    s_clause 0x1
3979; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3980; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3981; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3982; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3983; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3984; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3985; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3986; GFX10-WGP-NEXT:    s_endpgm
3987;
3988; GFX10-CU-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
3989; GFX10-CU:       ; %bb.0: ; %entry
3990; GFX10-CU-NEXT:    s_clause 0x1
3991; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3992; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3993; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3994; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3995; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3996; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3997; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3998; GFX10-CU-NEXT:    s_endpgm
3999;
4000; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
4001; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4002; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
4003; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
4004; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
4005; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
4006; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4007; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4008; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4009; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
4010; SKIP-CACHE-INV-NEXT:    s_endpgm
4011;
4012; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
4013; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4014; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4015; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4016; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4017; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4018; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4019; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
4020; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4021;
4022; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
4023; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4024; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4025; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4026; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4027; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4028; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4029; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
4030; GFX90A-TGSPLIT-NEXT:    s_endpgm
4031;
4032; GFX940-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
4033; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4034; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4035; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4036; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4037; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4038; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4039; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
4040; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4041;
4042; GFX940-TGSPLIT-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
4043; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4044; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4045; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4046; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4047; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4048; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4049; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
4050; GFX940-TGSPLIT-NEXT:    s_endpgm
4051;
4052; GFX11-WGP-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
4053; GFX11-WGP:       ; %bb.0: ; %entry
4054; GFX11-WGP-NEXT:    s_clause 0x1
4055; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
4056; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
4057; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
4058; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4059; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4060; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
4061; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4062; GFX11-WGP-NEXT:    s_endpgm
4063;
4064; GFX11-CU-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
4065; GFX11-CU:       ; %bb.0: ; %entry
4066; GFX11-CU-NEXT:    s_clause 0x1
4067; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
4068; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
4069; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
4070; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4071; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4072; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
4073; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4074; GFX11-CU-NEXT:    s_endpgm
4075    i32 addrspace(1)* %out, i32 %in, i32 %old) {
4076entry:
4077  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
4078  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst seq_cst
4079  ret void
4080}
4081
4082define amdgpu_kernel void @global_singlethread_monotonic_monotonic_ret_cmpxchg(
4083; GFX6-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
4084; GFX6:       ; %bb.0: ; %entry
4085; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
4086; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4087; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4088; GFX6-NEXT:    s_mov_b32 s2, -1
4089; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4090; GFX6-NEXT:    v_mov_b32_e32 v0, s6
4091; GFX6-NEXT:    v_mov_b32_e32 v1, s7
4092; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4093; GFX6-NEXT:    s_waitcnt vmcnt(0)
4094; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4095; GFX6-NEXT:    s_endpgm
4096;
4097; GFX7-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
4098; GFX7:       ; %bb.0: ; %entry
4099; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4100; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4101; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4102; GFX7-NEXT:    s_add_u32 s4, s0, 16
4103; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4104; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4105; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4106; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4107; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4108; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4109; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4110; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4111; GFX7-NEXT:    s_waitcnt vmcnt(0)
4112; GFX7-NEXT:    flat_store_dword v[0:1], v2
4113; GFX7-NEXT:    s_endpgm
4114;
4115; GFX10-WGP-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
4116; GFX10-WGP:       ; %bb.0: ; %entry
4117; GFX10-WGP-NEXT:    s_clause 0x1
4118; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4119; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4120; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
4121; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4122; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4123; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4124; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4125; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4126; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
4127; GFX10-WGP-NEXT:    s_endpgm
4128;
4129; GFX10-CU-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
4130; GFX10-CU:       ; %bb.0: ; %entry
4131; GFX10-CU-NEXT:    s_clause 0x1
4132; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4133; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4134; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
4135; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4136; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4137; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4138; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4139; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4140; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
4141; GFX10-CU-NEXT:    s_endpgm
4142;
4143; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
4144; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4145; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
4146; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
4147; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
4148; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
4149; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4150; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4151; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4152; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4153; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4154; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4155; SKIP-CACHE-INV-NEXT:    s_endpgm
4156;
4157; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
4158; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4159; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4160; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4161; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4162; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4163; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4164; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4165; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4166; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4167; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4168;
4169; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
4170; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4171; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4172; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4173; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4174; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4175; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4176; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4177; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4178; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4179; GFX90A-TGSPLIT-NEXT:    s_endpgm
4180;
4181; GFX940-NOTTGSPLIT-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
4182; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4183; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4184; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4185; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4186; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4187; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4188; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4189; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4190; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4191; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4192;
4193; GFX940-TGSPLIT-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
4194; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4195; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4196; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4197; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4198; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4199; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4200; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4201; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4202; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4203; GFX940-TGSPLIT-NEXT:    s_endpgm
4204;
4205; GFX11-WGP-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
4206; GFX11-WGP:       ; %bb.0: ; %entry
4207; GFX11-WGP-NEXT:    s_clause 0x1
4208; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
4209; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
4210; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
4211; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4212; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4213; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
4214; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
4215; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
4216; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4217; GFX11-WGP-NEXT:    s_endpgm
4218;
4219; GFX11-CU-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
4220; GFX11-CU:       ; %bb.0: ; %entry
4221; GFX11-CU-NEXT:    s_clause 0x1
4222; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
4223; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
4224; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
4225; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4226; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4227; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
4228; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
4229; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
4230; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4231; GFX11-CU-NEXT:    s_endpgm
4232    i32 addrspace(1)* %out, i32 %in, i32 %old) {
4233entry:
4234  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
4235  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic monotonic
4236  %val0 = extractvalue { i32, i1 } %val, 0
4237  store i32 %val0, i32 addrspace(1)* %out, align 4
4238  ret void
4239}
4240
4241define amdgpu_kernel void @global_singlethread_acquire_monotonic_ret_cmpxchg(
4242; GFX6-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
4243; GFX6:       ; %bb.0: ; %entry
4244; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
4245; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4246; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4247; GFX6-NEXT:    s_mov_b32 s2, -1
4248; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4249; GFX6-NEXT:    v_mov_b32_e32 v0, s6
4250; GFX6-NEXT:    v_mov_b32_e32 v1, s7
4251; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4252; GFX6-NEXT:    s_waitcnt vmcnt(0)
4253; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4254; GFX6-NEXT:    s_endpgm
4255;
4256; GFX7-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
4257; GFX7:       ; %bb.0: ; %entry
4258; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4259; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4260; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4261; GFX7-NEXT:    s_add_u32 s4, s0, 16
4262; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4263; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4264; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4265; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4266; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4267; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4268; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4269; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4270; GFX7-NEXT:    s_waitcnt vmcnt(0)
4271; GFX7-NEXT:    flat_store_dword v[0:1], v2
4272; GFX7-NEXT:    s_endpgm
4273;
4274; GFX10-WGP-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
4275; GFX10-WGP:       ; %bb.0: ; %entry
4276; GFX10-WGP-NEXT:    s_clause 0x1
4277; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4278; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4279; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
4280; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4281; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4282; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4283; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4284; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4285; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
4286; GFX10-WGP-NEXT:    s_endpgm
4287;
4288; GFX10-CU-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
4289; GFX10-CU:       ; %bb.0: ; %entry
4290; GFX10-CU-NEXT:    s_clause 0x1
4291; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4292; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4293; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
4294; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4295; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4296; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4297; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4298; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4299; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
4300; GFX10-CU-NEXT:    s_endpgm
4301;
4302; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
4303; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4304; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
4305; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
4306; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
4307; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
4308; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4309; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4310; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4311; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4312; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4313; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4314; SKIP-CACHE-INV-NEXT:    s_endpgm
4315;
4316; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
4317; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4318; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4319; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4320; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4321; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4322; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4323; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4324; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4325; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4326; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4327;
4328; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
4329; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4330; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4331; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4332; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4333; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4334; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4335; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4336; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4337; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4338; GFX90A-TGSPLIT-NEXT:    s_endpgm
4339;
4340; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
4341; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4342; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4343; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4344; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4345; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4346; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4347; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4348; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4349; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4350; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4351;
4352; GFX940-TGSPLIT-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
4353; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4354; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4355; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4356; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4357; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4358; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4359; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4360; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4361; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4362; GFX940-TGSPLIT-NEXT:    s_endpgm
4363;
4364; GFX11-WGP-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
4365; GFX11-WGP:       ; %bb.0: ; %entry
4366; GFX11-WGP-NEXT:    s_clause 0x1
4367; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
4368; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
4369; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
4370; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4371; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4372; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
4373; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
4374; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
4375; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4376; GFX11-WGP-NEXT:    s_endpgm
4377;
4378; GFX11-CU-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
4379; GFX11-CU:       ; %bb.0: ; %entry
4380; GFX11-CU-NEXT:    s_clause 0x1
4381; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
4382; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
4383; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
4384; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4385; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4386; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
4387; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
4388; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
4389; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4390; GFX11-CU-NEXT:    s_endpgm
4391    i32 addrspace(1)* %out, i32 %in, i32 %old) {
4392entry:
4393  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
4394  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire monotonic
4395  %val0 = extractvalue { i32, i1 } %val, 0
4396  store i32 %val0, i32 addrspace(1)* %out, align 4
4397  ret void
4398}
4399
4400define amdgpu_kernel void @global_singlethread_release_monotonic_ret_cmpxchg(
4401; GFX6-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
4402; GFX6:       ; %bb.0: ; %entry
4403; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
4404; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4405; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4406; GFX6-NEXT:    s_mov_b32 s2, -1
4407; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4408; GFX6-NEXT:    v_mov_b32_e32 v0, s6
4409; GFX6-NEXT:    v_mov_b32_e32 v1, s7
4410; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4411; GFX6-NEXT:    s_waitcnt vmcnt(0)
4412; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4413; GFX6-NEXT:    s_endpgm
4414;
4415; GFX7-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
4416; GFX7:       ; %bb.0: ; %entry
4417; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4418; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4419; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4420; GFX7-NEXT:    s_add_u32 s4, s0, 16
4421; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4422; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4423; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4424; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4425; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4426; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4427; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4428; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4429; GFX7-NEXT:    s_waitcnt vmcnt(0)
4430; GFX7-NEXT:    flat_store_dword v[0:1], v2
4431; GFX7-NEXT:    s_endpgm
4432;
4433; GFX10-WGP-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
4434; GFX10-WGP:       ; %bb.0: ; %entry
4435; GFX10-WGP-NEXT:    s_clause 0x1
4436; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4437; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4438; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
4439; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4440; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4441; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4442; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4443; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4444; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
4445; GFX10-WGP-NEXT:    s_endpgm
4446;
4447; GFX10-CU-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
4448; GFX10-CU:       ; %bb.0: ; %entry
4449; GFX10-CU-NEXT:    s_clause 0x1
4450; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4451; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4452; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
4453; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4454; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4455; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4456; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4457; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4458; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
4459; GFX10-CU-NEXT:    s_endpgm
4460;
4461; SKIP-CACHE-INV-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
4462; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4463; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
4464; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
4465; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
4466; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
4467; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4468; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4469; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4470; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4471; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4472; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4473; SKIP-CACHE-INV-NEXT:    s_endpgm
4474;
4475; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
4476; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4477; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4478; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4479; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4480; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4481; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4482; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4483; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4484; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4485; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4486;
4487; GFX90A-TGSPLIT-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
4488; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4489; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4490; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4491; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4492; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4493; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4494; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4495; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4496; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4497; GFX90A-TGSPLIT-NEXT:    s_endpgm
4498;
4499; GFX940-NOTTGSPLIT-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
4500; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4501; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4502; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4503; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4504; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4505; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4506; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4507; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4508; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4509; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4510;
4511; GFX940-TGSPLIT-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
4512; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4513; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4514; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4515; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4516; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4517; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4518; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4519; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4520; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4521; GFX940-TGSPLIT-NEXT:    s_endpgm
4522;
4523; GFX11-WGP-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
4524; GFX11-WGP:       ; %bb.0: ; %entry
4525; GFX11-WGP-NEXT:    s_clause 0x1
4526; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
4527; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
4528; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
4529; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4530; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4531; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
4532; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
4533; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
4534; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4535; GFX11-WGP-NEXT:    s_endpgm
4536;
4537; GFX11-CU-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
4538; GFX11-CU:       ; %bb.0: ; %entry
4539; GFX11-CU-NEXT:    s_clause 0x1
4540; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
4541; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
4542; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
4543; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4544; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4545; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
4546; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
4547; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
4548; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4549; GFX11-CU-NEXT:    s_endpgm
4550    i32 addrspace(1)* %out, i32 %in, i32 %old) {
4551entry:
4552  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
4553  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") release monotonic
4554  %val0 = extractvalue { i32, i1 } %val, 0
4555  store i32 %val0, i32 addrspace(1)* %out, align 4
4556  ret void
4557}
4558
4559define amdgpu_kernel void @global_singlethread_acq_rel_monotonic_ret_cmpxchg(
4560; GFX6-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
4561; GFX6:       ; %bb.0: ; %entry
4562; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
4563; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4564; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4565; GFX6-NEXT:    s_mov_b32 s2, -1
4566; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4567; GFX6-NEXT:    v_mov_b32_e32 v0, s6
4568; GFX6-NEXT:    v_mov_b32_e32 v1, s7
4569; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4570; GFX6-NEXT:    s_waitcnt vmcnt(0)
4571; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4572; GFX6-NEXT:    s_endpgm
4573;
4574; GFX7-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
4575; GFX7:       ; %bb.0: ; %entry
4576; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4577; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4578; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4579; GFX7-NEXT:    s_add_u32 s4, s0, 16
4580; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4581; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4582; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4583; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4584; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4585; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4586; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4587; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4588; GFX7-NEXT:    s_waitcnt vmcnt(0)
4589; GFX7-NEXT:    flat_store_dword v[0:1], v2
4590; GFX7-NEXT:    s_endpgm
4591;
4592; GFX10-WGP-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
4593; GFX10-WGP:       ; %bb.0: ; %entry
4594; GFX10-WGP-NEXT:    s_clause 0x1
4595; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4596; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4597; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
4598; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4599; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4600; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4601; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4602; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4603; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
4604; GFX10-WGP-NEXT:    s_endpgm
4605;
4606; GFX10-CU-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
4607; GFX10-CU:       ; %bb.0: ; %entry
4608; GFX10-CU-NEXT:    s_clause 0x1
4609; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4610; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4611; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
4612; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4613; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4614; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4615; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4616; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4617; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
4618; GFX10-CU-NEXT:    s_endpgm
4619;
4620; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
4621; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4622; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
4623; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
4624; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
4625; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
4626; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4627; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4628; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4629; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4630; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4631; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4632; SKIP-CACHE-INV-NEXT:    s_endpgm
4633;
4634; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
4635; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4636; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4637; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4638; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4639; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4640; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4641; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4642; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4643; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4644; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4645;
4646; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
4647; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4648; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4649; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4650; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4651; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4652; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4653; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4654; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4655; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4656; GFX90A-TGSPLIT-NEXT:    s_endpgm
4657;
4658; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
4659; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4660; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4661; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4662; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4663; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4664; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4665; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4666; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4667; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4668; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4669;
4670; GFX940-TGSPLIT-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
4671; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4672; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4673; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4674; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4675; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4676; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4677; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4678; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4679; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4680; GFX940-TGSPLIT-NEXT:    s_endpgm
4681;
4682; GFX11-WGP-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
4683; GFX11-WGP:       ; %bb.0: ; %entry
4684; GFX11-WGP-NEXT:    s_clause 0x1
4685; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
4686; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
4687; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
4688; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4689; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4690; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
4691; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
4692; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
4693; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4694; GFX11-WGP-NEXT:    s_endpgm
4695;
4696; GFX11-CU-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
4697; GFX11-CU:       ; %bb.0: ; %entry
4698; GFX11-CU-NEXT:    s_clause 0x1
4699; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
4700; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
4701; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
4702; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4703; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4704; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
4705; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
4706; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
4707; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4708; GFX11-CU-NEXT:    s_endpgm
4709    i32 addrspace(1)* %out, i32 %in, i32 %old) {
4710entry:
4711  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
4712  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel monotonic
4713  %val0 = extractvalue { i32, i1 } %val, 0
4714  store i32 %val0, i32 addrspace(1)* %out, align 4
4715  ret void
4716}
4717
4718define amdgpu_kernel void @global_singlethread_seq_cst_monotonic_ret_cmpxchg(
4719; GFX6-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
4720; GFX6:       ; %bb.0: ; %entry
4721; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
4722; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4723; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4724; GFX6-NEXT:    s_mov_b32 s2, -1
4725; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4726; GFX6-NEXT:    v_mov_b32_e32 v0, s6
4727; GFX6-NEXT:    v_mov_b32_e32 v1, s7
4728; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4729; GFX6-NEXT:    s_waitcnt vmcnt(0)
4730; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4731; GFX6-NEXT:    s_endpgm
4732;
4733; GFX7-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
4734; GFX7:       ; %bb.0: ; %entry
4735; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4736; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4737; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4738; GFX7-NEXT:    s_add_u32 s4, s0, 16
4739; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4740; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4741; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4742; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4743; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4744; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4745; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4746; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4747; GFX7-NEXT:    s_waitcnt vmcnt(0)
4748; GFX7-NEXT:    flat_store_dword v[0:1], v2
4749; GFX7-NEXT:    s_endpgm
4750;
4751; GFX10-WGP-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
4752; GFX10-WGP:       ; %bb.0: ; %entry
4753; GFX10-WGP-NEXT:    s_clause 0x1
4754; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4755; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4756; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
4757; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4758; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4759; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4760; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4761; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4762; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
4763; GFX10-WGP-NEXT:    s_endpgm
4764;
4765; GFX10-CU-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
4766; GFX10-CU:       ; %bb.0: ; %entry
4767; GFX10-CU-NEXT:    s_clause 0x1
4768; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4769; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4770; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
4771; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4772; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4773; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4774; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4775; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4776; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
4777; GFX10-CU-NEXT:    s_endpgm
4778;
4779; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
4780; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4781; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
4782; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
4783; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
4784; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
4785; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4786; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4787; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4788; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4789; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4790; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4791; SKIP-CACHE-INV-NEXT:    s_endpgm
4792;
4793; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
4794; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4795; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4796; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4797; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4798; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4799; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4800; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4801; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4802; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4803; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4804;
4805; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
4806; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4807; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4808; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4809; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4810; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4811; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4812; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4813; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4814; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4815; GFX90A-TGSPLIT-NEXT:    s_endpgm
4816;
4817; GFX940-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
4818; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4819; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4820; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4821; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4822; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4823; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4824; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4825; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4826; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4827; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4828;
4829; GFX940-TGSPLIT-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
4830; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4831; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4832; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4833; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4834; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4835; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4836; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4837; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4838; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4839; GFX940-TGSPLIT-NEXT:    s_endpgm
4840;
4841; GFX11-WGP-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
4842; GFX11-WGP:       ; %bb.0: ; %entry
4843; GFX11-WGP-NEXT:    s_clause 0x1
4844; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
4845; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
4846; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
4847; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4848; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4849; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
4850; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
4851; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
4852; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4853; GFX11-WGP-NEXT:    s_endpgm
4854;
4855; GFX11-CU-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
4856; GFX11-CU:       ; %bb.0: ; %entry
4857; GFX11-CU-NEXT:    s_clause 0x1
4858; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
4859; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
4860; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
4861; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4862; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4863; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
4864; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
4865; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
4866; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4867; GFX11-CU-NEXT:    s_endpgm
4868    i32 addrspace(1)* %out, i32 %in, i32 %old) {
4869entry:
4870  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
4871  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst monotonic
4872  %val0 = extractvalue { i32, i1 } %val, 0
4873  store i32 %val0, i32 addrspace(1)* %out, align 4
4874  ret void
4875}
4876
4877define amdgpu_kernel void @global_singlethread_monotonic_acquire_ret_cmpxchg(
4878; GFX6-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
4879; GFX6:       ; %bb.0: ; %entry
4880; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
4881; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4882; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4883; GFX6-NEXT:    s_mov_b32 s2, -1
4884; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4885; GFX6-NEXT:    v_mov_b32_e32 v0, s6
4886; GFX6-NEXT:    v_mov_b32_e32 v1, s7
4887; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4888; GFX6-NEXT:    s_waitcnt vmcnt(0)
4889; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4890; GFX6-NEXT:    s_endpgm
4891;
4892; GFX7-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
4893; GFX7:       ; %bb.0: ; %entry
4894; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4895; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4896; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4897; GFX7-NEXT:    s_add_u32 s4, s0, 16
4898; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4899; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4900; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4901; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4902; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4903; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4904; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4905; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4906; GFX7-NEXT:    s_waitcnt vmcnt(0)
4907; GFX7-NEXT:    flat_store_dword v[0:1], v2
4908; GFX7-NEXT:    s_endpgm
4909;
4910; GFX10-WGP-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
4911; GFX10-WGP:       ; %bb.0: ; %entry
4912; GFX10-WGP-NEXT:    s_clause 0x1
4913; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4914; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4915; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
4916; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4917; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4918; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4919; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4920; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4921; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
4922; GFX10-WGP-NEXT:    s_endpgm
4923;
4924; GFX10-CU-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
4925; GFX10-CU:       ; %bb.0: ; %entry
4926; GFX10-CU-NEXT:    s_clause 0x1
4927; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4928; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4929; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
4930; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4931; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4932; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4933; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4934; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4935; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
4936; GFX10-CU-NEXT:    s_endpgm
4937;
4938; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
4939; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4940; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
4941; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
4942; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
4943; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
4944; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4945; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4946; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4947; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4948; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4949; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4950; SKIP-CACHE-INV-NEXT:    s_endpgm
4951;
4952; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
4953; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4954; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4955; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4956; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4957; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4958; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4959; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4960; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4961; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4962; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4963;
4964; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
4965; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4966; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4967; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4968; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4969; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4970; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4971; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4972; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4973; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4974; GFX90A-TGSPLIT-NEXT:    s_endpgm
4975;
4976; GFX940-NOTTGSPLIT-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
4977; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4978; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4979; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4980; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4981; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4982; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4983; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4984; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4985; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4986; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4987;
4988; GFX940-TGSPLIT-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
4989; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4990; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4991; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4992; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4993; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4994; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4995; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4996; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4997; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4998; GFX940-TGSPLIT-NEXT:    s_endpgm
4999;
5000; GFX11-WGP-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
5001; GFX11-WGP:       ; %bb.0: ; %entry
5002; GFX11-WGP-NEXT:    s_clause 0x1
5003; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
5004; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
5005; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
5006; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5007; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5008; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
5009; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
5010; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
5011; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5012; GFX11-WGP-NEXT:    s_endpgm
5013;
5014; GFX11-CU-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
5015; GFX11-CU:       ; %bb.0: ; %entry
5016; GFX11-CU-NEXT:    s_clause 0x1
5017; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
5018; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
5019; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
5020; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5021; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5022; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
5023; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
5024; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
5025; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5026; GFX11-CU-NEXT:    s_endpgm
5027    i32 addrspace(1)* %out, i32 %in, i32 %old) {
5028entry:
5029  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
5030  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic acquire
5031  %val0 = extractvalue { i32, i1 } %val, 0
5032  store i32 %val0, i32 addrspace(1)* %out, align 4
5033  ret void
5034}
5035
5036define amdgpu_kernel void @global_singlethread_acquire_acquire_ret_cmpxchg(
5037; GFX6-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
5038; GFX6:       ; %bb.0: ; %entry
5039; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
5040; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5041; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5042; GFX6-NEXT:    s_mov_b32 s2, -1
5043; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5044; GFX6-NEXT:    v_mov_b32_e32 v0, s6
5045; GFX6-NEXT:    v_mov_b32_e32 v1, s7
5046; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
5047; GFX6-NEXT:    s_waitcnt vmcnt(0)
5048; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5049; GFX6-NEXT:    s_endpgm
5050;
5051; GFX7-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
5052; GFX7:       ; %bb.0: ; %entry
5053; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5054; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5055; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5056; GFX7-NEXT:    s_add_u32 s4, s0, 16
5057; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5058; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5059; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5060; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5061; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5062; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5063; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5064; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5065; GFX7-NEXT:    s_waitcnt vmcnt(0)
5066; GFX7-NEXT:    flat_store_dword v[0:1], v2
5067; GFX7-NEXT:    s_endpgm
5068;
5069; GFX10-WGP-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
5070; GFX10-WGP:       ; %bb.0: ; %entry
5071; GFX10-WGP-NEXT:    s_clause 0x1
5072; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5073; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5074; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
5075; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5076; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5077; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5078; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5079; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5080; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
5081; GFX10-WGP-NEXT:    s_endpgm
5082;
5083; GFX10-CU-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
5084; GFX10-CU:       ; %bb.0: ; %entry
5085; GFX10-CU-NEXT:    s_clause 0x1
5086; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5087; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5088; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
5089; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5090; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5091; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5092; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5093; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5094; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
5095; GFX10-CU-NEXT:    s_endpgm
5096;
5097; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
5098; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5099; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
5100; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
5101; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
5102; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
5103; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5104; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5105; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5106; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
5107; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5108; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5109; SKIP-CACHE-INV-NEXT:    s_endpgm
5110;
5111; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
5112; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5113; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5114; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5115; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5116; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5117; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5118; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5119; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5120; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
5121; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5122;
5123; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
5124; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5125; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5126; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5127; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5128; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5129; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5130; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5131; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5132; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
5133; GFX90A-TGSPLIT-NEXT:    s_endpgm
5134;
5135; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
5136; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5137; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5138; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
5139; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5140; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5141; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5142; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
5143; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5144; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
5145; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5146;
5147; GFX940-TGSPLIT-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
5148; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5149; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5150; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
5151; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5152; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5153; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5154; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
5155; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5156; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
5157; GFX940-TGSPLIT-NEXT:    s_endpgm
5158;
5159; GFX11-WGP-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
5160; GFX11-WGP:       ; %bb.0: ; %entry
5161; GFX11-WGP-NEXT:    s_clause 0x1
5162; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
5163; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
5164; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
5165; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5166; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5167; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
5168; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
5169; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
5170; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5171; GFX11-WGP-NEXT:    s_endpgm
5172;
5173; GFX11-CU-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
5174; GFX11-CU:       ; %bb.0: ; %entry
5175; GFX11-CU-NEXT:    s_clause 0x1
5176; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
5177; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
5178; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
5179; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5180; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5181; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
5182; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
5183; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
5184; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5185; GFX11-CU-NEXT:    s_endpgm
5186    i32 addrspace(1)* %out, i32 %in, i32 %old) {
5187entry:
5188  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
5189  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire acquire
5190  %val0 = extractvalue { i32, i1 } %val, 0
5191  store i32 %val0, i32 addrspace(1)* %out, align 4
5192  ret void
5193}
5194
5195define amdgpu_kernel void @global_singlethread_release_acquire_ret_cmpxchg(
5196; GFX6-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
5197; GFX6:       ; %bb.0: ; %entry
5198; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
5199; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5200; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5201; GFX6-NEXT:    s_mov_b32 s2, -1
5202; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5203; GFX6-NEXT:    v_mov_b32_e32 v0, s6
5204; GFX6-NEXT:    v_mov_b32_e32 v1, s7
5205; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
5206; GFX6-NEXT:    s_waitcnt vmcnt(0)
5207; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5208; GFX6-NEXT:    s_endpgm
5209;
5210; GFX7-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
5211; GFX7:       ; %bb.0: ; %entry
5212; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5213; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5214; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5215; GFX7-NEXT:    s_add_u32 s4, s0, 16
5216; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5217; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5218; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5219; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5220; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5221; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5222; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5223; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5224; GFX7-NEXT:    s_waitcnt vmcnt(0)
5225; GFX7-NEXT:    flat_store_dword v[0:1], v2
5226; GFX7-NEXT:    s_endpgm
5227;
5228; GFX10-WGP-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
5229; GFX10-WGP:       ; %bb.0: ; %entry
5230; GFX10-WGP-NEXT:    s_clause 0x1
5231; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5232; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5233; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
5234; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5235; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5236; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5237; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5238; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5239; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
5240; GFX10-WGP-NEXT:    s_endpgm
5241;
5242; GFX10-CU-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
5243; GFX10-CU:       ; %bb.0: ; %entry
5244; GFX10-CU-NEXT:    s_clause 0x1
5245; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5246; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5247; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
5248; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5249; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5250; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5251; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5252; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5253; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
5254; GFX10-CU-NEXT:    s_endpgm
5255;
5256; SKIP-CACHE-INV-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
5257; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5258; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
5259; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
5260; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
5261; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
5262; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5263; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5264; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5265; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
5266; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5267; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5268; SKIP-CACHE-INV-NEXT:    s_endpgm
5269;
5270; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
5271; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5272; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5273; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5274; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5275; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5276; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5277; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5278; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5279; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
5280; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5281;
5282; GFX90A-TGSPLIT-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
5283; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5284; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5285; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5286; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5287; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5288; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5289; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5290; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5291; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
5292; GFX90A-TGSPLIT-NEXT:    s_endpgm
5293;
5294; GFX940-NOTTGSPLIT-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
5295; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5296; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5297; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
5298; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5299; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5300; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5301; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
5302; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5303; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
5304; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5305;
5306; GFX940-TGSPLIT-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
5307; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5308; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5309; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
5310; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5311; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5312; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5313; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
5314; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5315; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
5316; GFX940-TGSPLIT-NEXT:    s_endpgm
5317;
5318; GFX11-WGP-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
5319; GFX11-WGP:       ; %bb.0: ; %entry
5320; GFX11-WGP-NEXT:    s_clause 0x1
5321; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
5322; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
5323; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
5324; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5325; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5326; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
5327; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
5328; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
5329; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5330; GFX11-WGP-NEXT:    s_endpgm
5331;
5332; GFX11-CU-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
5333; GFX11-CU:       ; %bb.0: ; %entry
5334; GFX11-CU-NEXT:    s_clause 0x1
5335; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
5336; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
5337; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
5338; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5339; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5340; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
5341; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
5342; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
5343; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5344; GFX11-CU-NEXT:    s_endpgm
5345    i32 addrspace(1)* %out, i32 %in, i32 %old) {
5346entry:
5347  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
5348  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") release acquire
5349  %val0 = extractvalue { i32, i1 } %val, 0
5350  store i32 %val0, i32 addrspace(1)* %out, align 4
5351  ret void
5352}
5353
5354define amdgpu_kernel void @global_singlethread_acq_rel_acquire_ret_cmpxchg(
5355; GFX6-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
5356; GFX6:       ; %bb.0: ; %entry
5357; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
5358; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5359; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5360; GFX6-NEXT:    s_mov_b32 s2, -1
5361; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5362; GFX6-NEXT:    v_mov_b32_e32 v0, s6
5363; GFX6-NEXT:    v_mov_b32_e32 v1, s7
5364; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
5365; GFX6-NEXT:    s_waitcnt vmcnt(0)
5366; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5367; GFX6-NEXT:    s_endpgm
5368;
5369; GFX7-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
5370; GFX7:       ; %bb.0: ; %entry
5371; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5372; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5373; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5374; GFX7-NEXT:    s_add_u32 s4, s0, 16
5375; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5376; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5377; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5378; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5379; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5380; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5381; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5382; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5383; GFX7-NEXT:    s_waitcnt vmcnt(0)
5384; GFX7-NEXT:    flat_store_dword v[0:1], v2
5385; GFX7-NEXT:    s_endpgm
5386;
5387; GFX10-WGP-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
5388; GFX10-WGP:       ; %bb.0: ; %entry
5389; GFX10-WGP-NEXT:    s_clause 0x1
5390; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5391; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5392; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
5393; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5394; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5395; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5396; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5397; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5398; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
5399; GFX10-WGP-NEXT:    s_endpgm
5400;
5401; GFX10-CU-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
5402; GFX10-CU:       ; %bb.0: ; %entry
5403; GFX10-CU-NEXT:    s_clause 0x1
5404; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5405; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5406; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
5407; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5408; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5409; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5410; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5411; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5412; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
5413; GFX10-CU-NEXT:    s_endpgm
5414;
5415; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
5416; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5417; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
5418; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
5419; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
5420; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
5421; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5422; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5423; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5424; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
5425; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5426; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5427; SKIP-CACHE-INV-NEXT:    s_endpgm
5428;
5429; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
5430; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5431; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5432; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5433; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5434; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5435; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5436; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5437; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5438; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
5439; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5440;
5441; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
5442; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5443; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5444; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5445; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5446; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5447; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5448; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5449; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5450; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
5451; GFX90A-TGSPLIT-NEXT:    s_endpgm
5452;
5453; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
5454; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5455; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5456; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
5457; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5458; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5459; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5460; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
5461; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5462; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
5463; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5464;
5465; GFX940-TGSPLIT-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
5466; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5467; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5468; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
5469; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5470; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5471; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5472; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
5473; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5474; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
5475; GFX940-TGSPLIT-NEXT:    s_endpgm
5476;
5477; GFX11-WGP-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
5478; GFX11-WGP:       ; %bb.0: ; %entry
5479; GFX11-WGP-NEXT:    s_clause 0x1
5480; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
5481; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
5482; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
5483; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5484; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5485; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
5486; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
5487; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
5488; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5489; GFX11-WGP-NEXT:    s_endpgm
5490;
5491; GFX11-CU-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
5492; GFX11-CU:       ; %bb.0: ; %entry
5493; GFX11-CU-NEXT:    s_clause 0x1
5494; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
5495; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
5496; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
5497; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5498; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5499; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
5500; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
5501; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
5502; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5503; GFX11-CU-NEXT:    s_endpgm
5504    i32 addrspace(1)* %out, i32 %in, i32 %old) {
5505entry:
5506  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
5507  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel acquire
5508  %val0 = extractvalue { i32, i1 } %val, 0
5509  store i32 %val0, i32 addrspace(1)* %out, align 4
5510  ret void
5511}
5512
5513define amdgpu_kernel void @global_singlethread_seq_cst_acquire_ret_cmpxchg(
5514; GFX6-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
5515; GFX6:       ; %bb.0: ; %entry
5516; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
5517; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5518; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5519; GFX6-NEXT:    s_mov_b32 s2, -1
5520; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5521; GFX6-NEXT:    v_mov_b32_e32 v0, s6
5522; GFX6-NEXT:    v_mov_b32_e32 v1, s7
5523; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
5524; GFX6-NEXT:    s_waitcnt vmcnt(0)
5525; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5526; GFX6-NEXT:    s_endpgm
5527;
5528; GFX7-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
5529; GFX7:       ; %bb.0: ; %entry
5530; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5531; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5532; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5533; GFX7-NEXT:    s_add_u32 s4, s0, 16
5534; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5535; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5536; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5537; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5538; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5539; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5540; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5541; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5542; GFX7-NEXT:    s_waitcnt vmcnt(0)
5543; GFX7-NEXT:    flat_store_dword v[0:1], v2
5544; GFX7-NEXT:    s_endpgm
5545;
5546; GFX10-WGP-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
5547; GFX10-WGP:       ; %bb.0: ; %entry
5548; GFX10-WGP-NEXT:    s_clause 0x1
5549; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5550; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5551; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
5552; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5553; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5554; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5555; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5556; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5557; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
5558; GFX10-WGP-NEXT:    s_endpgm
5559;
5560; GFX10-CU-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
5561; GFX10-CU:       ; %bb.0: ; %entry
5562; GFX10-CU-NEXT:    s_clause 0x1
5563; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5564; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5565; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
5566; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5567; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5568; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5569; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5570; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5571; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
5572; GFX10-CU-NEXT:    s_endpgm
5573;
5574; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
5575; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5576; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
5577; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
5578; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
5579; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
5580; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5581; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5582; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5583; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
5584; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5585; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5586; SKIP-CACHE-INV-NEXT:    s_endpgm
5587;
5588; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
5589; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5590; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5591; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5592; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5593; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5594; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5595; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5596; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5597; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
5598; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5599;
5600; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
5601; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5602; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5603; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5604; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5605; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5606; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5607; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5608; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5609; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
5610; GFX90A-TGSPLIT-NEXT:    s_endpgm
5611;
5612; GFX940-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
5613; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5614; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5615; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
5616; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5617; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5618; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5619; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
5620; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5621; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
5622; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5623;
5624; GFX940-TGSPLIT-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
5625; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5626; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5627; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
5628; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5629; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5630; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5631; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
5632; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5633; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
5634; GFX940-TGSPLIT-NEXT:    s_endpgm
5635;
5636; GFX11-WGP-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
5637; GFX11-WGP:       ; %bb.0: ; %entry
5638; GFX11-WGP-NEXT:    s_clause 0x1
5639; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
5640; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
5641; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
5642; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5643; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5644; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
5645; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
5646; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
5647; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5648; GFX11-WGP-NEXT:    s_endpgm
5649;
5650; GFX11-CU-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
5651; GFX11-CU:       ; %bb.0: ; %entry
5652; GFX11-CU-NEXT:    s_clause 0x1
5653; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
5654; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
5655; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
5656; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5657; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5658; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
5659; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
5660; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
5661; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5662; GFX11-CU-NEXT:    s_endpgm
5663    i32 addrspace(1)* %out, i32 %in, i32 %old) {
5664entry:
5665  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
5666  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst acquire
5667  %val0 = extractvalue { i32, i1 } %val, 0
5668  store i32 %val0, i32 addrspace(1)* %out, align 4
5669  ret void
5670}
5671
5672define amdgpu_kernel void @global_singlethread_monotonic_seq_cst_ret_cmpxchg(
5673; GFX6-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
5674; GFX6:       ; %bb.0: ; %entry
5675; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
5676; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5677; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5678; GFX6-NEXT:    s_mov_b32 s2, -1
5679; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5680; GFX6-NEXT:    v_mov_b32_e32 v0, s6
5681; GFX6-NEXT:    v_mov_b32_e32 v1, s7
5682; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
5683; GFX6-NEXT:    s_waitcnt vmcnt(0)
5684; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5685; GFX6-NEXT:    s_endpgm
5686;
5687; GFX7-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
5688; GFX7:       ; %bb.0: ; %entry
5689; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5690; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5691; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5692; GFX7-NEXT:    s_add_u32 s4, s0, 16
5693; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5694; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5695; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5696; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5697; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5698; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5699; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5700; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5701; GFX7-NEXT:    s_waitcnt vmcnt(0)
5702; GFX7-NEXT:    flat_store_dword v[0:1], v2
5703; GFX7-NEXT:    s_endpgm
5704;
5705; GFX10-WGP-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
5706; GFX10-WGP:       ; %bb.0: ; %entry
5707; GFX10-WGP-NEXT:    s_clause 0x1
5708; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5709; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5710; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
5711; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5712; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5713; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5714; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5715; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5716; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
5717; GFX10-WGP-NEXT:    s_endpgm
5718;
5719; GFX10-CU-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
5720; GFX10-CU:       ; %bb.0: ; %entry
5721; GFX10-CU-NEXT:    s_clause 0x1
5722; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5723; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5724; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
5725; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5726; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5727; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5728; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5729; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5730; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
5731; GFX10-CU-NEXT:    s_endpgm
5732;
5733; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
5734; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5735; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
5736; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
5737; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
5738; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
5739; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5740; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5741; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5742; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
5743; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5744; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5745; SKIP-CACHE-INV-NEXT:    s_endpgm
5746;
5747; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
5748; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5749; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5750; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5751; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5752; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5753; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5754; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5755; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5756; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
5757; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5758;
5759; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
5760; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5761; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5762; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5763; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5764; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5765; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5766; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5767; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5768; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
5769; GFX90A-TGSPLIT-NEXT:    s_endpgm
5770;
5771; GFX940-NOTTGSPLIT-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
5772; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5773; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5774; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
5775; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5776; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5777; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5778; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
5779; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5780; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
5781; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5782;
5783; GFX940-TGSPLIT-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
5784; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5785; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5786; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
5787; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5788; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5789; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5790; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
5791; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5792; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
5793; GFX940-TGSPLIT-NEXT:    s_endpgm
5794;
5795; GFX11-WGP-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
5796; GFX11-WGP:       ; %bb.0: ; %entry
5797; GFX11-WGP-NEXT:    s_clause 0x1
5798; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
5799; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
5800; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
5801; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5802; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5803; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
5804; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
5805; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
5806; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5807; GFX11-WGP-NEXT:    s_endpgm
5808;
5809; GFX11-CU-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
5810; GFX11-CU:       ; %bb.0: ; %entry
5811; GFX11-CU-NEXT:    s_clause 0x1
5812; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
5813; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
5814; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
5815; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5816; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5817; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
5818; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
5819; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
5820; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5821; GFX11-CU-NEXT:    s_endpgm
5822    i32 addrspace(1)* %out, i32 %in, i32 %old) {
5823entry:
5824  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
5825  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic seq_cst
5826  %val0 = extractvalue { i32, i1 } %val, 0
5827  store i32 %val0, i32 addrspace(1)* %out, align 4
5828  ret void
5829}
5830
5831define amdgpu_kernel void @global_singlethread_acquire_seq_cst_ret_cmpxchg(
5832; GFX6-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
5833; GFX6:       ; %bb.0: ; %entry
5834; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
5835; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5836; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5837; GFX6-NEXT:    s_mov_b32 s2, -1
5838; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5839; GFX6-NEXT:    v_mov_b32_e32 v0, s6
5840; GFX6-NEXT:    v_mov_b32_e32 v1, s7
5841; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
5842; GFX6-NEXT:    s_waitcnt vmcnt(0)
5843; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5844; GFX6-NEXT:    s_endpgm
5845;
5846; GFX7-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
5847; GFX7:       ; %bb.0: ; %entry
5848; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5849; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5850; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5851; GFX7-NEXT:    s_add_u32 s4, s0, 16
5852; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5853; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5854; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5855; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5856; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5857; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5858; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5859; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5860; GFX7-NEXT:    s_waitcnt vmcnt(0)
5861; GFX7-NEXT:    flat_store_dword v[0:1], v2
5862; GFX7-NEXT:    s_endpgm
5863;
5864; GFX10-WGP-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
5865; GFX10-WGP:       ; %bb.0: ; %entry
5866; GFX10-WGP-NEXT:    s_clause 0x1
5867; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5868; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5869; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
5870; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5871; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5872; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5873; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5874; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5875; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
5876; GFX10-WGP-NEXT:    s_endpgm
5877;
5878; GFX10-CU-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
5879; GFX10-CU:       ; %bb.0: ; %entry
5880; GFX10-CU-NEXT:    s_clause 0x1
5881; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5882; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5883; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
5884; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5885; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5886; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5887; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5888; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5889; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
5890; GFX10-CU-NEXT:    s_endpgm
5891;
5892; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
5893; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5894; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
5895; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
5896; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
5897; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
5898; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5899; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5900; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5901; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
5902; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5903; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5904; SKIP-CACHE-INV-NEXT:    s_endpgm
5905;
5906; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
5907; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5908; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5909; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5910; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5911; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5912; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5913; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5914; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5915; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
5916; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5917;
5918; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
5919; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5920; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5921; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5922; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5923; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5924; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5925; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5926; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5927; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
5928; GFX90A-TGSPLIT-NEXT:    s_endpgm
5929;
5930; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
5931; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5932; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5933; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
5934; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5935; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5936; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5937; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
5938; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5939; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
5940; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5941;
5942; GFX940-TGSPLIT-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
5943; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5944; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5945; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
5946; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5947; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5948; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5949; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
5950; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5951; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
5952; GFX940-TGSPLIT-NEXT:    s_endpgm
5953;
5954; GFX11-WGP-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
5955; GFX11-WGP:       ; %bb.0: ; %entry
5956; GFX11-WGP-NEXT:    s_clause 0x1
5957; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
5958; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
5959; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
5960; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5961; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5962; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
5963; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
5964; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
5965; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5966; GFX11-WGP-NEXT:    s_endpgm
5967;
5968; GFX11-CU-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
5969; GFX11-CU:       ; %bb.0: ; %entry
5970; GFX11-CU-NEXT:    s_clause 0x1
5971; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
5972; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
5973; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
5974; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5975; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5976; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
5977; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
5978; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
5979; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5980; GFX11-CU-NEXT:    s_endpgm
5981    i32 addrspace(1)* %out, i32 %in, i32 %old) {
5982entry:
5983  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
5984  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire seq_cst
5985  %val0 = extractvalue { i32, i1 } %val, 0
5986  store i32 %val0, i32 addrspace(1)* %out, align 4
5987  ret void
5988}
5989
5990define amdgpu_kernel void @global_singlethread_release_seq_cst_ret_cmpxchg(
5991; GFX6-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
5992; GFX6:       ; %bb.0: ; %entry
5993; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
5994; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5995; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5996; GFX6-NEXT:    s_mov_b32 s2, -1
5997; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5998; GFX6-NEXT:    v_mov_b32_e32 v0, s6
5999; GFX6-NEXT:    v_mov_b32_e32 v1, s7
6000; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
6001; GFX6-NEXT:    s_waitcnt vmcnt(0)
6002; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
6003; GFX6-NEXT:    s_endpgm
6004;
6005; GFX7-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
6006; GFX7:       ; %bb.0: ; %entry
6007; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6008; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6009; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6010; GFX7-NEXT:    s_add_u32 s4, s0, 16
6011; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6012; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6013; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6014; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6015; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6016; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6017; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6018; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6019; GFX7-NEXT:    s_waitcnt vmcnt(0)
6020; GFX7-NEXT:    flat_store_dword v[0:1], v2
6021; GFX7-NEXT:    s_endpgm
6022;
6023; GFX10-WGP-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
6024; GFX10-WGP:       ; %bb.0: ; %entry
6025; GFX10-WGP-NEXT:    s_clause 0x1
6026; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6027; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6028; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
6029; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6030; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6031; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6032; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
6033; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6034; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
6035; GFX10-WGP-NEXT:    s_endpgm
6036;
6037; GFX10-CU-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
6038; GFX10-CU:       ; %bb.0: ; %entry
6039; GFX10-CU-NEXT:    s_clause 0x1
6040; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6041; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6042; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
6043; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6044; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6045; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6046; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
6047; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6048; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
6049; GFX10-CU-NEXT:    s_endpgm
6050;
6051; SKIP-CACHE-INV-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
6052; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6053; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
6054; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6055; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
6056; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
6057; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6058; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6059; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6060; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
6061; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6062; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
6063; SKIP-CACHE-INV-NEXT:    s_endpgm
6064;
6065; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
6066; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6067; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6068; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6069; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6070; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6071; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6072; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
6073; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6074; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
6075; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6076;
6077; GFX90A-TGSPLIT-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
6078; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6079; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6080; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6081; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6082; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6083; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6084; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
6085; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6086; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
6087; GFX90A-TGSPLIT-NEXT:    s_endpgm
6088;
6089; GFX940-NOTTGSPLIT-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
6090; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6091; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
6092; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
6093; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6094; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6095; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6096; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
6097; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6098; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
6099; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6100;
6101; GFX940-TGSPLIT-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
6102; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6103; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
6104; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
6105; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6106; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6107; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6108; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
6109; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6110; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
6111; GFX940-TGSPLIT-NEXT:    s_endpgm
6112;
6113; GFX11-WGP-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
6114; GFX11-WGP:       ; %bb.0: ; %entry
6115; GFX11-WGP-NEXT:    s_clause 0x1
6116; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
6117; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6118; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
6119; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6120; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6121; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
6122; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
6123; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
6124; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6125; GFX11-WGP-NEXT:    s_endpgm
6126;
6127; GFX11-CU-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
6128; GFX11-CU:       ; %bb.0: ; %entry
6129; GFX11-CU-NEXT:    s_clause 0x1
6130; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
6131; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6132; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
6133; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6134; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6135; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
6136; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
6137; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
6138; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6139; GFX11-CU-NEXT:    s_endpgm
6140    i32 addrspace(1)* %out, i32 %in, i32 %old) {
6141entry:
6142  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
6143  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") release seq_cst
6144  %val0 = extractvalue { i32, i1 } %val, 0
6145  store i32 %val0, i32 addrspace(1)* %out, align 4
6146  ret void
6147}
6148
6149define amdgpu_kernel void @global_singlethread_acq_rel_seq_cst_ret_cmpxchg(
6150; GFX6-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6151; GFX6:       ; %bb.0: ; %entry
6152; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
6153; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6154; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
6155; GFX6-NEXT:    s_mov_b32 s2, -1
6156; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6157; GFX6-NEXT:    v_mov_b32_e32 v0, s6
6158; GFX6-NEXT:    v_mov_b32_e32 v1, s7
6159; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
6160; GFX6-NEXT:    s_waitcnt vmcnt(0)
6161; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
6162; GFX6-NEXT:    s_endpgm
6163;
6164; GFX7-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6165; GFX7:       ; %bb.0: ; %entry
6166; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6167; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6168; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6169; GFX7-NEXT:    s_add_u32 s4, s0, 16
6170; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6171; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6172; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6173; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6174; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6175; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6176; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6177; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6178; GFX7-NEXT:    s_waitcnt vmcnt(0)
6179; GFX7-NEXT:    flat_store_dword v[0:1], v2
6180; GFX7-NEXT:    s_endpgm
6181;
6182; GFX10-WGP-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6183; GFX10-WGP:       ; %bb.0: ; %entry
6184; GFX10-WGP-NEXT:    s_clause 0x1
6185; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6186; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6187; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
6188; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6189; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6190; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6191; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
6192; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6193; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
6194; GFX10-WGP-NEXT:    s_endpgm
6195;
6196; GFX10-CU-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6197; GFX10-CU:       ; %bb.0: ; %entry
6198; GFX10-CU-NEXT:    s_clause 0x1
6199; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6200; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6201; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
6202; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6203; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6204; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6205; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
6206; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6207; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
6208; GFX10-CU-NEXT:    s_endpgm
6209;
6210; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6211; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6212; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
6213; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6214; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
6215; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
6216; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6217; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6218; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6219; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
6220; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6221; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
6222; SKIP-CACHE-INV-NEXT:    s_endpgm
6223;
6224; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6225; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6226; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6227; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6228; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6229; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6230; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6231; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
6232; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6233; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
6234; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6235;
6236; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6237; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6238; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6239; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6240; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6241; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6242; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6243; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
6244; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6245; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
6246; GFX90A-TGSPLIT-NEXT:    s_endpgm
6247;
6248; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6249; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6250; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
6251; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
6252; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6253; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6254; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6255; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
6256; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6257; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
6258; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6259;
6260; GFX940-TGSPLIT-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6261; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6262; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
6263; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
6264; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6265; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6266; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6267; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
6268; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6269; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
6270; GFX940-TGSPLIT-NEXT:    s_endpgm
6271;
6272; GFX11-WGP-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6273; GFX11-WGP:       ; %bb.0: ; %entry
6274; GFX11-WGP-NEXT:    s_clause 0x1
6275; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
6276; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6277; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
6278; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6279; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6280; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
6281; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
6282; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
6283; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6284; GFX11-WGP-NEXT:    s_endpgm
6285;
6286; GFX11-CU-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6287; GFX11-CU:       ; %bb.0: ; %entry
6288; GFX11-CU-NEXT:    s_clause 0x1
6289; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
6290; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6291; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
6292; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6293; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6294; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
6295; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
6296; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
6297; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6298; GFX11-CU-NEXT:    s_endpgm
6299    i32 addrspace(1)* %out, i32 %in, i32 %old) {
6300entry:
6301  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
6302  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel seq_cst
6303  %val0 = extractvalue { i32, i1 } %val, 0
6304  store i32 %val0, i32 addrspace(1)* %out, align 4
6305  ret void
6306}
6307
6308define amdgpu_kernel void @global_singlethread_seq_cst_seq_cst_ret_cmpxchg(
6309; GFX6-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6310; GFX6:       ; %bb.0: ; %entry
6311; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
6312; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6313; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
6314; GFX6-NEXT:    s_mov_b32 s2, -1
6315; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6316; GFX6-NEXT:    v_mov_b32_e32 v0, s6
6317; GFX6-NEXT:    v_mov_b32_e32 v1, s7
6318; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
6319; GFX6-NEXT:    s_waitcnt vmcnt(0)
6320; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
6321; GFX6-NEXT:    s_endpgm
6322;
6323; GFX7-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6324; GFX7:       ; %bb.0: ; %entry
6325; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6326; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6327; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6328; GFX7-NEXT:    s_add_u32 s4, s0, 16
6329; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6330; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6331; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6332; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6333; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6334; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6335; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6336; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6337; GFX7-NEXT:    s_waitcnt vmcnt(0)
6338; GFX7-NEXT:    flat_store_dword v[0:1], v2
6339; GFX7-NEXT:    s_endpgm
6340;
6341; GFX10-WGP-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6342; GFX10-WGP:       ; %bb.0: ; %entry
6343; GFX10-WGP-NEXT:    s_clause 0x1
6344; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6345; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6346; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
6347; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6348; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6349; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6350; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
6351; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6352; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
6353; GFX10-WGP-NEXT:    s_endpgm
6354;
6355; GFX10-CU-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6356; GFX10-CU:       ; %bb.0: ; %entry
6357; GFX10-CU-NEXT:    s_clause 0x1
6358; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6359; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6360; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
6361; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6362; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6363; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6364; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
6365; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6366; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
6367; GFX10-CU-NEXT:    s_endpgm
6368;
6369; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6370; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6371; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
6372; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6373; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
6374; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
6375; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6376; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6377; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6378; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
6379; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6380; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
6381; SKIP-CACHE-INV-NEXT:    s_endpgm
6382;
6383; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6384; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6385; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6386; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6387; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6388; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6389; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6390; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
6391; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6392; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
6393; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6394;
6395; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6396; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6397; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6398; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6399; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6400; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6401; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6402; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
6403; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6404; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
6405; GFX90A-TGSPLIT-NEXT:    s_endpgm
6406;
6407; GFX940-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6408; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6409; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
6410; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
6411; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6412; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6413; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6414; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
6415; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6416; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
6417; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6418;
6419; GFX940-TGSPLIT-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6420; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6421; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
6422; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
6423; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6424; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6425; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6426; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
6427; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6428; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
6429; GFX940-TGSPLIT-NEXT:    s_endpgm
6430;
6431; GFX11-WGP-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6432; GFX11-WGP:       ; %bb.0: ; %entry
6433; GFX11-WGP-NEXT:    s_clause 0x1
6434; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
6435; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6436; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
6437; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6438; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6439; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
6440; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
6441; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
6442; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6443; GFX11-WGP-NEXT:    s_endpgm
6444;
6445; GFX11-CU-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6446; GFX11-CU:       ; %bb.0: ; %entry
6447; GFX11-CU-NEXT:    s_clause 0x1
6448; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
6449; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6450; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
6451; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6452; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6453; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
6454; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
6455; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
6456; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6457; GFX11-CU-NEXT:    s_endpgm
6458    i32 addrspace(1)* %out, i32 %in, i32 %old) {
6459entry:
6460  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
6461  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst seq_cst
6462  %val0 = extractvalue { i32, i1 } %val, 0
6463  store i32 %val0, i32 addrspace(1)* %out, align 4
6464  ret void
6465}
6466
6467define amdgpu_kernel void @global_singlethread_one_as_unordered_load(
6468; GFX6-LABEL: global_singlethread_one_as_unordered_load:
6469; GFX6:       ; %bb.0: ; %entry
6470; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6471; GFX6-NEXT:    s_mov_b32 s7, 0x100f000
6472; GFX6-NEXT:    s_mov_b32 s6, -1
6473; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6474; GFX6-NEXT:    s_mov_b32 s4, s0
6475; GFX6-NEXT:    s_mov_b32 s5, s1
6476; GFX6-NEXT:    buffer_load_dword v0, off, s[4:7], 0
6477; GFX6-NEXT:    s_mov_b32 s4, s2
6478; GFX6-NEXT:    s_mov_b32 s5, s3
6479; GFX6-NEXT:    s_waitcnt vmcnt(0)
6480; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
6481; GFX6-NEXT:    s_endpgm
6482;
6483; GFX7-LABEL: global_singlethread_one_as_unordered_load:
6484; GFX7:       ; %bb.0: ; %entry
6485; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6486; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6487; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6488; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6489; GFX7-NEXT:    flat_load_dword v2, v[0:1]
6490; GFX7-NEXT:    v_mov_b32_e32 v0, s2
6491; GFX7-NEXT:    v_mov_b32_e32 v1, s3
6492; GFX7-NEXT:    s_waitcnt vmcnt(0)
6493; GFX7-NEXT:    flat_store_dword v[0:1], v2
6494; GFX7-NEXT:    s_endpgm
6495;
6496; GFX10-WGP-LABEL: global_singlethread_one_as_unordered_load:
6497; GFX10-WGP:       ; %bb.0: ; %entry
6498; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6499; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
6500; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6501; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
6502; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6503; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
6504; GFX10-WGP-NEXT:    s_endpgm
6505;
6506; GFX10-CU-LABEL: global_singlethread_one_as_unordered_load:
6507; GFX10-CU:       ; %bb.0: ; %entry
6508; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6509; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
6510; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6511; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
6512; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6513; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
6514; GFX10-CU-NEXT:    s_endpgm
6515;
6516; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_unordered_load:
6517; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6518; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6519; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
6520; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
6521; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6522; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s0
6523; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s1
6524; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[4:7], 0
6525; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s2
6526; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s3
6527; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6528; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
6529; SKIP-CACHE-INV-NEXT:    s_endpgm
6530;
6531; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_unordered_load:
6532; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6533; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6534; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6535; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6536; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
6537; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6538; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6539; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6540;
6541; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_unordered_load:
6542; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6543; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6544; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6545; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6546; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
6547; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6548; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6549; GFX90A-TGSPLIT-NEXT:    s_endpgm
6550;
6551; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_unordered_load:
6552; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6553; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6554; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6555; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6556; GFX940-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
6557; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6558; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6559; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6560;
6561; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_unordered_load:
6562; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6563; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6564; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6565; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6566; GFX940-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
6567; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6568; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6569; GFX940-TGSPLIT-NEXT:    s_endpgm
6570;
6571; GFX11-WGP-LABEL: global_singlethread_one_as_unordered_load:
6572; GFX11-WGP:       ; %bb.0: ; %entry
6573; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6574; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
6575; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6576; GFX11-WGP-NEXT:    global_load_b32 v1, v0, s[0:1]
6577; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
6578; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[2:3]
6579; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6580; GFX11-WGP-NEXT:    s_endpgm
6581;
6582; GFX11-CU-LABEL: global_singlethread_one_as_unordered_load:
6583; GFX11-CU:       ; %bb.0: ; %entry
6584; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6585; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
6586; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6587; GFX11-CU-NEXT:    global_load_b32 v1, v0, s[0:1]
6588; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
6589; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[2:3]
6590; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6591; GFX11-CU-NEXT:    s_endpgm
6592    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
6593entry:
6594  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread-one-as") unordered, align 4
6595  store i32 %val, i32 addrspace(1)* %out
6596  ret void
6597}
6598
6599define amdgpu_kernel void @global_singlethread_one_as_monotonic_load(
6600; GFX6-LABEL: global_singlethread_one_as_monotonic_load:
6601; GFX6:       ; %bb.0: ; %entry
6602; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6603; GFX6-NEXT:    s_mov_b32 s7, 0x100f000
6604; GFX6-NEXT:    s_mov_b32 s6, -1
6605; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6606; GFX6-NEXT:    s_mov_b32 s4, s0
6607; GFX6-NEXT:    s_mov_b32 s5, s1
6608; GFX6-NEXT:    buffer_load_dword v0, off, s[4:7], 0
6609; GFX6-NEXT:    s_mov_b32 s4, s2
6610; GFX6-NEXT:    s_mov_b32 s5, s3
6611; GFX6-NEXT:    s_waitcnt vmcnt(0)
6612; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
6613; GFX6-NEXT:    s_endpgm
6614;
6615; GFX7-LABEL: global_singlethread_one_as_monotonic_load:
6616; GFX7:       ; %bb.0: ; %entry
6617; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6618; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6619; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6620; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6621; GFX7-NEXT:    flat_load_dword v2, v[0:1]
6622; GFX7-NEXT:    v_mov_b32_e32 v0, s2
6623; GFX7-NEXT:    v_mov_b32_e32 v1, s3
6624; GFX7-NEXT:    s_waitcnt vmcnt(0)
6625; GFX7-NEXT:    flat_store_dword v[0:1], v2
6626; GFX7-NEXT:    s_endpgm
6627;
6628; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_load:
6629; GFX10-WGP:       ; %bb.0: ; %entry
6630; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6631; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
6632; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6633; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
6634; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6635; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
6636; GFX10-WGP-NEXT:    s_endpgm
6637;
6638; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_load:
6639; GFX10-CU:       ; %bb.0: ; %entry
6640; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6641; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
6642; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6643; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
6644; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6645; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
6646; GFX10-CU-NEXT:    s_endpgm
6647;
6648; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_load:
6649; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6650; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6651; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
6652; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
6653; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6654; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s0
6655; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s1
6656; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[4:7], 0
6657; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s2
6658; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s3
6659; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6660; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
6661; SKIP-CACHE-INV-NEXT:    s_endpgm
6662;
6663; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_load:
6664; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6665; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6666; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6667; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6668; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
6669; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6670; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6671; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6672;
6673; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_load:
6674; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6675; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6676; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6677; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6678; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
6679; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6680; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6681; GFX90A-TGSPLIT-NEXT:    s_endpgm
6682;
6683; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_load:
6684; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6685; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6686; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6687; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6688; GFX940-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
6689; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6690; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6691; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6692;
6693; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_load:
6694; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6695; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6696; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6697; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6698; GFX940-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
6699; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6700; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6701; GFX940-TGSPLIT-NEXT:    s_endpgm
6702;
6703; GFX11-WGP-LABEL: global_singlethread_one_as_monotonic_load:
6704; GFX11-WGP:       ; %bb.0: ; %entry
6705; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6706; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
6707; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6708; GFX11-WGP-NEXT:    global_load_b32 v1, v0, s[0:1]
6709; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
6710; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[2:3]
6711; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6712; GFX11-WGP-NEXT:    s_endpgm
6713;
6714; GFX11-CU-LABEL: global_singlethread_one_as_monotonic_load:
6715; GFX11-CU:       ; %bb.0: ; %entry
6716; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6717; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
6718; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6719; GFX11-CU-NEXT:    global_load_b32 v1, v0, s[0:1]
6720; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
6721; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[2:3]
6722; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6723; GFX11-CU-NEXT:    s_endpgm
6724    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
6725entry:
6726  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread-one-as") monotonic, align 4
6727  store i32 %val, i32 addrspace(1)* %out
6728  ret void
6729}
6730
6731define amdgpu_kernel void @global_singlethread_one_as_acquire_load(
6732; GFX6-LABEL: global_singlethread_one_as_acquire_load:
6733; GFX6:       ; %bb.0: ; %entry
6734; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6735; GFX6-NEXT:    s_mov_b32 s7, 0x100f000
6736; GFX6-NEXT:    s_mov_b32 s6, -1
6737; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6738; GFX6-NEXT:    s_mov_b32 s4, s0
6739; GFX6-NEXT:    s_mov_b32 s5, s1
6740; GFX6-NEXT:    buffer_load_dword v0, off, s[4:7], 0
6741; GFX6-NEXT:    s_mov_b32 s4, s2
6742; GFX6-NEXT:    s_mov_b32 s5, s3
6743; GFX6-NEXT:    s_waitcnt vmcnt(0)
6744; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
6745; GFX6-NEXT:    s_endpgm
6746;
6747; GFX7-LABEL: global_singlethread_one_as_acquire_load:
6748; GFX7:       ; %bb.0: ; %entry
6749; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6750; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6751; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6752; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6753; GFX7-NEXT:    flat_load_dword v2, v[0:1]
6754; GFX7-NEXT:    v_mov_b32_e32 v0, s2
6755; GFX7-NEXT:    v_mov_b32_e32 v1, s3
6756; GFX7-NEXT:    s_waitcnt vmcnt(0)
6757; GFX7-NEXT:    flat_store_dword v[0:1], v2
6758; GFX7-NEXT:    s_endpgm
6759;
6760; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_load:
6761; GFX10-WGP:       ; %bb.0: ; %entry
6762; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6763; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
6764; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6765; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
6766; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6767; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
6768; GFX10-WGP-NEXT:    s_endpgm
6769;
6770; GFX10-CU-LABEL: global_singlethread_one_as_acquire_load:
6771; GFX10-CU:       ; %bb.0: ; %entry
6772; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6773; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
6774; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6775; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
6776; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6777; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
6778; GFX10-CU-NEXT:    s_endpgm
6779;
6780; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_load:
6781; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6782; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6783; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
6784; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
6785; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6786; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s0
6787; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s1
6788; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[4:7], 0
6789; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s2
6790; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s3
6791; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6792; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
6793; SKIP-CACHE-INV-NEXT:    s_endpgm
6794;
6795; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_load:
6796; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6797; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6798; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6799; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6800; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
6801; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6802; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6803; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6804;
6805; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_load:
6806; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6807; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6808; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6809; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6810; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
6811; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6812; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6813; GFX90A-TGSPLIT-NEXT:    s_endpgm
6814;
6815; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_load:
6816; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6817; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6818; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6819; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6820; GFX940-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
6821; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6822; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6823; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6824;
6825; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acquire_load:
6826; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6827; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6828; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6829; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6830; GFX940-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
6831; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6832; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6833; GFX940-TGSPLIT-NEXT:    s_endpgm
6834;
6835; GFX11-WGP-LABEL: global_singlethread_one_as_acquire_load:
6836; GFX11-WGP:       ; %bb.0: ; %entry
6837; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6838; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
6839; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6840; GFX11-WGP-NEXT:    global_load_b32 v1, v0, s[0:1]
6841; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
6842; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[2:3]
6843; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6844; GFX11-WGP-NEXT:    s_endpgm
6845;
6846; GFX11-CU-LABEL: global_singlethread_one_as_acquire_load:
6847; GFX11-CU:       ; %bb.0: ; %entry
6848; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6849; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
6850; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6851; GFX11-CU-NEXT:    global_load_b32 v1, v0, s[0:1]
6852; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
6853; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[2:3]
6854; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6855; GFX11-CU-NEXT:    s_endpgm
6856    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
6857entry:
6858  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread-one-as") acquire, align 4
6859  store i32 %val, i32 addrspace(1)* %out
6860  ret void
6861}
6862
6863define amdgpu_kernel void @global_singlethread_one_as_seq_cst_load(
6864; GFX6-LABEL: global_singlethread_one_as_seq_cst_load:
6865; GFX6:       ; %bb.0: ; %entry
6866; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6867; GFX6-NEXT:    s_mov_b32 s7, 0x100f000
6868; GFX6-NEXT:    s_mov_b32 s6, -1
6869; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6870; GFX6-NEXT:    s_mov_b32 s4, s0
6871; GFX6-NEXT:    s_mov_b32 s5, s1
6872; GFX6-NEXT:    buffer_load_dword v0, off, s[4:7], 0
6873; GFX6-NEXT:    s_mov_b32 s4, s2
6874; GFX6-NEXT:    s_mov_b32 s5, s3
6875; GFX6-NEXT:    s_waitcnt vmcnt(0)
6876; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
6877; GFX6-NEXT:    s_endpgm
6878;
6879; GFX7-LABEL: global_singlethread_one_as_seq_cst_load:
6880; GFX7:       ; %bb.0: ; %entry
6881; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6882; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6883; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6884; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6885; GFX7-NEXT:    flat_load_dword v2, v[0:1]
6886; GFX7-NEXT:    v_mov_b32_e32 v0, s2
6887; GFX7-NEXT:    v_mov_b32_e32 v1, s3
6888; GFX7-NEXT:    s_waitcnt vmcnt(0)
6889; GFX7-NEXT:    flat_store_dword v[0:1], v2
6890; GFX7-NEXT:    s_endpgm
6891;
6892; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_load:
6893; GFX10-WGP:       ; %bb.0: ; %entry
6894; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6895; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
6896; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6897; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
6898; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6899; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
6900; GFX10-WGP-NEXT:    s_endpgm
6901;
6902; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_load:
6903; GFX10-CU:       ; %bb.0: ; %entry
6904; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6905; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
6906; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6907; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
6908; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6909; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
6910; GFX10-CU-NEXT:    s_endpgm
6911;
6912; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_load:
6913; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6914; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6915; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
6916; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
6917; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6918; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s0
6919; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s1
6920; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[4:7], 0
6921; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s2
6922; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s3
6923; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6924; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
6925; SKIP-CACHE-INV-NEXT:    s_endpgm
6926;
6927; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_load:
6928; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6929; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6930; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6931; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6932; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
6933; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6934; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6935; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6936;
6937; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_load:
6938; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6939; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6940; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6941; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6942; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
6943; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6944; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6945; GFX90A-TGSPLIT-NEXT:    s_endpgm
6946;
6947; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_load:
6948; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6949; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6950; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6951; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6952; GFX940-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
6953; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6954; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6955; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6956;
6957; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_load:
6958; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6959; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6960; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6961; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6962; GFX940-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
6963; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6964; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6965; GFX940-TGSPLIT-NEXT:    s_endpgm
6966;
6967; GFX11-WGP-LABEL: global_singlethread_one_as_seq_cst_load:
6968; GFX11-WGP:       ; %bb.0: ; %entry
6969; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6970; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
6971; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6972; GFX11-WGP-NEXT:    global_load_b32 v1, v0, s[0:1]
6973; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
6974; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[2:3]
6975; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6976; GFX11-WGP-NEXT:    s_endpgm
6977;
6978; GFX11-CU-LABEL: global_singlethread_one_as_seq_cst_load:
6979; GFX11-CU:       ; %bb.0: ; %entry
6980; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6981; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
6982; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6983; GFX11-CU-NEXT:    global_load_b32 v1, v0, s[0:1]
6984; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
6985; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[2:3]
6986; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6987; GFX11-CU-NEXT:    s_endpgm
6988    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
6989entry:
6990  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread-one-as") seq_cst, align 4
6991  store i32 %val, i32 addrspace(1)* %out
6992  ret void
6993}
6994
6995define amdgpu_kernel void @global_singlethread_one_as_unordered_store(
6996; GFX6-LABEL: global_singlethread_one_as_unordered_store:
6997; GFX6:       ; %bb.0: ; %entry
6998; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
6999; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
7000; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7001; GFX6-NEXT:    s_mov_b32 s2, -1
7002; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7003; GFX6-NEXT:    v_mov_b32_e32 v0, s6
7004; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
7005; GFX6-NEXT:    s_endpgm
7006;
7007; GFX7-LABEL: global_singlethread_one_as_unordered_store:
7008; GFX7:       ; %bb.0: ; %entry
7009; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
7010; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
7011; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7012; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7013; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7014; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7015; GFX7-NEXT:    flat_store_dword v[0:1], v2
7016; GFX7-NEXT:    s_endpgm
7017;
7018; GFX10-WGP-LABEL: global_singlethread_one_as_unordered_store:
7019; GFX10-WGP:       ; %bb.0: ; %entry
7020; GFX10-WGP-NEXT:    s_clause 0x1
7021; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
7022; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7023; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
7024; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7025; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
7026; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
7027; GFX10-WGP-NEXT:    s_endpgm
7028;
7029; GFX10-CU-LABEL: global_singlethread_one_as_unordered_store:
7030; GFX10-CU:       ; %bb.0: ; %entry
7031; GFX10-CU-NEXT:    s_clause 0x1
7032; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
7033; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7034; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
7035; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7036; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
7037; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
7038; GFX10-CU-NEXT:    s_endpgm
7039;
7040; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_unordered_store:
7041; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7042; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x0
7043; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7044; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
7045; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
7046; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7047; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7048; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
7049; SKIP-CACHE-INV-NEXT:    s_endpgm
7050;
7051; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_unordered_store:
7052; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7053; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7054; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7055; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7056; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7057; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7058; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
7059; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7060;
7061; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_unordered_store:
7062; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7063; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7064; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7065; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7066; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7067; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7068; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
7069; GFX90A-TGSPLIT-NEXT:    s_endpgm
7070;
7071; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_unordered_store:
7072; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7073; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7074; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7075; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7076; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7077; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
7078; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
7079; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7080;
7081; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_unordered_store:
7082; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7083; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7084; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7085; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7086; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7087; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
7088; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
7089; GFX940-TGSPLIT-NEXT:    s_endpgm
7090;
7091; GFX11-WGP-LABEL: global_singlethread_one_as_unordered_store:
7092; GFX11-WGP:       ; %bb.0: ; %entry
7093; GFX11-WGP-NEXT:    s_clause 0x1
7094; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x0
7095; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7096; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7097; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
7098; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[0:1]
7099; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7100; GFX11-WGP-NEXT:    s_endpgm
7101;
7102; GFX11-CU-LABEL: global_singlethread_one_as_unordered_store:
7103; GFX11-CU:       ; %bb.0: ; %entry
7104; GFX11-CU-NEXT:    s_clause 0x1
7105; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x0
7106; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7107; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7108; GFX11-CU-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
7109; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
7110; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7111; GFX11-CU-NEXT:    s_endpgm
7112    i32 %in, i32 addrspace(1)* %out) {
7113entry:
7114  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread-one-as") unordered, align 4
7115  ret void
7116}
7117
7118define amdgpu_kernel void @global_singlethread_one_as_monotonic_store(
7119; GFX6-LABEL: global_singlethread_one_as_monotonic_store:
7120; GFX6:       ; %bb.0: ; %entry
7121; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
7122; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
7123; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7124; GFX6-NEXT:    s_mov_b32 s2, -1
7125; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7126; GFX6-NEXT:    v_mov_b32_e32 v0, s6
7127; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
7128; GFX6-NEXT:    s_endpgm
7129;
7130; GFX7-LABEL: global_singlethread_one_as_monotonic_store:
7131; GFX7:       ; %bb.0: ; %entry
7132; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
7133; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
7134; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7135; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7136; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7137; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7138; GFX7-NEXT:    flat_store_dword v[0:1], v2
7139; GFX7-NEXT:    s_endpgm
7140;
7141; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_store:
7142; GFX10-WGP:       ; %bb.0: ; %entry
7143; GFX10-WGP-NEXT:    s_clause 0x1
7144; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
7145; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7146; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
7147; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7148; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
7149; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
7150; GFX10-WGP-NEXT:    s_endpgm
7151;
7152; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_store:
7153; GFX10-CU:       ; %bb.0: ; %entry
7154; GFX10-CU-NEXT:    s_clause 0x1
7155; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
7156; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7157; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
7158; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7159; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
7160; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
7161; GFX10-CU-NEXT:    s_endpgm
7162;
7163; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_store:
7164; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7165; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x0
7166; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7167; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
7168; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
7169; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7170; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7171; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
7172; SKIP-CACHE-INV-NEXT:    s_endpgm
7173;
7174; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_store:
7175; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7176; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7177; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7178; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7179; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7180; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7181; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
7182; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7183;
7184; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_store:
7185; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7186; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7187; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7188; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7189; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7190; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7191; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
7192; GFX90A-TGSPLIT-NEXT:    s_endpgm
7193;
7194; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_store:
7195; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7196; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7197; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7198; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7199; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7200; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
7201; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
7202; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7203;
7204; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_store:
7205; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7206; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7207; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7208; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7209; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7210; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
7211; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
7212; GFX940-TGSPLIT-NEXT:    s_endpgm
7213;
7214; GFX11-WGP-LABEL: global_singlethread_one_as_monotonic_store:
7215; GFX11-WGP:       ; %bb.0: ; %entry
7216; GFX11-WGP-NEXT:    s_clause 0x1
7217; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x0
7218; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7219; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7220; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
7221; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[0:1]
7222; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7223; GFX11-WGP-NEXT:    s_endpgm
7224;
7225; GFX11-CU-LABEL: global_singlethread_one_as_monotonic_store:
7226; GFX11-CU:       ; %bb.0: ; %entry
7227; GFX11-CU-NEXT:    s_clause 0x1
7228; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x0
7229; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7230; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7231; GFX11-CU-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
7232; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
7233; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7234; GFX11-CU-NEXT:    s_endpgm
7235    i32 %in, i32 addrspace(1)* %out) {
7236entry:
7237  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread-one-as") monotonic, align 4
7238  ret void
7239}
7240
7241define amdgpu_kernel void @global_singlethread_one_as_release_store(
7242; GFX6-LABEL: global_singlethread_one_as_release_store:
7243; GFX6:       ; %bb.0: ; %entry
7244; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
7245; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
7246; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7247; GFX6-NEXT:    s_mov_b32 s2, -1
7248; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7249; GFX6-NEXT:    v_mov_b32_e32 v0, s6
7250; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
7251; GFX6-NEXT:    s_endpgm
7252;
7253; GFX7-LABEL: global_singlethread_one_as_release_store:
7254; GFX7:       ; %bb.0: ; %entry
7255; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
7256; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
7257; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7258; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7259; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7260; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7261; GFX7-NEXT:    flat_store_dword v[0:1], v2
7262; GFX7-NEXT:    s_endpgm
7263;
7264; GFX10-WGP-LABEL: global_singlethread_one_as_release_store:
7265; GFX10-WGP:       ; %bb.0: ; %entry
7266; GFX10-WGP-NEXT:    s_clause 0x1
7267; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
7268; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7269; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
7270; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7271; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
7272; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
7273; GFX10-WGP-NEXT:    s_endpgm
7274;
7275; GFX10-CU-LABEL: global_singlethread_one_as_release_store:
7276; GFX10-CU:       ; %bb.0: ; %entry
7277; GFX10-CU-NEXT:    s_clause 0x1
7278; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
7279; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7280; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
7281; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7282; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
7283; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
7284; GFX10-CU-NEXT:    s_endpgm
7285;
7286; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_store:
7287; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7288; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x0
7289; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7290; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
7291; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
7292; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7293; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7294; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
7295; SKIP-CACHE-INV-NEXT:    s_endpgm
7296;
7297; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_store:
7298; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7299; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7300; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7301; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7302; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7303; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7304; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
7305; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7306;
7307; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_release_store:
7308; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7309; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7310; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7311; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7312; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7313; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7314; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
7315; GFX90A-TGSPLIT-NEXT:    s_endpgm
7316;
7317; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_store:
7318; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7319; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7320; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7321; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7322; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7323; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
7324; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
7325; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7326;
7327; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_release_store:
7328; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7329; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7330; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7331; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7332; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7333; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
7334; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
7335; GFX940-TGSPLIT-NEXT:    s_endpgm
7336;
7337; GFX11-WGP-LABEL: global_singlethread_one_as_release_store:
7338; GFX11-WGP:       ; %bb.0: ; %entry
7339; GFX11-WGP-NEXT:    s_clause 0x1
7340; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x0
7341; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7342; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7343; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
7344; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[0:1]
7345; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7346; GFX11-WGP-NEXT:    s_endpgm
7347;
7348; GFX11-CU-LABEL: global_singlethread_one_as_release_store:
7349; GFX11-CU:       ; %bb.0: ; %entry
7350; GFX11-CU-NEXT:    s_clause 0x1
7351; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x0
7352; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7353; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7354; GFX11-CU-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
7355; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
7356; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7357; GFX11-CU-NEXT:    s_endpgm
7358    i32 %in, i32 addrspace(1)* %out) {
7359entry:
7360  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread-one-as") release, align 4
7361  ret void
7362}
7363
7364define amdgpu_kernel void @global_singlethread_one_as_seq_cst_store(
7365; GFX6-LABEL: global_singlethread_one_as_seq_cst_store:
7366; GFX6:       ; %bb.0: ; %entry
7367; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
7368; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
7369; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7370; GFX6-NEXT:    s_mov_b32 s2, -1
7371; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7372; GFX6-NEXT:    v_mov_b32_e32 v0, s6
7373; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
7374; GFX6-NEXT:    s_endpgm
7375;
7376; GFX7-LABEL: global_singlethread_one_as_seq_cst_store:
7377; GFX7:       ; %bb.0: ; %entry
7378; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
7379; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
7380; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7381; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7382; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7383; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7384; GFX7-NEXT:    flat_store_dword v[0:1], v2
7385; GFX7-NEXT:    s_endpgm
7386;
7387; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_store:
7388; GFX10-WGP:       ; %bb.0: ; %entry
7389; GFX10-WGP-NEXT:    s_clause 0x1
7390; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
7391; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7392; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
7393; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7394; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
7395; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
7396; GFX10-WGP-NEXT:    s_endpgm
7397;
7398; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_store:
7399; GFX10-CU:       ; %bb.0: ; %entry
7400; GFX10-CU-NEXT:    s_clause 0x1
7401; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
7402; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7403; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
7404; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7405; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
7406; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
7407; GFX10-CU-NEXT:    s_endpgm
7408;
7409; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_store:
7410; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7411; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x0
7412; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7413; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
7414; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
7415; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7416; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7417; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
7418; SKIP-CACHE-INV-NEXT:    s_endpgm
7419;
7420; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_store:
7421; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7422; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7423; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7424; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7425; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7426; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7427; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
7428; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7429;
7430; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_store:
7431; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7432; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7433; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7434; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7435; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7436; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7437; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
7438; GFX90A-TGSPLIT-NEXT:    s_endpgm
7439;
7440; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_store:
7441; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7442; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7443; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7444; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7445; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7446; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
7447; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
7448; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7449;
7450; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_store:
7451; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7452; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7453; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7454; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7455; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7456; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
7457; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
7458; GFX940-TGSPLIT-NEXT:    s_endpgm
7459;
7460; GFX11-WGP-LABEL: global_singlethread_one_as_seq_cst_store:
7461; GFX11-WGP:       ; %bb.0: ; %entry
7462; GFX11-WGP-NEXT:    s_clause 0x1
7463; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x0
7464; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7465; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7466; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
7467; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[0:1]
7468; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7469; GFX11-WGP-NEXT:    s_endpgm
7470;
7471; GFX11-CU-LABEL: global_singlethread_one_as_seq_cst_store:
7472; GFX11-CU:       ; %bb.0: ; %entry
7473; GFX11-CU-NEXT:    s_clause 0x1
7474; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x0
7475; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7476; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7477; GFX11-CU-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
7478; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
7479; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7480; GFX11-CU-NEXT:    s_endpgm
7481    i32 %in, i32 addrspace(1)* %out) {
7482entry:
7483  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread-one-as") seq_cst, align 4
7484  ret void
7485}
7486
7487define amdgpu_kernel void @global_singlethread_one_as_monotonic_atomicrmw(
7488; GFX6-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
7489; GFX6:       ; %bb.0: ; %entry
7490; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
7491; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7492; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7493; GFX6-NEXT:    s_mov_b32 s2, -1
7494; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7495; GFX6-NEXT:    v_mov_b32_e32 v0, s6
7496; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
7497; GFX6-NEXT:    s_endpgm
7498;
7499; GFX7-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
7500; GFX7:       ; %bb.0: ; %entry
7501; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7502; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7503; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7504; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7505; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7506; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7507; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
7508; GFX7-NEXT:    s_endpgm
7509;
7510; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
7511; GFX10-WGP:       ; %bb.0: ; %entry
7512; GFX10-WGP-NEXT:    s_clause 0x1
7513; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7514; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7515; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
7516; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7517; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
7518; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
7519; GFX10-WGP-NEXT:    s_endpgm
7520;
7521; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
7522; GFX10-CU:       ; %bb.0: ; %entry
7523; GFX10-CU-NEXT:    s_clause 0x1
7524; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7525; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7526; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
7527; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7528; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
7529; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
7530; GFX10-CU-NEXT:    s_endpgm
7531;
7532; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
7533; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7534; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
7535; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7536; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
7537; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
7538; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7539; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7540; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
7541; SKIP-CACHE-INV-NEXT:    s_endpgm
7542;
7543; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
7544; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7545; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7546; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7547; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7548; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7549; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7550; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
7551; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7552;
7553; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
7554; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7555; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7556; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7557; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7558; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7559; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7560; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
7561; GFX90A-TGSPLIT-NEXT:    s_endpgm
7562;
7563; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
7564; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7565; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7566; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7567; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7568; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7569; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
7570; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
7571; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7572;
7573; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
7574; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7575; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7576; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7577; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7578; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7579; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
7580; GFX940-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
7581; GFX940-TGSPLIT-NEXT:    s_endpgm
7582;
7583; GFX11-WGP-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
7584; GFX11-WGP:       ; %bb.0: ; %entry
7585; GFX11-WGP-NEXT:    s_clause 0x1
7586; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x8
7587; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7588; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7589; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
7590; GFX11-WGP-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
7591; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7592; GFX11-WGP-NEXT:    s_endpgm
7593;
7594; GFX11-CU-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
7595; GFX11-CU:       ; %bb.0: ; %entry
7596; GFX11-CU-NEXT:    s_clause 0x1
7597; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x8
7598; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7599; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7600; GFX11-CU-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
7601; GFX11-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
7602; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7603; GFX11-CU-NEXT:    s_endpgm
7604    i32 addrspace(1)* %out, i32 %in) {
7605entry:
7606  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") monotonic
7607  ret void
7608}
7609
7610define amdgpu_kernel void @global_singlethread_one_as_acquire_atomicrmw(
7611; GFX6-LABEL: global_singlethread_one_as_acquire_atomicrmw:
7612; GFX6:       ; %bb.0: ; %entry
7613; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
7614; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7615; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7616; GFX6-NEXT:    s_mov_b32 s2, -1
7617; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7618; GFX6-NEXT:    v_mov_b32_e32 v0, s6
7619; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
7620; GFX6-NEXT:    s_endpgm
7621;
7622; GFX7-LABEL: global_singlethread_one_as_acquire_atomicrmw:
7623; GFX7:       ; %bb.0: ; %entry
7624; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7625; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7626; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7627; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7628; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7629; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7630; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
7631; GFX7-NEXT:    s_endpgm
7632;
7633; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_atomicrmw:
7634; GFX10-WGP:       ; %bb.0: ; %entry
7635; GFX10-WGP-NEXT:    s_clause 0x1
7636; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7637; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7638; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
7639; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7640; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
7641; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
7642; GFX10-WGP-NEXT:    s_endpgm
7643;
7644; GFX10-CU-LABEL: global_singlethread_one_as_acquire_atomicrmw:
7645; GFX10-CU:       ; %bb.0: ; %entry
7646; GFX10-CU-NEXT:    s_clause 0x1
7647; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7648; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7649; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
7650; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7651; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
7652; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
7653; GFX10-CU-NEXT:    s_endpgm
7654;
7655; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_atomicrmw:
7656; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7657; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
7658; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7659; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
7660; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
7661; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7662; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7663; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
7664; SKIP-CACHE-INV-NEXT:    s_endpgm
7665;
7666; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_atomicrmw:
7667; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7668; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7669; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7670; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7671; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7672; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7673; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
7674; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7675;
7676; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_atomicrmw:
7677; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7678; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7679; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7680; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7681; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7682; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7683; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
7684; GFX90A-TGSPLIT-NEXT:    s_endpgm
7685;
7686; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_atomicrmw:
7687; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7688; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7689; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7690; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7691; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7692; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
7693; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
7694; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7695;
7696; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acquire_atomicrmw:
7697; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7698; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7699; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7700; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7701; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7702; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
7703; GFX940-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
7704; GFX940-TGSPLIT-NEXT:    s_endpgm
7705;
7706; GFX11-WGP-LABEL: global_singlethread_one_as_acquire_atomicrmw:
7707; GFX11-WGP:       ; %bb.0: ; %entry
7708; GFX11-WGP-NEXT:    s_clause 0x1
7709; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x8
7710; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7711; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7712; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
7713; GFX11-WGP-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
7714; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7715; GFX11-WGP-NEXT:    s_endpgm
7716;
7717; GFX11-CU-LABEL: global_singlethread_one_as_acquire_atomicrmw:
7718; GFX11-CU:       ; %bb.0: ; %entry
7719; GFX11-CU-NEXT:    s_clause 0x1
7720; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x8
7721; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7722; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7723; GFX11-CU-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
7724; GFX11-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
7725; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7726; GFX11-CU-NEXT:    s_endpgm
7727    i32 addrspace(1)* %out, i32 %in) {
7728entry:
7729  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") acquire
7730  ret void
7731}
7732
7733define amdgpu_kernel void @global_singlethread_one_as_release_atomicrmw(
7734; GFX6-LABEL: global_singlethread_one_as_release_atomicrmw:
7735; GFX6:       ; %bb.0: ; %entry
7736; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
7737; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7738; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7739; GFX6-NEXT:    s_mov_b32 s2, -1
7740; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7741; GFX6-NEXT:    v_mov_b32_e32 v0, s6
7742; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
7743; GFX6-NEXT:    s_endpgm
7744;
7745; GFX7-LABEL: global_singlethread_one_as_release_atomicrmw:
7746; GFX7:       ; %bb.0: ; %entry
7747; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7748; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7749; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7750; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7751; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7752; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7753; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
7754; GFX7-NEXT:    s_endpgm
7755;
7756; GFX10-WGP-LABEL: global_singlethread_one_as_release_atomicrmw:
7757; GFX10-WGP:       ; %bb.0: ; %entry
7758; GFX10-WGP-NEXT:    s_clause 0x1
7759; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7760; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7761; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
7762; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7763; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
7764; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
7765; GFX10-WGP-NEXT:    s_endpgm
7766;
7767; GFX10-CU-LABEL: global_singlethread_one_as_release_atomicrmw:
7768; GFX10-CU:       ; %bb.0: ; %entry
7769; GFX10-CU-NEXT:    s_clause 0x1
7770; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7771; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7772; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
7773; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7774; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
7775; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
7776; GFX10-CU-NEXT:    s_endpgm
7777;
7778; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_atomicrmw:
7779; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7780; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
7781; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7782; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
7783; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
7784; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7785; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7786; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
7787; SKIP-CACHE-INV-NEXT:    s_endpgm
7788;
7789; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_atomicrmw:
7790; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7791; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7792; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7793; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7794; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7795; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7796; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
7797; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7798;
7799; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_release_atomicrmw:
7800; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7801; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7802; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7803; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7804; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7805; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7806; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
7807; GFX90A-TGSPLIT-NEXT:    s_endpgm
7808;
7809; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_atomicrmw:
7810; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7811; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7812; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7813; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7814; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7815; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
7816; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
7817; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7818;
7819; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_release_atomicrmw:
7820; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7821; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7822; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7823; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7824; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7825; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
7826; GFX940-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
7827; GFX940-TGSPLIT-NEXT:    s_endpgm
7828;
7829; GFX11-WGP-LABEL: global_singlethread_one_as_release_atomicrmw:
7830; GFX11-WGP:       ; %bb.0: ; %entry
7831; GFX11-WGP-NEXT:    s_clause 0x1
7832; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x8
7833; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7834; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7835; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
7836; GFX11-WGP-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
7837; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7838; GFX11-WGP-NEXT:    s_endpgm
7839;
7840; GFX11-CU-LABEL: global_singlethread_one_as_release_atomicrmw:
7841; GFX11-CU:       ; %bb.0: ; %entry
7842; GFX11-CU-NEXT:    s_clause 0x1
7843; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x8
7844; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7845; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7846; GFX11-CU-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
7847; GFX11-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
7848; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7849; GFX11-CU-NEXT:    s_endpgm
7850    i32 addrspace(1)* %out, i32 %in) {
7851entry:
7852  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") release
7853  ret void
7854}
7855
7856define amdgpu_kernel void @global_singlethread_one_as_acq_rel_atomicrmw(
7857; GFX6-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
7858; GFX6:       ; %bb.0: ; %entry
7859; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
7860; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7861; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7862; GFX6-NEXT:    s_mov_b32 s2, -1
7863; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7864; GFX6-NEXT:    v_mov_b32_e32 v0, s6
7865; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
7866; GFX6-NEXT:    s_endpgm
7867;
7868; GFX7-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
7869; GFX7:       ; %bb.0: ; %entry
7870; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7871; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7872; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7873; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7874; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7875; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7876; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
7877; GFX7-NEXT:    s_endpgm
7878;
7879; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
7880; GFX10-WGP:       ; %bb.0: ; %entry
7881; GFX10-WGP-NEXT:    s_clause 0x1
7882; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7883; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7884; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
7885; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7886; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
7887; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
7888; GFX10-WGP-NEXT:    s_endpgm
7889;
7890; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
7891; GFX10-CU:       ; %bb.0: ; %entry
7892; GFX10-CU-NEXT:    s_clause 0x1
7893; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7894; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7895; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
7896; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7897; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
7898; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
7899; GFX10-CU-NEXT:    s_endpgm
7900;
7901; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
7902; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7903; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
7904; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7905; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
7906; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
7907; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7908; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7909; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
7910; SKIP-CACHE-INV-NEXT:    s_endpgm
7911;
7912; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
7913; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7914; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7915; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7916; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7917; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7918; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7919; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
7920; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7921;
7922; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
7923; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7924; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7925; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7926; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7927; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7928; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7929; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
7930; GFX90A-TGSPLIT-NEXT:    s_endpgm
7931;
7932; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
7933; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7934; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7935; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7936; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7937; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7938; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
7939; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
7940; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7941;
7942; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
7943; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7944; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7945; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7946; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7947; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7948; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
7949; GFX940-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
7950; GFX940-TGSPLIT-NEXT:    s_endpgm
7951;
7952; GFX11-WGP-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
7953; GFX11-WGP:       ; %bb.0: ; %entry
7954; GFX11-WGP-NEXT:    s_clause 0x1
7955; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x8
7956; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7957; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7958; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
7959; GFX11-WGP-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
7960; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7961; GFX11-WGP-NEXT:    s_endpgm
7962;
7963; GFX11-CU-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
7964; GFX11-CU:       ; %bb.0: ; %entry
7965; GFX11-CU-NEXT:    s_clause 0x1
7966; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x8
7967; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7968; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7969; GFX11-CU-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
7970; GFX11-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
7971; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7972; GFX11-CU-NEXT:    s_endpgm
7973    i32 addrspace(1)* %out, i32 %in) {
7974entry:
7975  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") acq_rel
7976  ret void
7977}
7978
7979define amdgpu_kernel void @global_singlethread_one_as_seq_cst_atomicrmw(
7980; GFX6-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
7981; GFX6:       ; %bb.0: ; %entry
7982; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
7983; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7984; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7985; GFX6-NEXT:    s_mov_b32 s2, -1
7986; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7987; GFX6-NEXT:    v_mov_b32_e32 v0, s6
7988; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
7989; GFX6-NEXT:    s_endpgm
7990;
7991; GFX7-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
7992; GFX7:       ; %bb.0: ; %entry
7993; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7994; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7995; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7996; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7997; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7998; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7999; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
8000; GFX7-NEXT:    s_endpgm
8001;
8002; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
8003; GFX10-WGP:       ; %bb.0: ; %entry
8004; GFX10-WGP-NEXT:    s_clause 0x1
8005; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
8006; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8007; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
8008; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8009; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
8010; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
8011; GFX10-WGP-NEXT:    s_endpgm
8012;
8013; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
8014; GFX10-CU:       ; %bb.0: ; %entry
8015; GFX10-CU-NEXT:    s_clause 0x1
8016; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8017; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8018; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
8019; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8020; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
8021; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
8022; GFX10-CU-NEXT:    s_endpgm
8023;
8024; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
8025; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8026; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
8027; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
8028; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
8029; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
8030; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8031; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8032; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
8033; SKIP-CACHE-INV-NEXT:    s_endpgm
8034;
8035; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
8036; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8037; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8038; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8039; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
8040; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8041; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8042; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
8043; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8044;
8045; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
8046; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8047; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8048; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8049; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
8050; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8051; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8052; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
8053; GFX90A-TGSPLIT-NEXT:    s_endpgm
8054;
8055; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
8056; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8057; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8058; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8059; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
8060; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8061; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
8062; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
8063; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8064;
8065; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
8066; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8067; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8068; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8069; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
8070; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8071; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
8072; GFX940-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
8073; GFX940-TGSPLIT-NEXT:    s_endpgm
8074;
8075; GFX11-WGP-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
8076; GFX11-WGP:       ; %bb.0: ; %entry
8077; GFX11-WGP-NEXT:    s_clause 0x1
8078; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x8
8079; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
8080; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8081; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
8082; GFX11-WGP-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
8083; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8084; GFX11-WGP-NEXT:    s_endpgm
8085;
8086; GFX11-CU-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
8087; GFX11-CU:       ; %bb.0: ; %entry
8088; GFX11-CU-NEXT:    s_clause 0x1
8089; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x8
8090; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
8091; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8092; GFX11-CU-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
8093; GFX11-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
8094; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8095; GFX11-CU-NEXT:    s_endpgm
8096    i32 addrspace(1)* %out, i32 %in) {
8097entry:
8098  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") seq_cst
8099  ret void
8100}
8101
8102define amdgpu_kernel void @global_singlethread_one_as_acquire_ret_atomicrmw(
8103; GFX6-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
8104; GFX6:       ; %bb.0: ; %entry
8105; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
8106; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8107; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
8108; GFX6-NEXT:    s_mov_b32 s2, -1
8109; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8110; GFX6-NEXT:    v_mov_b32_e32 v0, s6
8111; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
8112; GFX6-NEXT:    s_waitcnt vmcnt(0)
8113; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
8114; GFX6-NEXT:    s_endpgm
8115;
8116; GFX7-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
8117; GFX7:       ; %bb.0: ; %entry
8118; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8119; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
8120; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8121; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8122; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8123; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8124; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8125; GFX7-NEXT:    s_waitcnt vmcnt(0)
8126; GFX7-NEXT:    flat_store_dword v[0:1], v2
8127; GFX7-NEXT:    s_endpgm
8128;
8129; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
8130; GFX10-WGP:       ; %bb.0: ; %entry
8131; GFX10-WGP-NEXT:    s_clause 0x1
8132; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
8133; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8134; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
8135; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8136; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
8137; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
8138; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8139; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
8140; GFX10-WGP-NEXT:    s_endpgm
8141;
8142; GFX10-CU-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
8143; GFX10-CU:       ; %bb.0: ; %entry
8144; GFX10-CU-NEXT:    s_clause 0x1
8145; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8146; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8147; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
8148; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8149; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
8150; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
8151; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8152; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
8153; GFX10-CU-NEXT:    s_endpgm
8154;
8155; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
8156; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8157; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
8158; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
8159; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
8160; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
8161; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8162; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8163; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
8164; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8165; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
8166; SKIP-CACHE-INV-NEXT:    s_endpgm
8167;
8168; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
8169; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8170; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8171; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8172; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
8173; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8174; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8175; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
8176; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8177; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
8178; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8179;
8180; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
8181; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8182; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8183; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8184; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
8185; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8186; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8187; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
8188; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8189; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
8190; GFX90A-TGSPLIT-NEXT:    s_endpgm
8191;
8192; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
8193; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8194; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8195; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8196; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
8197; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8198; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
8199; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[2:3] sc0
8200; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8201; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
8202; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8203;
8204; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
8205; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8206; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8207; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8208; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
8209; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8210; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
8211; GFX940-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[2:3] sc0
8212; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8213; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
8214; GFX940-TGSPLIT-NEXT:    s_endpgm
8215;
8216; GFX11-WGP-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
8217; GFX11-WGP:       ; %bb.0: ; %entry
8218; GFX11-WGP-NEXT:    s_clause 0x1
8219; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x8
8220; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
8221; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8222; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
8223; GFX11-WGP-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
8224; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8225; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[0:1]
8226; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8227; GFX11-WGP-NEXT:    s_endpgm
8228;
8229; GFX11-CU-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
8230; GFX11-CU:       ; %bb.0: ; %entry
8231; GFX11-CU-NEXT:    s_clause 0x1
8232; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x8
8233; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
8234; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8235; GFX11-CU-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
8236; GFX11-CU-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
8237; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
8238; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
8239; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8240; GFX11-CU-NEXT:    s_endpgm
8241    i32 addrspace(1)* %out, i32 %in) {
8242entry:
8243  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") acquire
8244  store i32 %val, i32 addrspace(1)* %out, align 4
8245  ret void
8246}
8247
8248define amdgpu_kernel void @global_singlethread_one_as_acq_rel_ret_atomicrmw(
8249; GFX6-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
8250; GFX6:       ; %bb.0: ; %entry
8251; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
8252; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8253; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
8254; GFX6-NEXT:    s_mov_b32 s2, -1
8255; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8256; GFX6-NEXT:    v_mov_b32_e32 v0, s6
8257; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
8258; GFX6-NEXT:    s_waitcnt vmcnt(0)
8259; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
8260; GFX6-NEXT:    s_endpgm
8261;
8262; GFX7-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
8263; GFX7:       ; %bb.0: ; %entry
8264; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8265; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
8266; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8267; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8268; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8269; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8270; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8271; GFX7-NEXT:    s_waitcnt vmcnt(0)
8272; GFX7-NEXT:    flat_store_dword v[0:1], v2
8273; GFX7-NEXT:    s_endpgm
8274;
8275; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
8276; GFX10-WGP:       ; %bb.0: ; %entry
8277; GFX10-WGP-NEXT:    s_clause 0x1
8278; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
8279; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8280; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
8281; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8282; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
8283; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
8284; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8285; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
8286; GFX10-WGP-NEXT:    s_endpgm
8287;
8288; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
8289; GFX10-CU:       ; %bb.0: ; %entry
8290; GFX10-CU-NEXT:    s_clause 0x1
8291; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8292; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8293; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
8294; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8295; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
8296; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
8297; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8298; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
8299; GFX10-CU-NEXT:    s_endpgm
8300;
8301; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
8302; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8303; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
8304; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
8305; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
8306; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
8307; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8308; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8309; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
8310; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8311; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
8312; SKIP-CACHE-INV-NEXT:    s_endpgm
8313;
8314; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
8315; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8316; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8317; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8318; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
8319; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8320; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8321; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
8322; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8323; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
8324; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8325;
8326; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
8327; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8328; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8329; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8330; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
8331; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8332; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8333; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
8334; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8335; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
8336; GFX90A-TGSPLIT-NEXT:    s_endpgm
8337;
8338; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
8339; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8340; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8341; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8342; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
8343; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8344; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
8345; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[2:3] sc0
8346; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8347; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
8348; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8349;
8350; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
8351; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8352; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8353; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8354; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
8355; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8356; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
8357; GFX940-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[2:3] sc0
8358; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8359; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
8360; GFX940-TGSPLIT-NEXT:    s_endpgm
8361;
8362; GFX11-WGP-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
8363; GFX11-WGP:       ; %bb.0: ; %entry
8364; GFX11-WGP-NEXT:    s_clause 0x1
8365; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x8
8366; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
8367; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8368; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
8369; GFX11-WGP-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
8370; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8371; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[0:1]
8372; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8373; GFX11-WGP-NEXT:    s_endpgm
8374;
8375; GFX11-CU-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
8376; GFX11-CU:       ; %bb.0: ; %entry
8377; GFX11-CU-NEXT:    s_clause 0x1
8378; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x8
8379; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
8380; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8381; GFX11-CU-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
8382; GFX11-CU-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
8383; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
8384; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
8385; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8386; GFX11-CU-NEXT:    s_endpgm
8387    i32 addrspace(1)* %out, i32 %in) {
8388entry:
8389  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") acq_rel
8390  store i32 %val, i32 addrspace(1)* %out, align 4
8391  ret void
8392}
8393
8394define amdgpu_kernel void @global_singlethread_one_as_seq_cst_ret_atomicrmw(
8395; GFX6-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
8396; GFX6:       ; %bb.0: ; %entry
8397; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
8398; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8399; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
8400; GFX6-NEXT:    s_mov_b32 s2, -1
8401; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8402; GFX6-NEXT:    v_mov_b32_e32 v0, s6
8403; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
8404; GFX6-NEXT:    s_waitcnt vmcnt(0)
8405; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
8406; GFX6-NEXT:    s_endpgm
8407;
8408; GFX7-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
8409; GFX7:       ; %bb.0: ; %entry
8410; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8411; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
8412; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8413; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8414; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8415; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8416; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8417; GFX7-NEXT:    s_waitcnt vmcnt(0)
8418; GFX7-NEXT:    flat_store_dword v[0:1], v2
8419; GFX7-NEXT:    s_endpgm
8420;
8421; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
8422; GFX10-WGP:       ; %bb.0: ; %entry
8423; GFX10-WGP-NEXT:    s_clause 0x1
8424; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
8425; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8426; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
8427; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8428; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
8429; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
8430; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8431; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
8432; GFX10-WGP-NEXT:    s_endpgm
8433;
8434; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
8435; GFX10-CU:       ; %bb.0: ; %entry
8436; GFX10-CU-NEXT:    s_clause 0x1
8437; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8438; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8439; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
8440; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8441; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
8442; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
8443; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8444; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
8445; GFX10-CU-NEXT:    s_endpgm
8446;
8447; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
8448; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8449; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
8450; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
8451; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
8452; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
8453; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8454; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8455; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
8456; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8457; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
8458; SKIP-CACHE-INV-NEXT:    s_endpgm
8459;
8460; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
8461; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8462; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8463; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8464; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
8465; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8466; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8467; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
8468; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8469; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
8470; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8471;
8472; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
8473; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8474; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8475; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8476; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
8477; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8478; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8479; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
8480; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8481; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
8482; GFX90A-TGSPLIT-NEXT:    s_endpgm
8483;
8484; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
8485; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8486; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8487; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8488; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
8489; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8490; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
8491; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[2:3] sc0
8492; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8493; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
8494; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8495;
8496; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
8497; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8498; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8499; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8500; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
8501; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8502; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
8503; GFX940-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[2:3] sc0
8504; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8505; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
8506; GFX940-TGSPLIT-NEXT:    s_endpgm
8507;
8508; GFX11-WGP-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
8509; GFX11-WGP:       ; %bb.0: ; %entry
8510; GFX11-WGP-NEXT:    s_clause 0x1
8511; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x8
8512; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
8513; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8514; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
8515; GFX11-WGP-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
8516; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8517; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[0:1]
8518; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8519; GFX11-WGP-NEXT:    s_endpgm
8520;
8521; GFX11-CU-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
8522; GFX11-CU:       ; %bb.0: ; %entry
8523; GFX11-CU-NEXT:    s_clause 0x1
8524; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x8
8525; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
8526; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8527; GFX11-CU-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
8528; GFX11-CU-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
8529; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
8530; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
8531; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8532; GFX11-CU-NEXT:    s_endpgm
8533    i32 addrspace(1)* %out, i32 %in) {
8534entry:
8535  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") seq_cst
8536  store i32 %val, i32 addrspace(1)* %out, align 4
8537  ret void
8538}
8539
8540define amdgpu_kernel void @global_singlethread_one_as_monotonic_monotonic_cmpxchg(
8541; GFX6-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
8542; GFX6:       ; %bb.0: ; %entry
8543; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
8544; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8545; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
8546; GFX6-NEXT:    s_mov_b32 s2, -1
8547; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8548; GFX6-NEXT:    v_mov_b32_e32 v0, s6
8549; GFX6-NEXT:    v_mov_b32_e32 v1, s7
8550; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
8551; GFX6-NEXT:    s_endpgm
8552;
8553; GFX7-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
8554; GFX7:       ; %bb.0: ; %entry
8555; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8556; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8557; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8558; GFX7-NEXT:    s_add_u32 s0, s0, 16
8559; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8560; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8561; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8562; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8563; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8564; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8565; GFX7-NEXT:    s_endpgm
8566;
8567; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
8568; GFX10-WGP:       ; %bb.0: ; %entry
8569; GFX10-WGP-NEXT:    s_clause 0x1
8570; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8571; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8572; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
8573; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8574; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8575; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8576; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8577; GFX10-WGP-NEXT:    s_endpgm
8578;
8579; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
8580; GFX10-CU:       ; %bb.0: ; %entry
8581; GFX10-CU-NEXT:    s_clause 0x1
8582; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8583; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8584; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
8585; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8586; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8587; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8588; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8589; GFX10-CU-NEXT:    s_endpgm
8590;
8591; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
8592; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8593; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
8594; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
8595; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
8596; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
8597; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8598; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8599; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8600; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
8601; SKIP-CACHE-INV-NEXT:    s_endpgm
8602;
8603; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
8604; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8605; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8606; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8607; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8608; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8609; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8610; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8611; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8612;
8613; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
8614; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8615; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8616; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8617; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8618; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8619; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8620; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8621; GFX90A-TGSPLIT-NEXT:    s_endpgm
8622;
8623; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
8624; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8625; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8626; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
8627; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8628; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8629; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8630; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
8631; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8632;
8633; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
8634; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8635; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8636; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
8637; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8638; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8639; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8640; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
8641; GFX940-TGSPLIT-NEXT:    s_endpgm
8642;
8643; GFX11-WGP-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
8644; GFX11-WGP:       ; %bb.0: ; %entry
8645; GFX11-WGP-NEXT:    s_clause 0x1
8646; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8647; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
8648; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
8649; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8650; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8651; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
8652; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8653; GFX11-WGP-NEXT:    s_endpgm
8654;
8655; GFX11-CU-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
8656; GFX11-CU:       ; %bb.0: ; %entry
8657; GFX11-CU-NEXT:    s_clause 0x1
8658; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8659; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
8660; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
8661; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8662; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8663; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
8664; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8665; GFX11-CU-NEXT:    s_endpgm
8666    i32 addrspace(1)* %out, i32 %in, i32 %old) {
8667entry:
8668  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
8669  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic monotonic
8670  ret void
8671}
8672
8673define amdgpu_kernel void @global_singlethread_one_as_acquire_monotonic_cmpxchg(
8674; GFX6-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
8675; GFX6:       ; %bb.0: ; %entry
8676; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
8677; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8678; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
8679; GFX6-NEXT:    s_mov_b32 s2, -1
8680; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8681; GFX6-NEXT:    v_mov_b32_e32 v0, s6
8682; GFX6-NEXT:    v_mov_b32_e32 v1, s7
8683; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
8684; GFX6-NEXT:    s_endpgm
8685;
8686; GFX7-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
8687; GFX7:       ; %bb.0: ; %entry
8688; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8689; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8690; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8691; GFX7-NEXT:    s_add_u32 s0, s0, 16
8692; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8693; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8694; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8695; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8696; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8697; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8698; GFX7-NEXT:    s_endpgm
8699;
8700; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
8701; GFX10-WGP:       ; %bb.0: ; %entry
8702; GFX10-WGP-NEXT:    s_clause 0x1
8703; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8704; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8705; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
8706; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8707; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8708; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8709; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8710; GFX10-WGP-NEXT:    s_endpgm
8711;
8712; GFX10-CU-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
8713; GFX10-CU:       ; %bb.0: ; %entry
8714; GFX10-CU-NEXT:    s_clause 0x1
8715; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8716; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8717; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
8718; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8719; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8720; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8721; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8722; GFX10-CU-NEXT:    s_endpgm
8723;
8724; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
8725; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8726; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
8727; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
8728; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
8729; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
8730; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8731; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8732; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8733; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
8734; SKIP-CACHE-INV-NEXT:    s_endpgm
8735;
8736; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
8737; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8738; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8739; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8740; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8741; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8742; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8743; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8744; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8745;
8746; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
8747; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8748; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8749; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8750; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8751; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8752; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8753; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8754; GFX90A-TGSPLIT-NEXT:    s_endpgm
8755;
8756; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
8757; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8758; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8759; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
8760; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8761; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8762; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8763; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
8764; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8765;
8766; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
8767; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8768; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8769; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
8770; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8771; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8772; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8773; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
8774; GFX940-TGSPLIT-NEXT:    s_endpgm
8775;
8776; GFX11-WGP-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
8777; GFX11-WGP:       ; %bb.0: ; %entry
8778; GFX11-WGP-NEXT:    s_clause 0x1
8779; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8780; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
8781; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
8782; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8783; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8784; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
8785; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8786; GFX11-WGP-NEXT:    s_endpgm
8787;
8788; GFX11-CU-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
8789; GFX11-CU:       ; %bb.0: ; %entry
8790; GFX11-CU-NEXT:    s_clause 0x1
8791; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8792; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
8793; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
8794; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8795; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8796; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
8797; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8798; GFX11-CU-NEXT:    s_endpgm
8799    i32 addrspace(1)* %out, i32 %in, i32 %old) {
8800entry:
8801  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
8802  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire monotonic
8803  ret void
8804}
8805
8806define amdgpu_kernel void @global_singlethread_one_as_release_monotonic_cmpxchg(
8807; GFX6-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
8808; GFX6:       ; %bb.0: ; %entry
8809; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
8810; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8811; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
8812; GFX6-NEXT:    s_mov_b32 s2, -1
8813; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8814; GFX6-NEXT:    v_mov_b32_e32 v0, s6
8815; GFX6-NEXT:    v_mov_b32_e32 v1, s7
8816; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
8817; GFX6-NEXT:    s_endpgm
8818;
8819; GFX7-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
8820; GFX7:       ; %bb.0: ; %entry
8821; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8822; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8823; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8824; GFX7-NEXT:    s_add_u32 s0, s0, 16
8825; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8826; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8827; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8828; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8829; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8830; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8831; GFX7-NEXT:    s_endpgm
8832;
8833; GFX10-WGP-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
8834; GFX10-WGP:       ; %bb.0: ; %entry
8835; GFX10-WGP-NEXT:    s_clause 0x1
8836; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8837; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8838; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
8839; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8840; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8841; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8842; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8843; GFX10-WGP-NEXT:    s_endpgm
8844;
8845; GFX10-CU-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
8846; GFX10-CU:       ; %bb.0: ; %entry
8847; GFX10-CU-NEXT:    s_clause 0x1
8848; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8849; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8850; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
8851; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8852; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8853; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8854; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8855; GFX10-CU-NEXT:    s_endpgm
8856;
8857; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
8858; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8859; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
8860; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
8861; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
8862; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
8863; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8864; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8865; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8866; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
8867; SKIP-CACHE-INV-NEXT:    s_endpgm
8868;
8869; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
8870; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8871; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8872; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8873; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8874; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8875; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8876; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8877; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8878;
8879; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
8880; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8881; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8882; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8883; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8884; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8885; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8886; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8887; GFX90A-TGSPLIT-NEXT:    s_endpgm
8888;
8889; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
8890; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8891; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8892; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
8893; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8894; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8895; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8896; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
8897; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8898;
8899; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
8900; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8901; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8902; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
8903; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8904; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8905; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8906; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
8907; GFX940-TGSPLIT-NEXT:    s_endpgm
8908;
8909; GFX11-WGP-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
8910; GFX11-WGP:       ; %bb.0: ; %entry
8911; GFX11-WGP-NEXT:    s_clause 0x1
8912; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8913; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
8914; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
8915; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8916; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8917; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
8918; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8919; GFX11-WGP-NEXT:    s_endpgm
8920;
8921; GFX11-CU-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
8922; GFX11-CU:       ; %bb.0: ; %entry
8923; GFX11-CU-NEXT:    s_clause 0x1
8924; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8925; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
8926; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
8927; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8928; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8929; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
8930; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8931; GFX11-CU-NEXT:    s_endpgm
8932    i32 addrspace(1)* %out, i32 %in, i32 %old) {
8933entry:
8934  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
8935  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release monotonic
8936  ret void
8937}
8938
8939define amdgpu_kernel void @global_singlethread_one_as_acq_rel_monotonic_cmpxchg(
8940; GFX6-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8941; GFX6:       ; %bb.0: ; %entry
8942; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
8943; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8944; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
8945; GFX6-NEXT:    s_mov_b32 s2, -1
8946; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8947; GFX6-NEXT:    v_mov_b32_e32 v0, s6
8948; GFX6-NEXT:    v_mov_b32_e32 v1, s7
8949; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
8950; GFX6-NEXT:    s_endpgm
8951;
8952; GFX7-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8953; GFX7:       ; %bb.0: ; %entry
8954; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8955; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8956; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8957; GFX7-NEXT:    s_add_u32 s0, s0, 16
8958; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8959; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8960; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8961; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8962; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8963; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8964; GFX7-NEXT:    s_endpgm
8965;
8966; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8967; GFX10-WGP:       ; %bb.0: ; %entry
8968; GFX10-WGP-NEXT:    s_clause 0x1
8969; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8970; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8971; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
8972; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8973; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8974; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8975; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8976; GFX10-WGP-NEXT:    s_endpgm
8977;
8978; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8979; GFX10-CU:       ; %bb.0: ; %entry
8980; GFX10-CU-NEXT:    s_clause 0x1
8981; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8982; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8983; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
8984; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8985; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8986; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8987; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8988; GFX10-CU-NEXT:    s_endpgm
8989;
8990; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8991; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8992; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
8993; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
8994; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
8995; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
8996; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8997; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8998; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8999; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
9000; SKIP-CACHE-INV-NEXT:    s_endpgm
9001;
9002; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
9003; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9004; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9005; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9006; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9007; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9008; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9009; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9010; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9011;
9012; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
9013; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9014; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9015; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9016; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9017; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9018; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9019; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9020; GFX90A-TGSPLIT-NEXT:    s_endpgm
9021;
9022; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
9023; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9024; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9025; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9026; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9027; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9028; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9029; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
9030; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9031;
9032; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
9033; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9034; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9035; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9036; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9037; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9038; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9039; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
9040; GFX940-TGSPLIT-NEXT:    s_endpgm
9041;
9042; GFX11-WGP-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
9043; GFX11-WGP:       ; %bb.0: ; %entry
9044; GFX11-WGP-NEXT:    s_clause 0x1
9045; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
9046; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
9047; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
9048; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9049; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9050; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
9051; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9052; GFX11-WGP-NEXT:    s_endpgm
9053;
9054; GFX11-CU-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
9055; GFX11-CU:       ; %bb.0: ; %entry
9056; GFX11-CU-NEXT:    s_clause 0x1
9057; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
9058; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
9059; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
9060; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9061; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9062; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
9063; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9064; GFX11-CU-NEXT:    s_endpgm
9065    i32 addrspace(1)* %out, i32 %in, i32 %old) {
9066entry:
9067  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
9068  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel monotonic
9069  ret void
9070}
9071
9072define amdgpu_kernel void @global_singlethread_one_as_seq_cst_monotonic_cmpxchg(
9073; GFX6-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
9074; GFX6:       ; %bb.0: ; %entry
9075; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
9076; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9077; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
9078; GFX6-NEXT:    s_mov_b32 s2, -1
9079; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9080; GFX6-NEXT:    v_mov_b32_e32 v0, s6
9081; GFX6-NEXT:    v_mov_b32_e32 v1, s7
9082; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
9083; GFX6-NEXT:    s_endpgm
9084;
9085; GFX7-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
9086; GFX7:       ; %bb.0: ; %entry
9087; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9088; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9089; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9090; GFX7-NEXT:    s_add_u32 s0, s0, 16
9091; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9092; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9093; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9094; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9095; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9096; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9097; GFX7-NEXT:    s_endpgm
9098;
9099; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
9100; GFX10-WGP:       ; %bb.0: ; %entry
9101; GFX10-WGP-NEXT:    s_clause 0x1
9102; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9103; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9104; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
9105; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9106; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9107; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9108; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9109; GFX10-WGP-NEXT:    s_endpgm
9110;
9111; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
9112; GFX10-CU:       ; %bb.0: ; %entry
9113; GFX10-CU-NEXT:    s_clause 0x1
9114; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9115; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9116; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
9117; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9118; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9119; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9120; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9121; GFX10-CU-NEXT:    s_endpgm
9122;
9123; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
9124; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9125; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
9126; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
9127; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
9128; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
9129; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9130; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
9131; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
9132; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
9133; SKIP-CACHE-INV-NEXT:    s_endpgm
9134;
9135; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
9136; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9137; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9138; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9139; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9140; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9141; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9142; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9143; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9144;
9145; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
9146; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9147; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9148; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9149; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9150; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9151; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9152; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9153; GFX90A-TGSPLIT-NEXT:    s_endpgm
9154;
9155; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
9156; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9157; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9158; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9159; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9160; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9161; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9162; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
9163; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9164;
9165; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
9166; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9167; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9168; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9169; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9170; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9171; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9172; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
9173; GFX940-TGSPLIT-NEXT:    s_endpgm
9174;
9175; GFX11-WGP-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
9176; GFX11-WGP:       ; %bb.0: ; %entry
9177; GFX11-WGP-NEXT:    s_clause 0x1
9178; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
9179; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
9180; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
9181; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9182; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9183; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
9184; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9185; GFX11-WGP-NEXT:    s_endpgm
9186;
9187; GFX11-CU-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
9188; GFX11-CU:       ; %bb.0: ; %entry
9189; GFX11-CU-NEXT:    s_clause 0x1
9190; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
9191; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
9192; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
9193; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9194; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9195; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
9196; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9197; GFX11-CU-NEXT:    s_endpgm
9198    i32 addrspace(1)* %out, i32 %in, i32 %old) {
9199entry:
9200  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
9201  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst monotonic
9202  ret void
9203}
9204
9205define amdgpu_kernel void @global_singlethread_one_as_monotonic_acquire_cmpxchg(
9206; GFX6-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
9207; GFX6:       ; %bb.0: ; %entry
9208; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
9209; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9210; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
9211; GFX6-NEXT:    s_mov_b32 s2, -1
9212; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9213; GFX6-NEXT:    v_mov_b32_e32 v0, s6
9214; GFX6-NEXT:    v_mov_b32_e32 v1, s7
9215; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
9216; GFX6-NEXT:    s_endpgm
9217;
9218; GFX7-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
9219; GFX7:       ; %bb.0: ; %entry
9220; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9221; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9222; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9223; GFX7-NEXT:    s_add_u32 s0, s0, 16
9224; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9225; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9226; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9227; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9228; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9229; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9230; GFX7-NEXT:    s_endpgm
9231;
9232; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
9233; GFX10-WGP:       ; %bb.0: ; %entry
9234; GFX10-WGP-NEXT:    s_clause 0x1
9235; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9236; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9237; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
9238; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9239; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9240; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9241; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9242; GFX10-WGP-NEXT:    s_endpgm
9243;
9244; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
9245; GFX10-CU:       ; %bb.0: ; %entry
9246; GFX10-CU-NEXT:    s_clause 0x1
9247; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9248; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9249; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
9250; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9251; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9252; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9253; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9254; GFX10-CU-NEXT:    s_endpgm
9255;
9256; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
9257; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9258; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
9259; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
9260; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
9261; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
9262; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9263; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
9264; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
9265; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
9266; SKIP-CACHE-INV-NEXT:    s_endpgm
9267;
9268; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
9269; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9270; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9271; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9272; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9273; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9274; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9275; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9276; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9277;
9278; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
9279; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9280; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9281; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9282; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9283; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9284; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9285; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9286; GFX90A-TGSPLIT-NEXT:    s_endpgm
9287;
9288; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
9289; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9290; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9291; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9292; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9293; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9294; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9295; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
9296; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9297;
9298; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
9299; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9300; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9301; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9302; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9303; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9304; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9305; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
9306; GFX940-TGSPLIT-NEXT:    s_endpgm
9307;
9308; GFX11-WGP-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
9309; GFX11-WGP:       ; %bb.0: ; %entry
9310; GFX11-WGP-NEXT:    s_clause 0x1
9311; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
9312; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
9313; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
9314; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9315; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9316; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
9317; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9318; GFX11-WGP-NEXT:    s_endpgm
9319;
9320; GFX11-CU-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
9321; GFX11-CU:       ; %bb.0: ; %entry
9322; GFX11-CU-NEXT:    s_clause 0x1
9323; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
9324; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
9325; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
9326; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9327; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9328; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
9329; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9330; GFX11-CU-NEXT:    s_endpgm
9331    i32 addrspace(1)* %out, i32 %in, i32 %old) {
9332entry:
9333  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
9334  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic acquire
9335  ret void
9336}
9337
9338define amdgpu_kernel void @global_singlethread_one_as_acquire_acquire_cmpxchg(
9339; GFX6-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
9340; GFX6:       ; %bb.0: ; %entry
9341; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
9342; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9343; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
9344; GFX6-NEXT:    s_mov_b32 s2, -1
9345; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9346; GFX6-NEXT:    v_mov_b32_e32 v0, s6
9347; GFX6-NEXT:    v_mov_b32_e32 v1, s7
9348; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
9349; GFX6-NEXT:    s_endpgm
9350;
9351; GFX7-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
9352; GFX7:       ; %bb.0: ; %entry
9353; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9354; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9355; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9356; GFX7-NEXT:    s_add_u32 s0, s0, 16
9357; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9358; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9359; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9360; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9361; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9362; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9363; GFX7-NEXT:    s_endpgm
9364;
9365; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
9366; GFX10-WGP:       ; %bb.0: ; %entry
9367; GFX10-WGP-NEXT:    s_clause 0x1
9368; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9369; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9370; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
9371; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9372; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9373; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9374; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9375; GFX10-WGP-NEXT:    s_endpgm
9376;
9377; GFX10-CU-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
9378; GFX10-CU:       ; %bb.0: ; %entry
9379; GFX10-CU-NEXT:    s_clause 0x1
9380; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9381; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9382; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
9383; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9384; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9385; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9386; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9387; GFX10-CU-NEXT:    s_endpgm
9388;
9389; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
9390; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9391; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
9392; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
9393; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
9394; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
9395; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9396; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
9397; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
9398; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
9399; SKIP-CACHE-INV-NEXT:    s_endpgm
9400;
9401; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
9402; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9403; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9404; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9405; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9406; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9407; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9408; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9409; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9410;
9411; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
9412; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9413; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9414; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9415; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9416; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9417; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9418; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9419; GFX90A-TGSPLIT-NEXT:    s_endpgm
9420;
9421; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
9422; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9423; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9424; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9425; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9426; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9427; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9428; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
9429; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9430;
9431; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
9432; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9433; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9434; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9435; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9436; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9437; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9438; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
9439; GFX940-TGSPLIT-NEXT:    s_endpgm
9440;
9441; GFX11-WGP-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
9442; GFX11-WGP:       ; %bb.0: ; %entry
9443; GFX11-WGP-NEXT:    s_clause 0x1
9444; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
9445; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
9446; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
9447; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9448; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9449; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
9450; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9451; GFX11-WGP-NEXT:    s_endpgm
9452;
9453; GFX11-CU-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
9454; GFX11-CU:       ; %bb.0: ; %entry
9455; GFX11-CU-NEXT:    s_clause 0x1
9456; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
9457; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
9458; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
9459; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9460; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9461; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
9462; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9463; GFX11-CU-NEXT:    s_endpgm
9464    i32 addrspace(1)* %out, i32 %in, i32 %old) {
9465entry:
9466  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
9467  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire acquire
9468  ret void
9469}
9470
9471define amdgpu_kernel void @global_singlethread_one_as_release_acquire_cmpxchg(
9472; GFX6-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
9473; GFX6:       ; %bb.0: ; %entry
9474; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
9475; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9476; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
9477; GFX6-NEXT:    s_mov_b32 s2, -1
9478; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9479; GFX6-NEXT:    v_mov_b32_e32 v0, s6
9480; GFX6-NEXT:    v_mov_b32_e32 v1, s7
9481; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
9482; GFX6-NEXT:    s_endpgm
9483;
9484; GFX7-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
9485; GFX7:       ; %bb.0: ; %entry
9486; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9487; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9488; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9489; GFX7-NEXT:    s_add_u32 s0, s0, 16
9490; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9491; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9492; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9493; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9494; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9495; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9496; GFX7-NEXT:    s_endpgm
9497;
9498; GFX10-WGP-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
9499; GFX10-WGP:       ; %bb.0: ; %entry
9500; GFX10-WGP-NEXT:    s_clause 0x1
9501; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9502; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9503; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
9504; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9505; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9506; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9507; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9508; GFX10-WGP-NEXT:    s_endpgm
9509;
9510; GFX10-CU-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
9511; GFX10-CU:       ; %bb.0: ; %entry
9512; GFX10-CU-NEXT:    s_clause 0x1
9513; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9514; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9515; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
9516; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9517; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9518; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9519; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9520; GFX10-CU-NEXT:    s_endpgm
9521;
9522; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
9523; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9524; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
9525; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
9526; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
9527; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
9528; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9529; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
9530; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
9531; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
9532; SKIP-CACHE-INV-NEXT:    s_endpgm
9533;
9534; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
9535; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9536; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9537; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9538; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9539; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9540; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9541; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9542; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9543;
9544; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
9545; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9546; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9547; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9548; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9549; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9550; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9551; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9552; GFX90A-TGSPLIT-NEXT:    s_endpgm
9553;
9554; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
9555; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9556; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9557; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9558; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9559; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9560; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9561; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
9562; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9563;
9564; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
9565; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9566; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9567; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9568; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9569; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9570; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9571; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
9572; GFX940-TGSPLIT-NEXT:    s_endpgm
9573;
9574; GFX11-WGP-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
9575; GFX11-WGP:       ; %bb.0: ; %entry
9576; GFX11-WGP-NEXT:    s_clause 0x1
9577; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
9578; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
9579; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
9580; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9581; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9582; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
9583; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9584; GFX11-WGP-NEXT:    s_endpgm
9585;
9586; GFX11-CU-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
9587; GFX11-CU:       ; %bb.0: ; %entry
9588; GFX11-CU-NEXT:    s_clause 0x1
9589; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
9590; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
9591; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
9592; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9593; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9594; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
9595; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9596; GFX11-CU-NEXT:    s_endpgm
9597    i32 addrspace(1)* %out, i32 %in, i32 %old) {
9598entry:
9599  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
9600  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release acquire
9601  ret void
9602}
9603
9604define amdgpu_kernel void @global_singlethread_one_as_acq_rel_acquire_cmpxchg(
9605; GFX6-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
9606; GFX6:       ; %bb.0: ; %entry
9607; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
9608; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9609; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
9610; GFX6-NEXT:    s_mov_b32 s2, -1
9611; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9612; GFX6-NEXT:    v_mov_b32_e32 v0, s6
9613; GFX6-NEXT:    v_mov_b32_e32 v1, s7
9614; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
9615; GFX6-NEXT:    s_endpgm
9616;
9617; GFX7-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
9618; GFX7:       ; %bb.0: ; %entry
9619; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9620; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9621; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9622; GFX7-NEXT:    s_add_u32 s0, s0, 16
9623; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9624; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9625; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9626; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9627; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9628; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9629; GFX7-NEXT:    s_endpgm
9630;
9631; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
9632; GFX10-WGP:       ; %bb.0: ; %entry
9633; GFX10-WGP-NEXT:    s_clause 0x1
9634; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9635; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9636; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
9637; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9638; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9639; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9640; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9641; GFX10-WGP-NEXT:    s_endpgm
9642;
9643; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
9644; GFX10-CU:       ; %bb.0: ; %entry
9645; GFX10-CU-NEXT:    s_clause 0x1
9646; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9647; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9648; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
9649; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9650; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9651; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9652; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9653; GFX10-CU-NEXT:    s_endpgm
9654;
9655; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
9656; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9657; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
9658; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
9659; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
9660; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
9661; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9662; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
9663; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
9664; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
9665; SKIP-CACHE-INV-NEXT:    s_endpgm
9666;
9667; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
9668; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9669; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9670; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9671; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9672; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9673; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9674; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9675; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9676;
9677; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
9678; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9679; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9680; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9681; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9682; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9683; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9684; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9685; GFX90A-TGSPLIT-NEXT:    s_endpgm
9686;
9687; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
9688; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9689; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9690; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9691; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9692; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9693; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9694; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
9695; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9696;
9697; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
9698; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9699; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9700; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9701; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9702; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9703; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9704; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
9705; GFX940-TGSPLIT-NEXT:    s_endpgm
9706;
9707; GFX11-WGP-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
9708; GFX11-WGP:       ; %bb.0: ; %entry
9709; GFX11-WGP-NEXT:    s_clause 0x1
9710; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
9711; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
9712; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
9713; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9714; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9715; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
9716; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9717; GFX11-WGP-NEXT:    s_endpgm
9718;
9719; GFX11-CU-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
9720; GFX11-CU:       ; %bb.0: ; %entry
9721; GFX11-CU-NEXT:    s_clause 0x1
9722; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
9723; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
9724; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
9725; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9726; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9727; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
9728; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9729; GFX11-CU-NEXT:    s_endpgm
9730    i32 addrspace(1)* %out, i32 %in, i32 %old) {
9731entry:
9732  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
9733  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel acquire
9734  ret void
9735}
9736
9737define amdgpu_kernel void @global_singlethread_one_as_seq_cst_acquire_cmpxchg(
9738; GFX6-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
9739; GFX6:       ; %bb.0: ; %entry
9740; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
9741; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9742; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
9743; GFX6-NEXT:    s_mov_b32 s2, -1
9744; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9745; GFX6-NEXT:    v_mov_b32_e32 v0, s6
9746; GFX6-NEXT:    v_mov_b32_e32 v1, s7
9747; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
9748; GFX6-NEXT:    s_endpgm
9749;
9750; GFX7-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
9751; GFX7:       ; %bb.0: ; %entry
9752; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9753; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9754; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9755; GFX7-NEXT:    s_add_u32 s0, s0, 16
9756; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9757; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9758; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9759; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9760; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9761; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9762; GFX7-NEXT:    s_endpgm
9763;
9764; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
9765; GFX10-WGP:       ; %bb.0: ; %entry
9766; GFX10-WGP-NEXT:    s_clause 0x1
9767; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9768; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9769; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
9770; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9771; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9772; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9773; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9774; GFX10-WGP-NEXT:    s_endpgm
9775;
9776; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
9777; GFX10-CU:       ; %bb.0: ; %entry
9778; GFX10-CU-NEXT:    s_clause 0x1
9779; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9780; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9781; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
9782; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9783; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9784; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9785; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9786; GFX10-CU-NEXT:    s_endpgm
9787;
9788; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
9789; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9790; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
9791; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
9792; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
9793; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
9794; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9795; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
9796; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
9797; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
9798; SKIP-CACHE-INV-NEXT:    s_endpgm
9799;
9800; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
9801; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9802; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9803; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9804; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9805; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9806; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9807; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9808; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9809;
9810; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
9811; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9812; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9813; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9814; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9815; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9816; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9817; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9818; GFX90A-TGSPLIT-NEXT:    s_endpgm
9819;
9820; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
9821; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9822; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9823; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9824; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9825; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9826; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9827; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
9828; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9829;
9830; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
9831; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9832; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9833; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9834; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9835; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9836; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9837; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
9838; GFX940-TGSPLIT-NEXT:    s_endpgm
9839;
9840; GFX11-WGP-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
9841; GFX11-WGP:       ; %bb.0: ; %entry
9842; GFX11-WGP-NEXT:    s_clause 0x1
9843; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
9844; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
9845; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
9846; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9847; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9848; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
9849; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9850; GFX11-WGP-NEXT:    s_endpgm
9851;
9852; GFX11-CU-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
9853; GFX11-CU:       ; %bb.0: ; %entry
9854; GFX11-CU-NEXT:    s_clause 0x1
9855; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
9856; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
9857; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
9858; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9859; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9860; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
9861; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9862; GFX11-CU-NEXT:    s_endpgm
9863    i32 addrspace(1)* %out, i32 %in, i32 %old) {
9864entry:
9865  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
9866  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst acquire
9867  ret void
9868}
9869
9870define amdgpu_kernel void @global_singlethread_one_as_monotonic_seq_cst_cmpxchg(
9871; GFX6-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9872; GFX6:       ; %bb.0: ; %entry
9873; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
9874; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9875; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
9876; GFX6-NEXT:    s_mov_b32 s2, -1
9877; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9878; GFX6-NEXT:    v_mov_b32_e32 v0, s6
9879; GFX6-NEXT:    v_mov_b32_e32 v1, s7
9880; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
9881; GFX6-NEXT:    s_endpgm
9882;
9883; GFX7-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9884; GFX7:       ; %bb.0: ; %entry
9885; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9886; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9887; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9888; GFX7-NEXT:    s_add_u32 s0, s0, 16
9889; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9890; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9891; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9892; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9893; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9894; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9895; GFX7-NEXT:    s_endpgm
9896;
9897; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9898; GFX10-WGP:       ; %bb.0: ; %entry
9899; GFX10-WGP-NEXT:    s_clause 0x1
9900; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9901; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9902; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
9903; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9904; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9905; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9906; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9907; GFX10-WGP-NEXT:    s_endpgm
9908;
9909; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9910; GFX10-CU:       ; %bb.0: ; %entry
9911; GFX10-CU-NEXT:    s_clause 0x1
9912; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9913; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9914; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
9915; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9916; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9917; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9918; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9919; GFX10-CU-NEXT:    s_endpgm
9920;
9921; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9922; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9923; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
9924; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
9925; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
9926; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
9927; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9928; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
9929; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
9930; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
9931; SKIP-CACHE-INV-NEXT:    s_endpgm
9932;
9933; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9934; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9935; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9936; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9937; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9938; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9939; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9940; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9941; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9942;
9943; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9944; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9945; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9946; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9947; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9948; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9949; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9950; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9951; GFX90A-TGSPLIT-NEXT:    s_endpgm
9952;
9953; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9954; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9955; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9956; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9957; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9958; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9959; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9960; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
9961; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9962;
9963; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9964; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9965; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9966; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9967; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9968; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9969; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9970; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
9971; GFX940-TGSPLIT-NEXT:    s_endpgm
9972;
9973; GFX11-WGP-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9974; GFX11-WGP:       ; %bb.0: ; %entry
9975; GFX11-WGP-NEXT:    s_clause 0x1
9976; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
9977; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
9978; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
9979; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9980; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9981; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
9982; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9983; GFX11-WGP-NEXT:    s_endpgm
9984;
9985; GFX11-CU-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9986; GFX11-CU:       ; %bb.0: ; %entry
9987; GFX11-CU-NEXT:    s_clause 0x1
9988; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
9989; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
9990; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
9991; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9992; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9993; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
9994; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9995; GFX11-CU-NEXT:    s_endpgm
9996    i32 addrspace(1)* %out, i32 %in, i32 %old) {
9997entry:
9998  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
9999  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic seq_cst
10000  ret void
10001}
10002
10003define amdgpu_kernel void @global_singlethread_one_as_acquire_seq_cst_cmpxchg(
10004; GFX6-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
10005; GFX6:       ; %bb.0: ; %entry
10006; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
10007; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10008; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
10009; GFX6-NEXT:    s_mov_b32 s2, -1
10010; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10011; GFX6-NEXT:    v_mov_b32_e32 v0, s6
10012; GFX6-NEXT:    v_mov_b32_e32 v1, s7
10013; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
10014; GFX6-NEXT:    s_endpgm
10015;
10016; GFX7-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
10017; GFX7:       ; %bb.0: ; %entry
10018; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10019; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10020; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10021; GFX7-NEXT:    s_add_u32 s0, s0, 16
10022; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10023; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10024; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10025; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10026; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10027; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10028; GFX7-NEXT:    s_endpgm
10029;
10030; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
10031; GFX10-WGP:       ; %bb.0: ; %entry
10032; GFX10-WGP-NEXT:    s_clause 0x1
10033; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10034; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10035; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
10036; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10037; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10038; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10039; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
10040; GFX10-WGP-NEXT:    s_endpgm
10041;
10042; GFX10-CU-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
10043; GFX10-CU:       ; %bb.0: ; %entry
10044; GFX10-CU-NEXT:    s_clause 0x1
10045; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10046; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10047; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
10048; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10049; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10050; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10051; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
10052; GFX10-CU-NEXT:    s_endpgm
10053;
10054; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
10055; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10056; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
10057; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
10058; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
10059; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
10060; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10061; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10062; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10063; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
10064; SKIP-CACHE-INV-NEXT:    s_endpgm
10065;
10066; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
10067; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10068; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10069; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10070; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10071; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10072; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10073; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
10074; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10075;
10076; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
10077; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10078; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10079; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10080; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10081; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10082; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10083; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
10084; GFX90A-TGSPLIT-NEXT:    s_endpgm
10085;
10086; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
10087; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10088; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10089; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10090; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10091; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10092; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10093; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
10094; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10095;
10096; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
10097; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10098; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10099; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10100; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10101; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10102; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10103; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
10104; GFX940-TGSPLIT-NEXT:    s_endpgm
10105;
10106; GFX11-WGP-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
10107; GFX11-WGP:       ; %bb.0: ; %entry
10108; GFX11-WGP-NEXT:    s_clause 0x1
10109; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
10110; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
10111; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
10112; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10113; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10114; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
10115; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10116; GFX11-WGP-NEXT:    s_endpgm
10117;
10118; GFX11-CU-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
10119; GFX11-CU:       ; %bb.0: ; %entry
10120; GFX11-CU-NEXT:    s_clause 0x1
10121; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
10122; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
10123; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
10124; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10125; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10126; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
10127; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10128; GFX11-CU-NEXT:    s_endpgm
10129    i32 addrspace(1)* %out, i32 %in, i32 %old) {
10130entry:
10131  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
10132  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire seq_cst
10133  ret void
10134}
10135
10136define amdgpu_kernel void @global_singlethread_one_as_release_seq_cst_cmpxchg(
10137; GFX6-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
10138; GFX6:       ; %bb.0: ; %entry
10139; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
10140; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10141; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
10142; GFX6-NEXT:    s_mov_b32 s2, -1
10143; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10144; GFX6-NEXT:    v_mov_b32_e32 v0, s6
10145; GFX6-NEXT:    v_mov_b32_e32 v1, s7
10146; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
10147; GFX6-NEXT:    s_endpgm
10148;
10149; GFX7-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
10150; GFX7:       ; %bb.0: ; %entry
10151; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10152; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10153; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10154; GFX7-NEXT:    s_add_u32 s0, s0, 16
10155; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10156; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10157; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10158; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10159; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10160; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10161; GFX7-NEXT:    s_endpgm
10162;
10163; GFX10-WGP-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
10164; GFX10-WGP:       ; %bb.0: ; %entry
10165; GFX10-WGP-NEXT:    s_clause 0x1
10166; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10167; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10168; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
10169; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10170; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10171; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10172; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
10173; GFX10-WGP-NEXT:    s_endpgm
10174;
10175; GFX10-CU-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
10176; GFX10-CU:       ; %bb.0: ; %entry
10177; GFX10-CU-NEXT:    s_clause 0x1
10178; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10179; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10180; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
10181; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10182; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10183; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10184; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
10185; GFX10-CU-NEXT:    s_endpgm
10186;
10187; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
10188; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10189; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
10190; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
10191; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
10192; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
10193; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10194; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10195; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10196; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
10197; SKIP-CACHE-INV-NEXT:    s_endpgm
10198;
10199; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
10200; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10201; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10202; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10203; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10204; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10205; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10206; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
10207; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10208;
10209; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
10210; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10211; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10212; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10213; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10214; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10215; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10216; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
10217; GFX90A-TGSPLIT-NEXT:    s_endpgm
10218;
10219; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
10220; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10221; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10222; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10223; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10224; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10225; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10226; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
10227; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10228;
10229; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
10230; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10231; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10232; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10233; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10234; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10235; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10236; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
10237; GFX940-TGSPLIT-NEXT:    s_endpgm
10238;
10239; GFX11-WGP-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
10240; GFX11-WGP:       ; %bb.0: ; %entry
10241; GFX11-WGP-NEXT:    s_clause 0x1
10242; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
10243; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
10244; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
10245; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10246; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10247; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
10248; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10249; GFX11-WGP-NEXT:    s_endpgm
10250;
10251; GFX11-CU-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
10252; GFX11-CU:       ; %bb.0: ; %entry
10253; GFX11-CU-NEXT:    s_clause 0x1
10254; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
10255; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
10256; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
10257; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10258; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10259; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
10260; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10261; GFX11-CU-NEXT:    s_endpgm
10262    i32 addrspace(1)* %out, i32 %in, i32 %old) {
10263entry:
10264  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
10265  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release seq_cst
10266  ret void
10267}
10268
10269define amdgpu_kernel void @global_singlethread_one_as_acq_rel_seq_cst_cmpxchg(
10270; GFX6-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10271; GFX6:       ; %bb.0: ; %entry
10272; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
10273; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10274; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
10275; GFX6-NEXT:    s_mov_b32 s2, -1
10276; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10277; GFX6-NEXT:    v_mov_b32_e32 v0, s6
10278; GFX6-NEXT:    v_mov_b32_e32 v1, s7
10279; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
10280; GFX6-NEXT:    s_endpgm
10281;
10282; GFX7-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10283; GFX7:       ; %bb.0: ; %entry
10284; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10285; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10286; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10287; GFX7-NEXT:    s_add_u32 s0, s0, 16
10288; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10289; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10290; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10291; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10292; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10293; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10294; GFX7-NEXT:    s_endpgm
10295;
10296; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10297; GFX10-WGP:       ; %bb.0: ; %entry
10298; GFX10-WGP-NEXT:    s_clause 0x1
10299; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10300; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10301; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
10302; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10303; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10304; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10305; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
10306; GFX10-WGP-NEXT:    s_endpgm
10307;
10308; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10309; GFX10-CU:       ; %bb.0: ; %entry
10310; GFX10-CU-NEXT:    s_clause 0x1
10311; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10312; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10313; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
10314; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10315; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10316; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10317; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
10318; GFX10-CU-NEXT:    s_endpgm
10319;
10320; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10321; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10322; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
10323; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
10324; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
10325; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
10326; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10327; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10328; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10329; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
10330; SKIP-CACHE-INV-NEXT:    s_endpgm
10331;
10332; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10333; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10334; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10335; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10336; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10337; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10338; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10339; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
10340; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10341;
10342; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10343; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10344; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10345; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10346; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10347; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10348; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10349; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
10350; GFX90A-TGSPLIT-NEXT:    s_endpgm
10351;
10352; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10353; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10354; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10355; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10356; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10357; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10358; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10359; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
10360; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10361;
10362; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10363; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10364; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10365; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10366; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10367; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10368; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10369; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
10370; GFX940-TGSPLIT-NEXT:    s_endpgm
10371;
10372; GFX11-WGP-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10373; GFX11-WGP:       ; %bb.0: ; %entry
10374; GFX11-WGP-NEXT:    s_clause 0x1
10375; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
10376; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
10377; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
10378; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10379; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10380; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
10381; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10382; GFX11-WGP-NEXT:    s_endpgm
10383;
10384; GFX11-CU-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10385; GFX11-CU:       ; %bb.0: ; %entry
10386; GFX11-CU-NEXT:    s_clause 0x1
10387; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
10388; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
10389; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
10390; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10391; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10392; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
10393; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10394; GFX11-CU-NEXT:    s_endpgm
10395    i32 addrspace(1)* %out, i32 %in, i32 %old) {
10396entry:
10397  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
10398  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel seq_cst
10399  ret void
10400}
10401
10402define amdgpu_kernel void @global_singlethread_one_as_seq_cst_seq_cst_cmpxchg(
10403; GFX6-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10404; GFX6:       ; %bb.0: ; %entry
10405; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
10406; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10407; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
10408; GFX6-NEXT:    s_mov_b32 s2, -1
10409; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10410; GFX6-NEXT:    v_mov_b32_e32 v0, s6
10411; GFX6-NEXT:    v_mov_b32_e32 v1, s7
10412; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
10413; GFX6-NEXT:    s_endpgm
10414;
10415; GFX7-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10416; GFX7:       ; %bb.0: ; %entry
10417; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10418; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10419; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10420; GFX7-NEXT:    s_add_u32 s0, s0, 16
10421; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10422; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10423; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10424; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10425; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10426; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10427; GFX7-NEXT:    s_endpgm
10428;
10429; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10430; GFX10-WGP:       ; %bb.0: ; %entry
10431; GFX10-WGP-NEXT:    s_clause 0x1
10432; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10433; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10434; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
10435; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10436; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10437; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10438; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
10439; GFX10-WGP-NEXT:    s_endpgm
10440;
10441; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10442; GFX10-CU:       ; %bb.0: ; %entry
10443; GFX10-CU-NEXT:    s_clause 0x1
10444; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10445; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10446; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
10447; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10448; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10449; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10450; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
10451; GFX10-CU-NEXT:    s_endpgm
10452;
10453; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10454; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10455; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
10456; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
10457; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
10458; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
10459; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10460; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10461; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10462; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
10463; SKIP-CACHE-INV-NEXT:    s_endpgm
10464;
10465; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10466; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10467; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10468; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10469; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10470; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10471; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10472; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
10473; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10474;
10475; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10476; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10477; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10478; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10479; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10480; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10481; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10482; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
10483; GFX90A-TGSPLIT-NEXT:    s_endpgm
10484;
10485; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10486; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10487; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10488; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10489; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10490; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10491; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10492; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
10493; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10494;
10495; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10496; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10497; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10498; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10499; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10500; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10501; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10502; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
10503; GFX940-TGSPLIT-NEXT:    s_endpgm
10504;
10505; GFX11-WGP-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10506; GFX11-WGP:       ; %bb.0: ; %entry
10507; GFX11-WGP-NEXT:    s_clause 0x1
10508; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
10509; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
10510; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
10511; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10512; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10513; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
10514; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10515; GFX11-WGP-NEXT:    s_endpgm
10516;
10517; GFX11-CU-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10518; GFX11-CU:       ; %bb.0: ; %entry
10519; GFX11-CU-NEXT:    s_clause 0x1
10520; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
10521; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
10522; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
10523; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10524; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10525; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
10526; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10527; GFX11-CU-NEXT:    s_endpgm
10528    i32 addrspace(1)* %out, i32 %in, i32 %old) {
10529entry:
10530  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
10531  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst seq_cst
10532  ret void
10533}
10534
10535define amdgpu_kernel void @global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg(
10536; GFX6-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10537; GFX6:       ; %bb.0: ; %entry
10538; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
10539; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10540; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
10541; GFX6-NEXT:    s_mov_b32 s2, -1
10542; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10543; GFX6-NEXT:    v_mov_b32_e32 v0, s6
10544; GFX6-NEXT:    v_mov_b32_e32 v1, s7
10545; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
10546; GFX6-NEXT:    s_waitcnt vmcnt(0)
10547; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
10548; GFX6-NEXT:    s_endpgm
10549;
10550; GFX7-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10551; GFX7:       ; %bb.0: ; %entry
10552; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10553; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10554; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10555; GFX7-NEXT:    s_add_u32 s4, s0, 16
10556; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10557; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10558; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10559; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10560; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10561; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10562; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10563; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10564; GFX7-NEXT:    s_waitcnt vmcnt(0)
10565; GFX7-NEXT:    flat_store_dword v[0:1], v2
10566; GFX7-NEXT:    s_endpgm
10567;
10568; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10569; GFX10-WGP:       ; %bb.0: ; %entry
10570; GFX10-WGP-NEXT:    s_clause 0x1
10571; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10572; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10573; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
10574; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10575; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10576; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10577; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10578; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10579; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
10580; GFX10-WGP-NEXT:    s_endpgm
10581;
10582; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10583; GFX10-CU:       ; %bb.0: ; %entry
10584; GFX10-CU-NEXT:    s_clause 0x1
10585; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10586; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10587; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
10588; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10589; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10590; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10591; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10592; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10593; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
10594; GFX10-CU-NEXT:    s_endpgm
10595;
10596; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10597; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10598; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
10599; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
10600; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
10601; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
10602; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10603; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10604; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10605; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
10606; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10607; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
10608; SKIP-CACHE-INV-NEXT:    s_endpgm
10609;
10610; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10611; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10612; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10613; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10614; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10615; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10616; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10617; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10618; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10619; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
10620; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10621;
10622; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10623; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10624; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10625; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10626; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10627; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10628; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10629; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10630; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10631; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
10632; GFX90A-TGSPLIT-NEXT:    s_endpgm
10633;
10634; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10635; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10636; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10637; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10638; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10639; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10640; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10641; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
10642; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10643; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
10644; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10645;
10646; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10647; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10648; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10649; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10650; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10651; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10652; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10653; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
10654; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10655; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
10656; GFX940-TGSPLIT-NEXT:    s_endpgm
10657;
10658; GFX11-WGP-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10659; GFX11-WGP:       ; %bb.0: ; %entry
10660; GFX11-WGP-NEXT:    s_clause 0x1
10661; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
10662; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
10663; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
10664; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10665; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10666; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
10667; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
10668; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
10669; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10670; GFX11-WGP-NEXT:    s_endpgm
10671;
10672; GFX11-CU-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10673; GFX11-CU:       ; %bb.0: ; %entry
10674; GFX11-CU-NEXT:    s_clause 0x1
10675; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
10676; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
10677; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
10678; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10679; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10680; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
10681; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
10682; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
10683; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10684; GFX11-CU-NEXT:    s_endpgm
10685    i32 addrspace(1)* %out, i32 %in, i32 %old) {
10686entry:
10687  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
10688  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic monotonic
10689  %val0 = extractvalue { i32, i1 } %val, 0
10690  store i32 %val0, i32 addrspace(1)* %out, align 4
10691  ret void
10692}
10693
10694define amdgpu_kernel void @global_singlethread_one_as_acquire_monotonic_ret_cmpxchg(
10695; GFX6-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10696; GFX6:       ; %bb.0: ; %entry
10697; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
10698; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10699; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
10700; GFX6-NEXT:    s_mov_b32 s2, -1
10701; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10702; GFX6-NEXT:    v_mov_b32_e32 v0, s6
10703; GFX6-NEXT:    v_mov_b32_e32 v1, s7
10704; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
10705; GFX6-NEXT:    s_waitcnt vmcnt(0)
10706; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
10707; GFX6-NEXT:    s_endpgm
10708;
10709; GFX7-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10710; GFX7:       ; %bb.0: ; %entry
10711; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10712; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10713; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10714; GFX7-NEXT:    s_add_u32 s4, s0, 16
10715; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10716; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10717; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10718; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10719; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10720; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10721; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10722; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10723; GFX7-NEXT:    s_waitcnt vmcnt(0)
10724; GFX7-NEXT:    flat_store_dword v[0:1], v2
10725; GFX7-NEXT:    s_endpgm
10726;
10727; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10728; GFX10-WGP:       ; %bb.0: ; %entry
10729; GFX10-WGP-NEXT:    s_clause 0x1
10730; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10731; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10732; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
10733; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10734; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10735; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10736; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10737; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10738; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
10739; GFX10-WGP-NEXT:    s_endpgm
10740;
10741; GFX10-CU-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10742; GFX10-CU:       ; %bb.0: ; %entry
10743; GFX10-CU-NEXT:    s_clause 0x1
10744; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10745; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10746; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
10747; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10748; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10749; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10750; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10751; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10752; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
10753; GFX10-CU-NEXT:    s_endpgm
10754;
10755; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10756; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10757; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
10758; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
10759; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
10760; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
10761; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10762; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10763; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10764; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
10765; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10766; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
10767; SKIP-CACHE-INV-NEXT:    s_endpgm
10768;
10769; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10770; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10771; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10772; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10773; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10774; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10775; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10776; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10777; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10778; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
10779; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10780;
10781; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10782; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10783; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10784; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10785; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10786; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10787; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10788; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10789; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10790; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
10791; GFX90A-TGSPLIT-NEXT:    s_endpgm
10792;
10793; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10794; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10795; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10796; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10797; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10798; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10799; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10800; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
10801; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10802; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
10803; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10804;
10805; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10806; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10807; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10808; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10809; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10810; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10811; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10812; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
10813; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10814; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
10815; GFX940-TGSPLIT-NEXT:    s_endpgm
10816;
10817; GFX11-WGP-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10818; GFX11-WGP:       ; %bb.0: ; %entry
10819; GFX11-WGP-NEXT:    s_clause 0x1
10820; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
10821; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
10822; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
10823; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10824; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10825; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
10826; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
10827; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
10828; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10829; GFX11-WGP-NEXT:    s_endpgm
10830;
10831; GFX11-CU-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10832; GFX11-CU:       ; %bb.0: ; %entry
10833; GFX11-CU-NEXT:    s_clause 0x1
10834; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
10835; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
10836; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
10837; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10838; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10839; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
10840; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
10841; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
10842; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10843; GFX11-CU-NEXT:    s_endpgm
10844    i32 addrspace(1)* %out, i32 %in, i32 %old) {
10845entry:
10846  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
10847  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire monotonic
10848  %val0 = extractvalue { i32, i1 } %val, 0
10849  store i32 %val0, i32 addrspace(1)* %out, align 4
10850  ret void
10851}
10852
10853define amdgpu_kernel void @global_singlethread_one_as_release_monotonic_ret_cmpxchg(
10854; GFX6-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
10855; GFX6:       ; %bb.0: ; %entry
10856; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
10857; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10858; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
10859; GFX6-NEXT:    s_mov_b32 s2, -1
10860; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10861; GFX6-NEXT:    v_mov_b32_e32 v0, s6
10862; GFX6-NEXT:    v_mov_b32_e32 v1, s7
10863; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
10864; GFX6-NEXT:    s_waitcnt vmcnt(0)
10865; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
10866; GFX6-NEXT:    s_endpgm
10867;
10868; GFX7-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
10869; GFX7:       ; %bb.0: ; %entry
10870; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10871; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10872; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10873; GFX7-NEXT:    s_add_u32 s4, s0, 16
10874; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10875; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10876; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10877; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10878; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10879; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10880; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10881; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10882; GFX7-NEXT:    s_waitcnt vmcnt(0)
10883; GFX7-NEXT:    flat_store_dword v[0:1], v2
10884; GFX7-NEXT:    s_endpgm
10885;
10886; GFX10-WGP-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
10887; GFX10-WGP:       ; %bb.0: ; %entry
10888; GFX10-WGP-NEXT:    s_clause 0x1
10889; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10890; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10891; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
10892; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10893; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10894; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10895; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10896; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10897; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
10898; GFX10-WGP-NEXT:    s_endpgm
10899;
10900; GFX10-CU-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
10901; GFX10-CU:       ; %bb.0: ; %entry
10902; GFX10-CU-NEXT:    s_clause 0x1
10903; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10904; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10905; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
10906; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10907; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10908; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10909; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10910; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10911; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
10912; GFX10-CU-NEXT:    s_endpgm
10913;
10914; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
10915; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10916; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
10917; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
10918; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
10919; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
10920; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10921; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10922; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10923; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
10924; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10925; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
10926; SKIP-CACHE-INV-NEXT:    s_endpgm
10927;
10928; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
10929; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10930; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10931; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10932; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10933; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10934; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10935; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10936; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10937; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
10938; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10939;
10940; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
10941; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10942; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10943; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10944; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10945; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10946; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10947; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10948; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10949; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
10950; GFX90A-TGSPLIT-NEXT:    s_endpgm
10951;
10952; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
10953; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10954; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10955; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10956; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10957; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10958; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10959; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
10960; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10961; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
10962; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10963;
10964; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
10965; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10966; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10967; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10968; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10969; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10970; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10971; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
10972; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10973; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
10974; GFX940-TGSPLIT-NEXT:    s_endpgm
10975;
10976; GFX11-WGP-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
10977; GFX11-WGP:       ; %bb.0: ; %entry
10978; GFX11-WGP-NEXT:    s_clause 0x1
10979; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
10980; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
10981; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
10982; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10983; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10984; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
10985; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
10986; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
10987; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10988; GFX11-WGP-NEXT:    s_endpgm
10989;
10990; GFX11-CU-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
10991; GFX11-CU:       ; %bb.0: ; %entry
10992; GFX11-CU-NEXT:    s_clause 0x1
10993; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
10994; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
10995; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
10996; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10997; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10998; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
10999; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11000; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
11001; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11002; GFX11-CU-NEXT:    s_endpgm
11003    i32 addrspace(1)* %out, i32 %in, i32 %old) {
11004entry:
11005  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
11006  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release monotonic
11007  %val0 = extractvalue { i32, i1 } %val, 0
11008  store i32 %val0, i32 addrspace(1)* %out, align 4
11009  ret void
11010}
11011
11012define amdgpu_kernel void @global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg(
11013; GFX6-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
11014; GFX6:       ; %bb.0: ; %entry
11015; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
11016; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11017; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
11018; GFX6-NEXT:    s_mov_b32 s2, -1
11019; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
11020; GFX6-NEXT:    v_mov_b32_e32 v0, s6
11021; GFX6-NEXT:    v_mov_b32_e32 v1, s7
11022; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
11023; GFX6-NEXT:    s_waitcnt vmcnt(0)
11024; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
11025; GFX6-NEXT:    s_endpgm
11026;
11027; GFX7-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
11028; GFX7:       ; %bb.0: ; %entry
11029; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11030; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11031; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11032; GFX7-NEXT:    s_add_u32 s4, s0, 16
11033; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11034; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11035; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11036; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11037; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11038; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11039; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11040; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11041; GFX7-NEXT:    s_waitcnt vmcnt(0)
11042; GFX7-NEXT:    flat_store_dword v[0:1], v2
11043; GFX7-NEXT:    s_endpgm
11044;
11045; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
11046; GFX10-WGP:       ; %bb.0: ; %entry
11047; GFX10-WGP-NEXT:    s_clause 0x1
11048; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11049; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11050; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
11051; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11052; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11053; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11054; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11055; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11056; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
11057; GFX10-WGP-NEXT:    s_endpgm
11058;
11059; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
11060; GFX10-CU:       ; %bb.0: ; %entry
11061; GFX10-CU-NEXT:    s_clause 0x1
11062; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11063; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11064; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
11065; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11066; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11067; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11068; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11069; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11070; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
11071; GFX10-CU-NEXT:    s_endpgm
11072;
11073; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
11074; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11075; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
11076; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
11077; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
11078; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
11079; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11080; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11081; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11082; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
11083; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11084; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
11085; SKIP-CACHE-INV-NEXT:    s_endpgm
11086;
11087; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
11088; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11089; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11090; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11091; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11092; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11093; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11094; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11095; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11096; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
11097; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11098;
11099; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
11100; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11101; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11102; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11103; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11104; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11105; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11106; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11107; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11108; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
11109; GFX90A-TGSPLIT-NEXT:    s_endpgm
11110;
11111; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
11112; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11113; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
11114; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
11115; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11116; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11117; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11118; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
11119; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11120; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
11121; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11122;
11123; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
11124; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11125; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
11126; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
11127; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11128; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11129; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11130; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
11131; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11132; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
11133; GFX940-TGSPLIT-NEXT:    s_endpgm
11134;
11135; GFX11-WGP-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
11136; GFX11-WGP:       ; %bb.0: ; %entry
11137; GFX11-WGP-NEXT:    s_clause 0x1
11138; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
11139; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
11140; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
11141; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11142; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11143; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
11144; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11145; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
11146; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11147; GFX11-WGP-NEXT:    s_endpgm
11148;
11149; GFX11-CU-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
11150; GFX11-CU:       ; %bb.0: ; %entry
11151; GFX11-CU-NEXT:    s_clause 0x1
11152; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
11153; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
11154; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
11155; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11156; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11157; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
11158; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11159; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
11160; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11161; GFX11-CU-NEXT:    s_endpgm
11162    i32 addrspace(1)* %out, i32 %in, i32 %old) {
11163entry:
11164  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
11165  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel monotonic
11166  %val0 = extractvalue { i32, i1 } %val, 0
11167  store i32 %val0, i32 addrspace(1)* %out, align 4
11168  ret void
11169}
11170
11171define amdgpu_kernel void @global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg(
11172; GFX6-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11173; GFX6:       ; %bb.0: ; %entry
11174; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
11175; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11176; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
11177; GFX6-NEXT:    s_mov_b32 s2, -1
11178; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
11179; GFX6-NEXT:    v_mov_b32_e32 v0, s6
11180; GFX6-NEXT:    v_mov_b32_e32 v1, s7
11181; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
11182; GFX6-NEXT:    s_waitcnt vmcnt(0)
11183; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
11184; GFX6-NEXT:    s_endpgm
11185;
11186; GFX7-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11187; GFX7:       ; %bb.0: ; %entry
11188; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11189; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11190; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11191; GFX7-NEXT:    s_add_u32 s4, s0, 16
11192; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11193; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11194; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11195; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11196; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11197; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11198; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11199; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11200; GFX7-NEXT:    s_waitcnt vmcnt(0)
11201; GFX7-NEXT:    flat_store_dword v[0:1], v2
11202; GFX7-NEXT:    s_endpgm
11203;
11204; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11205; GFX10-WGP:       ; %bb.0: ; %entry
11206; GFX10-WGP-NEXT:    s_clause 0x1
11207; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11208; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11209; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
11210; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11211; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11212; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11213; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11214; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11215; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
11216; GFX10-WGP-NEXT:    s_endpgm
11217;
11218; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11219; GFX10-CU:       ; %bb.0: ; %entry
11220; GFX10-CU-NEXT:    s_clause 0x1
11221; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11222; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11223; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
11224; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11225; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11226; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11227; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11228; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11229; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
11230; GFX10-CU-NEXT:    s_endpgm
11231;
11232; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11233; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11234; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
11235; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
11236; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
11237; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
11238; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11239; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11240; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11241; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
11242; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11243; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
11244; SKIP-CACHE-INV-NEXT:    s_endpgm
11245;
11246; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11247; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11248; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11249; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11250; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11251; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11252; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11253; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11254; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11255; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
11256; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11257;
11258; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11259; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11260; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11261; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11262; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11263; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11264; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11265; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11266; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11267; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
11268; GFX90A-TGSPLIT-NEXT:    s_endpgm
11269;
11270; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11271; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11272; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
11273; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
11274; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11275; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11276; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11277; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
11278; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11279; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
11280; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11281;
11282; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11283; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11284; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
11285; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
11286; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11287; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11288; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11289; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
11290; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11291; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
11292; GFX940-TGSPLIT-NEXT:    s_endpgm
11293;
11294; GFX11-WGP-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11295; GFX11-WGP:       ; %bb.0: ; %entry
11296; GFX11-WGP-NEXT:    s_clause 0x1
11297; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
11298; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
11299; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
11300; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11301; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11302; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
11303; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11304; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
11305; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11306; GFX11-WGP-NEXT:    s_endpgm
11307;
11308; GFX11-CU-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11309; GFX11-CU:       ; %bb.0: ; %entry
11310; GFX11-CU-NEXT:    s_clause 0x1
11311; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
11312; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
11313; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
11314; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11315; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11316; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
11317; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11318; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
11319; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11320; GFX11-CU-NEXT:    s_endpgm
11321    i32 addrspace(1)* %out, i32 %in, i32 %old) {
11322entry:
11323  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
11324  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst monotonic
11325  %val0 = extractvalue { i32, i1 } %val, 0
11326  store i32 %val0, i32 addrspace(1)* %out, align 4
11327  ret void
11328}
11329
11330define amdgpu_kernel void @global_singlethread_one_as_monotonic_acquire_ret_cmpxchg(
11331; GFX6-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11332; GFX6:       ; %bb.0: ; %entry
11333; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
11334; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11335; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
11336; GFX6-NEXT:    s_mov_b32 s2, -1
11337; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
11338; GFX6-NEXT:    v_mov_b32_e32 v0, s6
11339; GFX6-NEXT:    v_mov_b32_e32 v1, s7
11340; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
11341; GFX6-NEXT:    s_waitcnt vmcnt(0)
11342; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
11343; GFX6-NEXT:    s_endpgm
11344;
11345; GFX7-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11346; GFX7:       ; %bb.0: ; %entry
11347; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11348; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11349; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11350; GFX7-NEXT:    s_add_u32 s4, s0, 16
11351; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11352; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11353; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11354; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11355; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11356; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11357; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11358; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11359; GFX7-NEXT:    s_waitcnt vmcnt(0)
11360; GFX7-NEXT:    flat_store_dword v[0:1], v2
11361; GFX7-NEXT:    s_endpgm
11362;
11363; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11364; GFX10-WGP:       ; %bb.0: ; %entry
11365; GFX10-WGP-NEXT:    s_clause 0x1
11366; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11367; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11368; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
11369; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11370; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11371; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11372; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11373; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11374; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
11375; GFX10-WGP-NEXT:    s_endpgm
11376;
11377; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11378; GFX10-CU:       ; %bb.0: ; %entry
11379; GFX10-CU-NEXT:    s_clause 0x1
11380; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11381; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11382; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
11383; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11384; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11385; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11386; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11387; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11388; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
11389; GFX10-CU-NEXT:    s_endpgm
11390;
11391; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11392; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11393; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
11394; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
11395; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
11396; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
11397; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11398; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11399; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11400; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
11401; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11402; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
11403; SKIP-CACHE-INV-NEXT:    s_endpgm
11404;
11405; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11406; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11407; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11408; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11409; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11410; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11411; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11412; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11413; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11414; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
11415; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11416;
11417; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11418; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11419; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11420; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11421; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11422; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11423; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11424; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11425; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11426; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
11427; GFX90A-TGSPLIT-NEXT:    s_endpgm
11428;
11429; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11430; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11431; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
11432; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
11433; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11434; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11435; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11436; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
11437; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11438; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
11439; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11440;
11441; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11442; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11443; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
11444; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
11445; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11446; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11447; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11448; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
11449; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11450; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
11451; GFX940-TGSPLIT-NEXT:    s_endpgm
11452;
11453; GFX11-WGP-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11454; GFX11-WGP:       ; %bb.0: ; %entry
11455; GFX11-WGP-NEXT:    s_clause 0x1
11456; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
11457; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
11458; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
11459; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11460; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11461; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
11462; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11463; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
11464; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11465; GFX11-WGP-NEXT:    s_endpgm
11466;
11467; GFX11-CU-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11468; GFX11-CU:       ; %bb.0: ; %entry
11469; GFX11-CU-NEXT:    s_clause 0x1
11470; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
11471; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
11472; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
11473; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11474; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11475; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
11476; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11477; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
11478; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11479; GFX11-CU-NEXT:    s_endpgm
11480    i32 addrspace(1)* %out, i32 %in, i32 %old) {
11481entry:
11482  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
11483  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic acquire
11484  %val0 = extractvalue { i32, i1 } %val, 0
11485  store i32 %val0, i32 addrspace(1)* %out, align 4
11486  ret void
11487}
11488
11489define amdgpu_kernel void @global_singlethread_one_as_acquire_acquire_ret_cmpxchg(
11490; GFX6-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11491; GFX6:       ; %bb.0: ; %entry
11492; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
11493; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11494; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
11495; GFX6-NEXT:    s_mov_b32 s2, -1
11496; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
11497; GFX6-NEXT:    v_mov_b32_e32 v0, s6
11498; GFX6-NEXT:    v_mov_b32_e32 v1, s7
11499; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
11500; GFX6-NEXT:    s_waitcnt vmcnt(0)
11501; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
11502; GFX6-NEXT:    s_endpgm
11503;
11504; GFX7-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11505; GFX7:       ; %bb.0: ; %entry
11506; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11507; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11508; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11509; GFX7-NEXT:    s_add_u32 s4, s0, 16
11510; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11511; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11512; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11513; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11514; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11515; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11516; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11517; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11518; GFX7-NEXT:    s_waitcnt vmcnt(0)
11519; GFX7-NEXT:    flat_store_dword v[0:1], v2
11520; GFX7-NEXT:    s_endpgm
11521;
11522; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11523; GFX10-WGP:       ; %bb.0: ; %entry
11524; GFX10-WGP-NEXT:    s_clause 0x1
11525; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11526; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11527; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
11528; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11529; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11530; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11531; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11532; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11533; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
11534; GFX10-WGP-NEXT:    s_endpgm
11535;
11536; GFX10-CU-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11537; GFX10-CU:       ; %bb.0: ; %entry
11538; GFX10-CU-NEXT:    s_clause 0x1
11539; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11540; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11541; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
11542; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11543; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11544; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11545; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11546; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11547; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
11548; GFX10-CU-NEXT:    s_endpgm
11549;
11550; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11551; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11552; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
11553; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
11554; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
11555; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
11556; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11557; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11558; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11559; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
11560; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11561; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
11562; SKIP-CACHE-INV-NEXT:    s_endpgm
11563;
11564; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11565; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11566; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11567; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11568; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11569; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11570; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11571; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11572; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11573; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
11574; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11575;
11576; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11577; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11578; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11579; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11580; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11581; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11582; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11583; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11584; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11585; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
11586; GFX90A-TGSPLIT-NEXT:    s_endpgm
11587;
11588; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11589; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11590; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
11591; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
11592; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11593; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11594; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11595; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
11596; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11597; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
11598; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11599;
11600; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11601; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11602; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
11603; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
11604; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11605; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11606; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11607; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
11608; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11609; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
11610; GFX940-TGSPLIT-NEXT:    s_endpgm
11611;
11612; GFX11-WGP-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11613; GFX11-WGP:       ; %bb.0: ; %entry
11614; GFX11-WGP-NEXT:    s_clause 0x1
11615; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
11616; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
11617; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
11618; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11619; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11620; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
11621; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11622; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
11623; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11624; GFX11-WGP-NEXT:    s_endpgm
11625;
11626; GFX11-CU-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11627; GFX11-CU:       ; %bb.0: ; %entry
11628; GFX11-CU-NEXT:    s_clause 0x1
11629; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
11630; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
11631; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
11632; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11633; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11634; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
11635; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11636; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
11637; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11638; GFX11-CU-NEXT:    s_endpgm
11639    i32 addrspace(1)* %out, i32 %in, i32 %old) {
11640entry:
11641  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
11642  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire acquire
11643  %val0 = extractvalue { i32, i1 } %val, 0
11644  store i32 %val0, i32 addrspace(1)* %out, align 4
11645  ret void
11646}
11647
11648define amdgpu_kernel void @global_singlethread_one_as_release_acquire_ret_cmpxchg(
11649; GFX6-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
11650; GFX6:       ; %bb.0: ; %entry
11651; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
11652; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11653; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
11654; GFX6-NEXT:    s_mov_b32 s2, -1
11655; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
11656; GFX6-NEXT:    v_mov_b32_e32 v0, s6
11657; GFX6-NEXT:    v_mov_b32_e32 v1, s7
11658; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
11659; GFX6-NEXT:    s_waitcnt vmcnt(0)
11660; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
11661; GFX6-NEXT:    s_endpgm
11662;
11663; GFX7-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
11664; GFX7:       ; %bb.0: ; %entry
11665; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11666; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11667; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11668; GFX7-NEXT:    s_add_u32 s4, s0, 16
11669; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11670; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11671; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11672; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11673; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11674; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11675; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11676; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11677; GFX7-NEXT:    s_waitcnt vmcnt(0)
11678; GFX7-NEXT:    flat_store_dword v[0:1], v2
11679; GFX7-NEXT:    s_endpgm
11680;
11681; GFX10-WGP-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
11682; GFX10-WGP:       ; %bb.0: ; %entry
11683; GFX10-WGP-NEXT:    s_clause 0x1
11684; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11685; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11686; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
11687; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11688; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11689; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11690; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11691; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11692; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
11693; GFX10-WGP-NEXT:    s_endpgm
11694;
11695; GFX10-CU-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
11696; GFX10-CU:       ; %bb.0: ; %entry
11697; GFX10-CU-NEXT:    s_clause 0x1
11698; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11699; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11700; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
11701; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11702; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11703; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11704; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11705; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11706; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
11707; GFX10-CU-NEXT:    s_endpgm
11708;
11709; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
11710; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11711; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
11712; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
11713; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
11714; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
11715; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11716; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11717; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11718; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
11719; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11720; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
11721; SKIP-CACHE-INV-NEXT:    s_endpgm
11722;
11723; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
11724; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11725; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11726; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11727; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11728; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11729; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11730; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11731; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11732; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
11733; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11734;
11735; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
11736; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11737; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11738; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11739; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11740; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11741; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11742; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11743; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11744; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
11745; GFX90A-TGSPLIT-NEXT:    s_endpgm
11746;
11747; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
11748; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11749; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
11750; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
11751; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11752; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11753; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11754; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
11755; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11756; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
11757; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11758;
11759; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
11760; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11761; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
11762; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
11763; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11764; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11765; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11766; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
11767; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11768; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
11769; GFX940-TGSPLIT-NEXT:    s_endpgm
11770;
11771; GFX11-WGP-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
11772; GFX11-WGP:       ; %bb.0: ; %entry
11773; GFX11-WGP-NEXT:    s_clause 0x1
11774; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
11775; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
11776; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
11777; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11778; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11779; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
11780; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11781; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
11782; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11783; GFX11-WGP-NEXT:    s_endpgm
11784;
11785; GFX11-CU-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
11786; GFX11-CU:       ; %bb.0: ; %entry
11787; GFX11-CU-NEXT:    s_clause 0x1
11788; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
11789; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
11790; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
11791; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11792; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11793; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
11794; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11795; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
11796; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11797; GFX11-CU-NEXT:    s_endpgm
11798    i32 addrspace(1)* %out, i32 %in, i32 %old) {
11799entry:
11800  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
11801  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release acquire
11802  %val0 = extractvalue { i32, i1 } %val, 0
11803  store i32 %val0, i32 addrspace(1)* %out, align 4
11804  ret void
11805}
11806
11807define amdgpu_kernel void @global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg(
11808; GFX6-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11809; GFX6:       ; %bb.0: ; %entry
11810; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
11811; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11812; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
11813; GFX6-NEXT:    s_mov_b32 s2, -1
11814; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
11815; GFX6-NEXT:    v_mov_b32_e32 v0, s6
11816; GFX6-NEXT:    v_mov_b32_e32 v1, s7
11817; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
11818; GFX6-NEXT:    s_waitcnt vmcnt(0)
11819; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
11820; GFX6-NEXT:    s_endpgm
11821;
11822; GFX7-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11823; GFX7:       ; %bb.0: ; %entry
11824; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11825; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11826; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11827; GFX7-NEXT:    s_add_u32 s4, s0, 16
11828; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11829; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11830; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11831; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11832; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11833; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11834; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11835; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11836; GFX7-NEXT:    s_waitcnt vmcnt(0)
11837; GFX7-NEXT:    flat_store_dword v[0:1], v2
11838; GFX7-NEXT:    s_endpgm
11839;
11840; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11841; GFX10-WGP:       ; %bb.0: ; %entry
11842; GFX10-WGP-NEXT:    s_clause 0x1
11843; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11844; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11845; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
11846; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11847; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11848; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11849; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11850; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11851; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
11852; GFX10-WGP-NEXT:    s_endpgm
11853;
11854; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11855; GFX10-CU:       ; %bb.0: ; %entry
11856; GFX10-CU-NEXT:    s_clause 0x1
11857; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11858; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11859; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
11860; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11861; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11862; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11863; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11864; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11865; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
11866; GFX10-CU-NEXT:    s_endpgm
11867;
11868; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11869; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11870; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
11871; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
11872; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
11873; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
11874; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11875; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11876; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11877; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
11878; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11879; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
11880; SKIP-CACHE-INV-NEXT:    s_endpgm
11881;
11882; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11883; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11884; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11885; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11886; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11887; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11888; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11889; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11890; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11891; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
11892; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11893;
11894; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11895; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11896; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11897; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11898; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11899; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11900; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11901; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11902; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11903; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
11904; GFX90A-TGSPLIT-NEXT:    s_endpgm
11905;
11906; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11907; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11908; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
11909; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
11910; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11911; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11912; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11913; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
11914; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11915; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
11916; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11917;
11918; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11919; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11920; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
11921; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
11922; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11923; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11924; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11925; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
11926; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11927; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
11928; GFX940-TGSPLIT-NEXT:    s_endpgm
11929;
11930; GFX11-WGP-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11931; GFX11-WGP:       ; %bb.0: ; %entry
11932; GFX11-WGP-NEXT:    s_clause 0x1
11933; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
11934; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
11935; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
11936; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11937; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11938; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
11939; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11940; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
11941; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11942; GFX11-WGP-NEXT:    s_endpgm
11943;
11944; GFX11-CU-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11945; GFX11-CU:       ; %bb.0: ; %entry
11946; GFX11-CU-NEXT:    s_clause 0x1
11947; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
11948; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
11949; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
11950; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11951; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11952; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
11953; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11954; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
11955; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
11956; GFX11-CU-NEXT:    s_endpgm
11957    i32 addrspace(1)* %out, i32 %in, i32 %old) {
11958entry:
11959  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
11960  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel acquire
11961  %val0 = extractvalue { i32, i1 } %val, 0
11962  store i32 %val0, i32 addrspace(1)* %out, align 4
11963  ret void
11964}
11965
11966define amdgpu_kernel void @global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg(
11967; GFX6-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
11968; GFX6:       ; %bb.0: ; %entry
11969; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
11970; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11971; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
11972; GFX6-NEXT:    s_mov_b32 s2, -1
11973; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
11974; GFX6-NEXT:    v_mov_b32_e32 v0, s6
11975; GFX6-NEXT:    v_mov_b32_e32 v1, s7
11976; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
11977; GFX6-NEXT:    s_waitcnt vmcnt(0)
11978; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
11979; GFX6-NEXT:    s_endpgm
11980;
11981; GFX7-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
11982; GFX7:       ; %bb.0: ; %entry
11983; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11984; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11985; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11986; GFX7-NEXT:    s_add_u32 s4, s0, 16
11987; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11988; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11989; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11990; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11991; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11992; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11993; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11994; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11995; GFX7-NEXT:    s_waitcnt vmcnt(0)
11996; GFX7-NEXT:    flat_store_dword v[0:1], v2
11997; GFX7-NEXT:    s_endpgm
11998;
11999; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
12000; GFX10-WGP:       ; %bb.0: ; %entry
12001; GFX10-WGP-NEXT:    s_clause 0x1
12002; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12003; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12004; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
12005; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12006; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12007; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12008; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12009; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12010; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
12011; GFX10-WGP-NEXT:    s_endpgm
12012;
12013; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
12014; GFX10-CU:       ; %bb.0: ; %entry
12015; GFX10-CU-NEXT:    s_clause 0x1
12016; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12017; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12018; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
12019; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12020; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12021; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12022; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12023; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12024; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
12025; GFX10-CU-NEXT:    s_endpgm
12026;
12027; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
12028; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12029; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
12030; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
12031; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
12032; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
12033; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12034; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12035; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12036; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
12037; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12038; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
12039; SKIP-CACHE-INV-NEXT:    s_endpgm
12040;
12041; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
12042; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12043; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12044; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12045; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12046; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12047; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12048; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12049; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12050; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
12051; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12052;
12053; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
12054; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12055; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12056; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12057; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12058; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12059; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12060; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12061; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12062; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
12063; GFX90A-TGSPLIT-NEXT:    s_endpgm
12064;
12065; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
12066; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12067; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
12068; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
12069; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12070; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12071; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12072; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
12073; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12074; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
12075; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12076;
12077; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
12078; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12079; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
12080; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
12081; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12082; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12083; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12084; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
12085; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12086; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
12087; GFX940-TGSPLIT-NEXT:    s_endpgm
12088;
12089; GFX11-WGP-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
12090; GFX11-WGP:       ; %bb.0: ; %entry
12091; GFX11-WGP-NEXT:    s_clause 0x1
12092; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
12093; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
12094; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
12095; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12096; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12097; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
12098; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12099; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
12100; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12101; GFX11-WGP-NEXT:    s_endpgm
12102;
12103; GFX11-CU-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
12104; GFX11-CU:       ; %bb.0: ; %entry
12105; GFX11-CU-NEXT:    s_clause 0x1
12106; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
12107; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
12108; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
12109; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12110; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12111; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
12112; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
12113; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
12114; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12115; GFX11-CU-NEXT:    s_endpgm
12116    i32 addrspace(1)* %out, i32 %in, i32 %old) {
12117entry:
12118  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
12119  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst acquire
12120  %val0 = extractvalue { i32, i1 } %val, 0
12121  store i32 %val0, i32 addrspace(1)* %out, align 4
12122  ret void
12123}
12124
12125define amdgpu_kernel void @global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg(
12126; GFX6-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12127; GFX6:       ; %bb.0: ; %entry
12128; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
12129; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12130; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
12131; GFX6-NEXT:    s_mov_b32 s2, -1
12132; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
12133; GFX6-NEXT:    v_mov_b32_e32 v0, s6
12134; GFX6-NEXT:    v_mov_b32_e32 v1, s7
12135; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
12136; GFX6-NEXT:    s_waitcnt vmcnt(0)
12137; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
12138; GFX6-NEXT:    s_endpgm
12139;
12140; GFX7-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12141; GFX7:       ; %bb.0: ; %entry
12142; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12143; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12144; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12145; GFX7-NEXT:    s_add_u32 s4, s0, 16
12146; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12147; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12148; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12149; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12150; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12151; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12152; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12153; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12154; GFX7-NEXT:    s_waitcnt vmcnt(0)
12155; GFX7-NEXT:    flat_store_dword v[0:1], v2
12156; GFX7-NEXT:    s_endpgm
12157;
12158; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12159; GFX10-WGP:       ; %bb.0: ; %entry
12160; GFX10-WGP-NEXT:    s_clause 0x1
12161; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12162; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12163; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
12164; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12165; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12166; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12167; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12168; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12169; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
12170; GFX10-WGP-NEXT:    s_endpgm
12171;
12172; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12173; GFX10-CU:       ; %bb.0: ; %entry
12174; GFX10-CU-NEXT:    s_clause 0x1
12175; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12176; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12177; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
12178; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12179; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12180; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12181; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12182; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12183; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
12184; GFX10-CU-NEXT:    s_endpgm
12185;
12186; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12187; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12188; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
12189; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
12190; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
12191; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
12192; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12193; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12194; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12195; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
12196; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12197; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
12198; SKIP-CACHE-INV-NEXT:    s_endpgm
12199;
12200; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12201; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12202; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12203; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12204; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12205; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12206; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12207; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12208; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12209; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
12210; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12211;
12212; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12213; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12214; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12215; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12216; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12217; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12218; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12219; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12220; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12221; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
12222; GFX90A-TGSPLIT-NEXT:    s_endpgm
12223;
12224; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12225; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12226; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
12227; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
12228; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12229; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12230; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12231; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
12232; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12233; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
12234; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12235;
12236; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12237; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12238; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
12239; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
12240; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12241; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12242; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12243; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
12244; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12245; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
12246; GFX940-TGSPLIT-NEXT:    s_endpgm
12247;
12248; GFX11-WGP-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12249; GFX11-WGP:       ; %bb.0: ; %entry
12250; GFX11-WGP-NEXT:    s_clause 0x1
12251; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
12252; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
12253; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
12254; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12255; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12256; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
12257; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12258; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
12259; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12260; GFX11-WGP-NEXT:    s_endpgm
12261;
12262; GFX11-CU-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12263; GFX11-CU:       ; %bb.0: ; %entry
12264; GFX11-CU-NEXT:    s_clause 0x1
12265; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
12266; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
12267; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
12268; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12269; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12270; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
12271; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
12272; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
12273; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12274; GFX11-CU-NEXT:    s_endpgm
12275    i32 addrspace(1)* %out, i32 %in, i32 %old) {
12276entry:
12277  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
12278  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic seq_cst
12279  %val0 = extractvalue { i32, i1 } %val, 0
12280  store i32 %val0, i32 addrspace(1)* %out, align 4
12281  ret void
12282}
12283
12284define amdgpu_kernel void @global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg(
12285; GFX6-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12286; GFX6:       ; %bb.0: ; %entry
12287; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
12288; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12289; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
12290; GFX6-NEXT:    s_mov_b32 s2, -1
12291; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
12292; GFX6-NEXT:    v_mov_b32_e32 v0, s6
12293; GFX6-NEXT:    v_mov_b32_e32 v1, s7
12294; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
12295; GFX6-NEXT:    s_waitcnt vmcnt(0)
12296; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
12297; GFX6-NEXT:    s_endpgm
12298;
12299; GFX7-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12300; GFX7:       ; %bb.0: ; %entry
12301; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12302; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12303; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12304; GFX7-NEXT:    s_add_u32 s4, s0, 16
12305; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12306; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12307; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12308; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12309; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12310; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12311; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12312; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12313; GFX7-NEXT:    s_waitcnt vmcnt(0)
12314; GFX7-NEXT:    flat_store_dword v[0:1], v2
12315; GFX7-NEXT:    s_endpgm
12316;
12317; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12318; GFX10-WGP:       ; %bb.0: ; %entry
12319; GFX10-WGP-NEXT:    s_clause 0x1
12320; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12321; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12322; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
12323; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12324; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12325; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12326; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12327; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12328; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
12329; GFX10-WGP-NEXT:    s_endpgm
12330;
12331; GFX10-CU-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12332; GFX10-CU:       ; %bb.0: ; %entry
12333; GFX10-CU-NEXT:    s_clause 0x1
12334; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12335; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12336; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
12337; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12338; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12339; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12340; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12341; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12342; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
12343; GFX10-CU-NEXT:    s_endpgm
12344;
12345; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12346; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12347; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
12348; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
12349; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
12350; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
12351; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12352; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12353; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12354; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
12355; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12356; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
12357; SKIP-CACHE-INV-NEXT:    s_endpgm
12358;
12359; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12360; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12361; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12362; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12363; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12364; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12365; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12366; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12367; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12368; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
12369; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12370;
12371; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12372; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12373; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12374; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12375; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12376; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12377; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12378; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12379; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12380; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
12381; GFX90A-TGSPLIT-NEXT:    s_endpgm
12382;
12383; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12384; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12385; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
12386; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
12387; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12388; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12389; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12390; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
12391; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12392; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
12393; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12394;
12395; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12396; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12397; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
12398; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
12399; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12400; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12401; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12402; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
12403; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12404; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
12405; GFX940-TGSPLIT-NEXT:    s_endpgm
12406;
12407; GFX11-WGP-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12408; GFX11-WGP:       ; %bb.0: ; %entry
12409; GFX11-WGP-NEXT:    s_clause 0x1
12410; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
12411; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
12412; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
12413; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12414; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12415; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
12416; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12417; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
12418; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12419; GFX11-WGP-NEXT:    s_endpgm
12420;
12421; GFX11-CU-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12422; GFX11-CU:       ; %bb.0: ; %entry
12423; GFX11-CU-NEXT:    s_clause 0x1
12424; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
12425; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
12426; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
12427; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12428; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12429; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
12430; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
12431; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
12432; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12433; GFX11-CU-NEXT:    s_endpgm
12434    i32 addrspace(1)* %out, i32 %in, i32 %old) {
12435entry:
12436  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
12437  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire seq_cst
12438  %val0 = extractvalue { i32, i1 } %val, 0
12439  store i32 %val0, i32 addrspace(1)* %out, align 4
12440  ret void
12441}
12442
12443define amdgpu_kernel void @global_singlethread_one_as_release_seq_cst_ret_cmpxchg(
12444; GFX6-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12445; GFX6:       ; %bb.0: ; %entry
12446; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
12447; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12448; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
12449; GFX6-NEXT:    s_mov_b32 s2, -1
12450; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
12451; GFX6-NEXT:    v_mov_b32_e32 v0, s6
12452; GFX6-NEXT:    v_mov_b32_e32 v1, s7
12453; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
12454; GFX6-NEXT:    s_waitcnt vmcnt(0)
12455; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
12456; GFX6-NEXT:    s_endpgm
12457;
12458; GFX7-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12459; GFX7:       ; %bb.0: ; %entry
12460; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12461; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12462; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12463; GFX7-NEXT:    s_add_u32 s4, s0, 16
12464; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12465; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12466; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12467; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12468; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12469; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12470; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12471; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12472; GFX7-NEXT:    s_waitcnt vmcnt(0)
12473; GFX7-NEXT:    flat_store_dword v[0:1], v2
12474; GFX7-NEXT:    s_endpgm
12475;
12476; GFX10-WGP-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12477; GFX10-WGP:       ; %bb.0: ; %entry
12478; GFX10-WGP-NEXT:    s_clause 0x1
12479; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12480; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12481; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
12482; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12483; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12484; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12485; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12486; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12487; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
12488; GFX10-WGP-NEXT:    s_endpgm
12489;
12490; GFX10-CU-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12491; GFX10-CU:       ; %bb.0: ; %entry
12492; GFX10-CU-NEXT:    s_clause 0x1
12493; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12494; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12495; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
12496; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12497; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12498; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12499; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12500; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12501; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
12502; GFX10-CU-NEXT:    s_endpgm
12503;
12504; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12505; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12506; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
12507; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
12508; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
12509; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
12510; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12511; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12512; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12513; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
12514; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12515; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
12516; SKIP-CACHE-INV-NEXT:    s_endpgm
12517;
12518; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12519; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12520; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12521; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12522; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12523; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12524; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12525; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12526; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12527; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
12528; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12529;
12530; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12531; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12532; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12533; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12534; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12535; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12536; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12537; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12538; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12539; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
12540; GFX90A-TGSPLIT-NEXT:    s_endpgm
12541;
12542; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12543; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12544; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
12545; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
12546; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12547; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12548; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12549; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
12550; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12551; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
12552; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12553;
12554; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12555; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12556; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
12557; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
12558; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12559; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12560; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12561; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
12562; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12563; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
12564; GFX940-TGSPLIT-NEXT:    s_endpgm
12565;
12566; GFX11-WGP-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12567; GFX11-WGP:       ; %bb.0: ; %entry
12568; GFX11-WGP-NEXT:    s_clause 0x1
12569; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
12570; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
12571; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
12572; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12573; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12574; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
12575; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12576; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
12577; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12578; GFX11-WGP-NEXT:    s_endpgm
12579;
12580; GFX11-CU-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12581; GFX11-CU:       ; %bb.0: ; %entry
12582; GFX11-CU-NEXT:    s_clause 0x1
12583; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
12584; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
12585; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
12586; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12587; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12588; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
12589; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
12590; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
12591; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12592; GFX11-CU-NEXT:    s_endpgm
12593    i32 addrspace(1)* %out, i32 %in, i32 %old) {
12594entry:
12595  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
12596  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release seq_cst
12597  %val0 = extractvalue { i32, i1 } %val, 0
12598  store i32 %val0, i32 addrspace(1)* %out, align 4
12599  ret void
12600}
12601
12602define amdgpu_kernel void @global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg(
12603; GFX6-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12604; GFX6:       ; %bb.0: ; %entry
12605; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
12606; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12607; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
12608; GFX6-NEXT:    s_mov_b32 s2, -1
12609; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
12610; GFX6-NEXT:    v_mov_b32_e32 v0, s6
12611; GFX6-NEXT:    v_mov_b32_e32 v1, s7
12612; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
12613; GFX6-NEXT:    s_waitcnt vmcnt(0)
12614; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
12615; GFX6-NEXT:    s_endpgm
12616;
12617; GFX7-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12618; GFX7:       ; %bb.0: ; %entry
12619; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12620; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12621; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12622; GFX7-NEXT:    s_add_u32 s4, s0, 16
12623; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12624; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12625; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12626; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12627; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12628; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12629; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12630; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12631; GFX7-NEXT:    s_waitcnt vmcnt(0)
12632; GFX7-NEXT:    flat_store_dword v[0:1], v2
12633; GFX7-NEXT:    s_endpgm
12634;
12635; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12636; GFX10-WGP:       ; %bb.0: ; %entry
12637; GFX10-WGP-NEXT:    s_clause 0x1
12638; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12639; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12640; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
12641; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12642; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12643; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12644; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12645; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12646; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
12647; GFX10-WGP-NEXT:    s_endpgm
12648;
12649; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12650; GFX10-CU:       ; %bb.0: ; %entry
12651; GFX10-CU-NEXT:    s_clause 0x1
12652; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12653; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12654; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
12655; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12656; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12657; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12658; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12659; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12660; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
12661; GFX10-CU-NEXT:    s_endpgm
12662;
12663; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12664; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12665; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
12666; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
12667; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
12668; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
12669; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12670; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12671; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12672; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
12673; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12674; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
12675; SKIP-CACHE-INV-NEXT:    s_endpgm
12676;
12677; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12678; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12679; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12680; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12681; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12682; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12683; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12684; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12685; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12686; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
12687; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12688;
12689; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12690; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12691; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12692; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12693; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12694; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12695; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12696; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12697; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12698; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
12699; GFX90A-TGSPLIT-NEXT:    s_endpgm
12700;
12701; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12702; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12703; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
12704; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
12705; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12706; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12707; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12708; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
12709; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12710; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
12711; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12712;
12713; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12714; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12715; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
12716; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
12717; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12718; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12719; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12720; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
12721; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12722; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
12723; GFX940-TGSPLIT-NEXT:    s_endpgm
12724;
12725; GFX11-WGP-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12726; GFX11-WGP:       ; %bb.0: ; %entry
12727; GFX11-WGP-NEXT:    s_clause 0x1
12728; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
12729; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
12730; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
12731; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12732; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12733; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
12734; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12735; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
12736; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12737; GFX11-WGP-NEXT:    s_endpgm
12738;
12739; GFX11-CU-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12740; GFX11-CU:       ; %bb.0: ; %entry
12741; GFX11-CU-NEXT:    s_clause 0x1
12742; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
12743; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
12744; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
12745; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12746; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12747; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
12748; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
12749; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
12750; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12751; GFX11-CU-NEXT:    s_endpgm
12752    i32 addrspace(1)* %out, i32 %in, i32 %old) {
12753entry:
12754  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
12755  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel seq_cst
12756  %val0 = extractvalue { i32, i1 } %val, 0
12757  store i32 %val0, i32 addrspace(1)* %out, align 4
12758  ret void
12759}
12760
12761define amdgpu_kernel void @global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg(
12762; GFX6-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12763; GFX6:       ; %bb.0: ; %entry
12764; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
12765; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12766; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
12767; GFX6-NEXT:    s_mov_b32 s2, -1
12768; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
12769; GFX6-NEXT:    v_mov_b32_e32 v0, s6
12770; GFX6-NEXT:    v_mov_b32_e32 v1, s7
12771; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
12772; GFX6-NEXT:    s_waitcnt vmcnt(0)
12773; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
12774; GFX6-NEXT:    s_endpgm
12775;
12776; GFX7-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12777; GFX7:       ; %bb.0: ; %entry
12778; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12779; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12780; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12781; GFX7-NEXT:    s_add_u32 s4, s0, 16
12782; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12783; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12784; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12785; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12786; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12787; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12788; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12789; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12790; GFX7-NEXT:    s_waitcnt vmcnt(0)
12791; GFX7-NEXT:    flat_store_dword v[0:1], v2
12792; GFX7-NEXT:    s_endpgm
12793;
12794; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12795; GFX10-WGP:       ; %bb.0: ; %entry
12796; GFX10-WGP-NEXT:    s_clause 0x1
12797; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12798; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12799; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
12800; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12801; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12802; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12803; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12804; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12805; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
12806; GFX10-WGP-NEXT:    s_endpgm
12807;
12808; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12809; GFX10-CU:       ; %bb.0: ; %entry
12810; GFX10-CU-NEXT:    s_clause 0x1
12811; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12812; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12813; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
12814; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12815; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12816; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12817; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12818; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12819; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
12820; GFX10-CU-NEXT:    s_endpgm
12821;
12822; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12823; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12824; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
12825; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
12826; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
12827; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
12828; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12829; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12830; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12831; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
12832; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12833; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
12834; SKIP-CACHE-INV-NEXT:    s_endpgm
12835;
12836; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12837; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12838; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12839; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12840; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12841; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12842; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12843; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12844; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12845; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
12846; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12847;
12848; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12849; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12850; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12851; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12852; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12853; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12854; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12855; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12856; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12857; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
12858; GFX90A-TGSPLIT-NEXT:    s_endpgm
12859;
12860; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12861; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12862; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
12863; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
12864; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12865; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12866; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12867; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
12868; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12869; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
12870; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12871;
12872; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12873; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12874; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
12875; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
12876; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12877; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12878; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12879; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
12880; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12881; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
12882; GFX940-TGSPLIT-NEXT:    s_endpgm
12883;
12884; GFX11-WGP-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12885; GFX11-WGP:       ; %bb.0: ; %entry
12886; GFX11-WGP-NEXT:    s_clause 0x1
12887; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
12888; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
12889; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
12890; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12891; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12892; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
12893; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12894; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
12895; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12896; GFX11-WGP-NEXT:    s_endpgm
12897;
12898; GFX11-CU-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12899; GFX11-CU:       ; %bb.0: ; %entry
12900; GFX11-CU-NEXT:    s_clause 0x1
12901; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
12902; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
12903; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
12904; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12905; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12906; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
12907; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
12908; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
12909; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12910; GFX11-CU-NEXT:    s_endpgm
12911    i32 addrspace(1)* %out, i32 %in, i32 %old) {
12912entry:
12913  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
12914  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst seq_cst
12915  %val0 = extractvalue { i32, i1 } %val, 0
12916  store i32 %val0, i32 addrspace(1)* %out, align 4
12917  ret void
12918}
12919
12920