1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx600 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX6 %s
3; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
4; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
5; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
6; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
7; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-NOTTGSPLIT %s
8; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-TGSPLIT %s
9; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-NOTTGSPLIT %s
10; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
11; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11-WGP %s
12; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11-CU %s
13
14define amdgpu_kernel void @global_singlethread_unordered_load(
15; GFX6-LABEL: global_singlethread_unordered_load:
16; GFX6:       ; %bb.0: ; %entry
17; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
18; GFX6-NEXT:    s_mov_b32 s7, 0x100f000
19; GFX6-NEXT:    s_mov_b32 s6, -1
20; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
21; GFX6-NEXT:    s_mov_b32 s4, s0
22; GFX6-NEXT:    s_mov_b32 s5, s1
23; GFX6-NEXT:    buffer_load_dword v0, off, s[4:7], 0
24; GFX6-NEXT:    s_mov_b32 s4, s2
25; GFX6-NEXT:    s_mov_b32 s5, s3
26; GFX6-NEXT:    s_waitcnt vmcnt(0)
27; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
28; GFX6-NEXT:    s_endpgm
29;
30; GFX7-LABEL: global_singlethread_unordered_load:
31; GFX7:       ; %bb.0: ; %entry
32; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
33; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
34; GFX7-NEXT:    v_mov_b32_e32 v0, s0
35; GFX7-NEXT:    v_mov_b32_e32 v1, s1
36; GFX7-NEXT:    flat_load_dword v2, v[0:1]
37; GFX7-NEXT:    v_mov_b32_e32 v0, s2
38; GFX7-NEXT:    v_mov_b32_e32 v1, s3
39; GFX7-NEXT:    s_waitcnt vmcnt(0)
40; GFX7-NEXT:    flat_store_dword v[0:1], v2
41; GFX7-NEXT:    s_endpgm
42;
43; GFX10-WGP-LABEL: global_singlethread_unordered_load:
44; GFX10-WGP:       ; %bb.0: ; %entry
45; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
46; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
47; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
48; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
49; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
50; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
51; GFX10-WGP-NEXT:    s_endpgm
52;
53; GFX10-CU-LABEL: global_singlethread_unordered_load:
54; GFX10-CU:       ; %bb.0: ; %entry
55; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
56; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
57; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
58; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
59; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
60; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
61; GFX10-CU-NEXT:    s_endpgm
62;
63; SKIP-CACHE-INV-LABEL: global_singlethread_unordered_load:
64; SKIP-CACHE-INV:       ; %bb.0: ; %entry
65; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
66; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
67; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
68; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
69; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s0
70; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s1
71; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[4:7], 0
72; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s2
73; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s3
74; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
75; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
76; SKIP-CACHE-INV-NEXT:    s_endpgm
77;
78; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_unordered_load:
79; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
80; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
81; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
82; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
83; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
84; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
85; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
86; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
87;
88; GFX90A-TGSPLIT-LABEL: global_singlethread_unordered_load:
89; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
90; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
91; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
92; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
93; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
94; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
95; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
96; GFX90A-TGSPLIT-NEXT:    s_endpgm
97;
98; GFX940-NOTTGSPLIT-LABEL: global_singlethread_unordered_load:
99; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
100; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
101; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
102; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
103; GFX940-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
104; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
105; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
106; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
107;
108; GFX940-TGSPLIT-LABEL: global_singlethread_unordered_load:
109; GFX940-TGSPLIT:       ; %bb.0: ; %entry
110; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
111; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
112; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
113; GFX940-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
114; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
115; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
116; GFX940-TGSPLIT-NEXT:    s_endpgm
117;
118; GFX11-WGP-LABEL: global_singlethread_unordered_load:
119; GFX11-WGP:       ; %bb.0: ; %entry
120; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
121; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
122; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
123; GFX11-WGP-NEXT:    global_load_b32 v1, v0, s[0:1]
124; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
125; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[2:3]
126; GFX11-WGP-NEXT:    s_endpgm
127;
128; GFX11-CU-LABEL: global_singlethread_unordered_load:
129; GFX11-CU:       ; %bb.0: ; %entry
130; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
131; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
132; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
133; GFX11-CU-NEXT:    global_load_b32 v1, v0, s[0:1]
134; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
135; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[2:3]
136; GFX11-CU-NEXT:    s_endpgm
137    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
138entry:
139  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread") unordered, align 4
140  store i32 %val, i32 addrspace(1)* %out
141  ret void
142}
143
144define amdgpu_kernel void @global_singlethread_monotonic_load(
145; GFX6-LABEL: global_singlethread_monotonic_load:
146; GFX6:       ; %bb.0: ; %entry
147; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
148; GFX6-NEXT:    s_mov_b32 s7, 0x100f000
149; GFX6-NEXT:    s_mov_b32 s6, -1
150; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
151; GFX6-NEXT:    s_mov_b32 s4, s0
152; GFX6-NEXT:    s_mov_b32 s5, s1
153; GFX6-NEXT:    buffer_load_dword v0, off, s[4:7], 0
154; GFX6-NEXT:    s_mov_b32 s4, s2
155; GFX6-NEXT:    s_mov_b32 s5, s3
156; GFX6-NEXT:    s_waitcnt vmcnt(0)
157; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
158; GFX6-NEXT:    s_endpgm
159;
160; GFX7-LABEL: global_singlethread_monotonic_load:
161; GFX7:       ; %bb.0: ; %entry
162; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
163; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
164; GFX7-NEXT:    v_mov_b32_e32 v0, s0
165; GFX7-NEXT:    v_mov_b32_e32 v1, s1
166; GFX7-NEXT:    flat_load_dword v2, v[0:1]
167; GFX7-NEXT:    v_mov_b32_e32 v0, s2
168; GFX7-NEXT:    v_mov_b32_e32 v1, s3
169; GFX7-NEXT:    s_waitcnt vmcnt(0)
170; GFX7-NEXT:    flat_store_dword v[0:1], v2
171; GFX7-NEXT:    s_endpgm
172;
173; GFX10-WGP-LABEL: global_singlethread_monotonic_load:
174; GFX10-WGP:       ; %bb.0: ; %entry
175; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
176; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
177; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
178; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
179; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
180; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
181; GFX10-WGP-NEXT:    s_endpgm
182;
183; GFX10-CU-LABEL: global_singlethread_monotonic_load:
184; GFX10-CU:       ; %bb.0: ; %entry
185; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
186; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
187; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
188; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
189; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
190; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
191; GFX10-CU-NEXT:    s_endpgm
192;
193; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_load:
194; SKIP-CACHE-INV:       ; %bb.0: ; %entry
195; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
196; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
197; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
198; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
199; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s0
200; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s1
201; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[4:7], 0
202; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s2
203; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s3
204; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
205; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
206; SKIP-CACHE-INV-NEXT:    s_endpgm
207;
208; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_load:
209; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
210; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
211; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
212; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
213; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
214; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
215; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
216; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
217;
218; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_load:
219; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
220; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
221; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
222; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
223; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
224; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
225; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
226; GFX90A-TGSPLIT-NEXT:    s_endpgm
227;
228; GFX940-NOTTGSPLIT-LABEL: global_singlethread_monotonic_load:
229; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
230; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
231; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
232; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
233; GFX940-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
234; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
235; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
236; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
237;
238; GFX940-TGSPLIT-LABEL: global_singlethread_monotonic_load:
239; GFX940-TGSPLIT:       ; %bb.0: ; %entry
240; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
241; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
242; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
243; GFX940-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
244; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
245; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
246; GFX940-TGSPLIT-NEXT:    s_endpgm
247;
248; GFX11-WGP-LABEL: global_singlethread_monotonic_load:
249; GFX11-WGP:       ; %bb.0: ; %entry
250; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
251; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
252; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
253; GFX11-WGP-NEXT:    global_load_b32 v1, v0, s[0:1]
254; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
255; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[2:3]
256; GFX11-WGP-NEXT:    s_endpgm
257;
258; GFX11-CU-LABEL: global_singlethread_monotonic_load:
259; GFX11-CU:       ; %bb.0: ; %entry
260; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
261; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
262; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
263; GFX11-CU-NEXT:    global_load_b32 v1, v0, s[0:1]
264; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
265; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[2:3]
266; GFX11-CU-NEXT:    s_endpgm
267    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
268entry:
269  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread") monotonic, align 4
270  store i32 %val, i32 addrspace(1)* %out
271  ret void
272}
273
274define amdgpu_kernel void @global_singlethread_acquire_load(
275; GFX6-LABEL: global_singlethread_acquire_load:
276; GFX6:       ; %bb.0: ; %entry
277; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
278; GFX6-NEXT:    s_mov_b32 s7, 0x100f000
279; GFX6-NEXT:    s_mov_b32 s6, -1
280; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
281; GFX6-NEXT:    s_mov_b32 s4, s0
282; GFX6-NEXT:    s_mov_b32 s5, s1
283; GFX6-NEXT:    buffer_load_dword v0, off, s[4:7], 0
284; GFX6-NEXT:    s_mov_b32 s4, s2
285; GFX6-NEXT:    s_mov_b32 s5, s3
286; GFX6-NEXT:    s_waitcnt vmcnt(0)
287; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
288; GFX6-NEXT:    s_endpgm
289;
290; GFX7-LABEL: global_singlethread_acquire_load:
291; GFX7:       ; %bb.0: ; %entry
292; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
293; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
294; GFX7-NEXT:    v_mov_b32_e32 v0, s0
295; GFX7-NEXT:    v_mov_b32_e32 v1, s1
296; GFX7-NEXT:    flat_load_dword v2, v[0:1]
297; GFX7-NEXT:    v_mov_b32_e32 v0, s2
298; GFX7-NEXT:    v_mov_b32_e32 v1, s3
299; GFX7-NEXT:    s_waitcnt vmcnt(0)
300; GFX7-NEXT:    flat_store_dword v[0:1], v2
301; GFX7-NEXT:    s_endpgm
302;
303; GFX10-WGP-LABEL: global_singlethread_acquire_load:
304; GFX10-WGP:       ; %bb.0: ; %entry
305; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
306; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
307; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
308; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
309; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
310; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
311; GFX10-WGP-NEXT:    s_endpgm
312;
313; GFX10-CU-LABEL: global_singlethread_acquire_load:
314; GFX10-CU:       ; %bb.0: ; %entry
315; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
316; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
317; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
318; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
319; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
320; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
321; GFX10-CU-NEXT:    s_endpgm
322;
323; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_load:
324; SKIP-CACHE-INV:       ; %bb.0: ; %entry
325; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
326; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
327; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
328; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
329; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s0
330; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s1
331; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[4:7], 0
332; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s2
333; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s3
334; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
335; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
336; SKIP-CACHE-INV-NEXT:    s_endpgm
337;
338; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_load:
339; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
340; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
341; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
342; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
343; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
344; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
345; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
346; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
347;
348; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_load:
349; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
350; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
351; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
352; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
353; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
354; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
355; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
356; GFX90A-TGSPLIT-NEXT:    s_endpgm
357;
358; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acquire_load:
359; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
360; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
361; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
362; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
363; GFX940-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
364; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
365; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
366; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
367;
368; GFX940-TGSPLIT-LABEL: global_singlethread_acquire_load:
369; GFX940-TGSPLIT:       ; %bb.0: ; %entry
370; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
371; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
372; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
373; GFX940-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
374; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
375; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
376; GFX940-TGSPLIT-NEXT:    s_endpgm
377;
378; GFX11-WGP-LABEL: global_singlethread_acquire_load:
379; GFX11-WGP:       ; %bb.0: ; %entry
380; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
381; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
382; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
383; GFX11-WGP-NEXT:    global_load_b32 v1, v0, s[0:1]
384; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
385; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[2:3]
386; GFX11-WGP-NEXT:    s_endpgm
387;
388; GFX11-CU-LABEL: global_singlethread_acquire_load:
389; GFX11-CU:       ; %bb.0: ; %entry
390; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
391; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
392; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
393; GFX11-CU-NEXT:    global_load_b32 v1, v0, s[0:1]
394; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
395; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[2:3]
396; GFX11-CU-NEXT:    s_endpgm
397    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
398entry:
399  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread") acquire, align 4
400  store i32 %val, i32 addrspace(1)* %out
401  ret void
402}
403
404define amdgpu_kernel void @global_singlethread_seq_cst_load(
405; GFX6-LABEL: global_singlethread_seq_cst_load:
406; GFX6:       ; %bb.0: ; %entry
407; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
408; GFX6-NEXT:    s_mov_b32 s7, 0x100f000
409; GFX6-NEXT:    s_mov_b32 s6, -1
410; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
411; GFX6-NEXT:    s_mov_b32 s4, s0
412; GFX6-NEXT:    s_mov_b32 s5, s1
413; GFX6-NEXT:    buffer_load_dword v0, off, s[4:7], 0
414; GFX6-NEXT:    s_mov_b32 s4, s2
415; GFX6-NEXT:    s_mov_b32 s5, s3
416; GFX6-NEXT:    s_waitcnt vmcnt(0)
417; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
418; GFX6-NEXT:    s_endpgm
419;
420; GFX7-LABEL: global_singlethread_seq_cst_load:
421; GFX7:       ; %bb.0: ; %entry
422; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
423; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
424; GFX7-NEXT:    v_mov_b32_e32 v0, s0
425; GFX7-NEXT:    v_mov_b32_e32 v1, s1
426; GFX7-NEXT:    flat_load_dword v2, v[0:1]
427; GFX7-NEXT:    v_mov_b32_e32 v0, s2
428; GFX7-NEXT:    v_mov_b32_e32 v1, s3
429; GFX7-NEXT:    s_waitcnt vmcnt(0)
430; GFX7-NEXT:    flat_store_dword v[0:1], v2
431; GFX7-NEXT:    s_endpgm
432;
433; GFX10-WGP-LABEL: global_singlethread_seq_cst_load:
434; GFX10-WGP:       ; %bb.0: ; %entry
435; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
436; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
437; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
438; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
439; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
440; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
441; GFX10-WGP-NEXT:    s_endpgm
442;
443; GFX10-CU-LABEL: global_singlethread_seq_cst_load:
444; GFX10-CU:       ; %bb.0: ; %entry
445; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
446; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
447; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
448; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
449; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
450; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
451; GFX10-CU-NEXT:    s_endpgm
452;
453; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_load:
454; SKIP-CACHE-INV:       ; %bb.0: ; %entry
455; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
456; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
457; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
458; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
459; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s0
460; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s1
461; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[4:7], 0
462; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s2
463; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s3
464; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
465; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
466; SKIP-CACHE-INV-NEXT:    s_endpgm
467;
468; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_load:
469; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
470; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
471; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
472; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
473; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
474; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
475; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
476; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
477;
478; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_load:
479; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
480; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
481; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
482; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
483; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
484; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
485; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
486; GFX90A-TGSPLIT-NEXT:    s_endpgm
487;
488; GFX940-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_load:
489; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
490; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
491; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
492; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
493; GFX940-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
494; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
495; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
496; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
497;
498; GFX940-TGSPLIT-LABEL: global_singlethread_seq_cst_load:
499; GFX940-TGSPLIT:       ; %bb.0: ; %entry
500; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
501; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
502; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
503; GFX940-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
504; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
505; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
506; GFX940-TGSPLIT-NEXT:    s_endpgm
507;
508; GFX11-WGP-LABEL: global_singlethread_seq_cst_load:
509; GFX11-WGP:       ; %bb.0: ; %entry
510; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
511; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
512; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
513; GFX11-WGP-NEXT:    global_load_b32 v1, v0, s[0:1]
514; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
515; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[2:3]
516; GFX11-WGP-NEXT:    s_endpgm
517;
518; GFX11-CU-LABEL: global_singlethread_seq_cst_load:
519; GFX11-CU:       ; %bb.0: ; %entry
520; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
521; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
522; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
523; GFX11-CU-NEXT:    global_load_b32 v1, v0, s[0:1]
524; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
525; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[2:3]
526; GFX11-CU-NEXT:    s_endpgm
527    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
528entry:
529  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread") seq_cst, align 4
530  store i32 %val, i32 addrspace(1)* %out
531  ret void
532}
533
534define amdgpu_kernel void @global_singlethread_unordered_store(
535; GFX6-LABEL: global_singlethread_unordered_store:
536; GFX6:       ; %bb.0: ; %entry
537; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
538; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
539; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
540; GFX6-NEXT:    s_mov_b32 s2, -1
541; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
542; GFX6-NEXT:    v_mov_b32_e32 v0, s6
543; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
544; GFX6-NEXT:    s_endpgm
545;
546; GFX7-LABEL: global_singlethread_unordered_store:
547; GFX7:       ; %bb.0: ; %entry
548; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
549; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
550; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
551; GFX7-NEXT:    v_mov_b32_e32 v0, s0
552; GFX7-NEXT:    v_mov_b32_e32 v1, s1
553; GFX7-NEXT:    v_mov_b32_e32 v2, s2
554; GFX7-NEXT:    flat_store_dword v[0:1], v2
555; GFX7-NEXT:    s_endpgm
556;
557; GFX10-WGP-LABEL: global_singlethread_unordered_store:
558; GFX10-WGP:       ; %bb.0: ; %entry
559; GFX10-WGP-NEXT:    s_clause 0x1
560; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
561; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
562; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
563; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
564; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
565; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
566; GFX10-WGP-NEXT:    s_endpgm
567;
568; GFX10-CU-LABEL: global_singlethread_unordered_store:
569; GFX10-CU:       ; %bb.0: ; %entry
570; GFX10-CU-NEXT:    s_clause 0x1
571; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
572; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
573; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
574; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
575; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
576; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
577; GFX10-CU-NEXT:    s_endpgm
578;
579; SKIP-CACHE-INV-LABEL: global_singlethread_unordered_store:
580; SKIP-CACHE-INV:       ; %bb.0: ; %entry
581; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x0
582; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
583; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
584; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
585; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
586; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
587; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
588; SKIP-CACHE-INV-NEXT:    s_endpgm
589;
590; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_unordered_store:
591; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
592; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
593; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
594; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
595; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
596; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
597; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
598; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
599;
600; GFX90A-TGSPLIT-LABEL: global_singlethread_unordered_store:
601; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
602; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
603; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
604; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
605; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
606; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
607; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
608; GFX90A-TGSPLIT-NEXT:    s_endpgm
609;
610; GFX940-NOTTGSPLIT-LABEL: global_singlethread_unordered_store:
611; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
612; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
613; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
614; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
615; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
616; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
617; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
618; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
619;
620; GFX940-TGSPLIT-LABEL: global_singlethread_unordered_store:
621; GFX940-TGSPLIT:       ; %bb.0: ; %entry
622; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
623; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
624; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
625; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
626; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
627; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
628; GFX940-TGSPLIT-NEXT:    s_endpgm
629;
630; GFX11-WGP-LABEL: global_singlethread_unordered_store:
631; GFX11-WGP:       ; %bb.0: ; %entry
632; GFX11-WGP-NEXT:    s_clause 0x1
633; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x0
634; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
635; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
636; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
637; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s2
638; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[0:1]
639; GFX11-WGP-NEXT:    s_endpgm
640;
641; GFX11-CU-LABEL: global_singlethread_unordered_store:
642; GFX11-CU:       ; %bb.0: ; %entry
643; GFX11-CU-NEXT:    s_clause 0x1
644; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x0
645; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
646; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
647; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
648; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s2
649; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
650; GFX11-CU-NEXT:    s_endpgm
651    i32 %in, i32 addrspace(1)* %out) {
652entry:
653  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread") unordered, align 4
654  ret void
655}
656
657define amdgpu_kernel void @global_singlethread_monotonic_store(
658; GFX6-LABEL: global_singlethread_monotonic_store:
659; GFX6:       ; %bb.0: ; %entry
660; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
661; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
662; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
663; GFX6-NEXT:    s_mov_b32 s2, -1
664; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
665; GFX6-NEXT:    v_mov_b32_e32 v0, s6
666; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
667; GFX6-NEXT:    s_endpgm
668;
669; GFX7-LABEL: global_singlethread_monotonic_store:
670; GFX7:       ; %bb.0: ; %entry
671; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
672; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
673; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
674; GFX7-NEXT:    v_mov_b32_e32 v0, s0
675; GFX7-NEXT:    v_mov_b32_e32 v1, s1
676; GFX7-NEXT:    v_mov_b32_e32 v2, s2
677; GFX7-NEXT:    flat_store_dword v[0:1], v2
678; GFX7-NEXT:    s_endpgm
679;
680; GFX10-WGP-LABEL: global_singlethread_monotonic_store:
681; GFX10-WGP:       ; %bb.0: ; %entry
682; GFX10-WGP-NEXT:    s_clause 0x1
683; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
684; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
685; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
686; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
687; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
688; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
689; GFX10-WGP-NEXT:    s_endpgm
690;
691; GFX10-CU-LABEL: global_singlethread_monotonic_store:
692; GFX10-CU:       ; %bb.0: ; %entry
693; GFX10-CU-NEXT:    s_clause 0x1
694; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
695; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
696; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
697; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
698; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
699; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
700; GFX10-CU-NEXT:    s_endpgm
701;
702; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_store:
703; SKIP-CACHE-INV:       ; %bb.0: ; %entry
704; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x0
705; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
706; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
707; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
708; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
709; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
710; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
711; SKIP-CACHE-INV-NEXT:    s_endpgm
712;
713; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_store:
714; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
715; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
716; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
717; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
718; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
719; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
720; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
721; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
722;
723; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_store:
724; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
725; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
726; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
727; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
728; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
729; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
730; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
731; GFX90A-TGSPLIT-NEXT:    s_endpgm
732;
733; GFX940-NOTTGSPLIT-LABEL: global_singlethread_monotonic_store:
734; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
735; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
736; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
737; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
738; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
739; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
740; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
741; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
742;
743; GFX940-TGSPLIT-LABEL: global_singlethread_monotonic_store:
744; GFX940-TGSPLIT:       ; %bb.0: ; %entry
745; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
746; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
747; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
748; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
749; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
750; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
751; GFX940-TGSPLIT-NEXT:    s_endpgm
752;
753; GFX11-WGP-LABEL: global_singlethread_monotonic_store:
754; GFX11-WGP:       ; %bb.0: ; %entry
755; GFX11-WGP-NEXT:    s_clause 0x1
756; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x0
757; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
758; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
759; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
760; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s2
761; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[0:1]
762; GFX11-WGP-NEXT:    s_endpgm
763;
764; GFX11-CU-LABEL: global_singlethread_monotonic_store:
765; GFX11-CU:       ; %bb.0: ; %entry
766; GFX11-CU-NEXT:    s_clause 0x1
767; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x0
768; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
769; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
770; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
771; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s2
772; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
773; GFX11-CU-NEXT:    s_endpgm
774    i32 %in, i32 addrspace(1)* %out) {
775entry:
776  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread") monotonic, align 4
777  ret void
778}
779
780define amdgpu_kernel void @global_singlethread_release_store(
781; GFX6-LABEL: global_singlethread_release_store:
782; GFX6:       ; %bb.0: ; %entry
783; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
784; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
785; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
786; GFX6-NEXT:    s_mov_b32 s2, -1
787; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
788; GFX6-NEXT:    v_mov_b32_e32 v0, s6
789; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
790; GFX6-NEXT:    s_endpgm
791;
792; GFX7-LABEL: global_singlethread_release_store:
793; GFX7:       ; %bb.0: ; %entry
794; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
795; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
796; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
797; GFX7-NEXT:    v_mov_b32_e32 v0, s0
798; GFX7-NEXT:    v_mov_b32_e32 v1, s1
799; GFX7-NEXT:    v_mov_b32_e32 v2, s2
800; GFX7-NEXT:    flat_store_dword v[0:1], v2
801; GFX7-NEXT:    s_endpgm
802;
803; GFX10-WGP-LABEL: global_singlethread_release_store:
804; GFX10-WGP:       ; %bb.0: ; %entry
805; GFX10-WGP-NEXT:    s_clause 0x1
806; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
807; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
808; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
809; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
810; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
811; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
812; GFX10-WGP-NEXT:    s_endpgm
813;
814; GFX10-CU-LABEL: global_singlethread_release_store:
815; GFX10-CU:       ; %bb.0: ; %entry
816; GFX10-CU-NEXT:    s_clause 0x1
817; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
818; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
819; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
820; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
821; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
822; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
823; GFX10-CU-NEXT:    s_endpgm
824;
825; SKIP-CACHE-INV-LABEL: global_singlethread_release_store:
826; SKIP-CACHE-INV:       ; %bb.0: ; %entry
827; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x0
828; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
829; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
830; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
831; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
832; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
833; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
834; SKIP-CACHE-INV-NEXT:    s_endpgm
835;
836; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_release_store:
837; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
838; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
839; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
840; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
841; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
842; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
843; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
844; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
845;
846; GFX90A-TGSPLIT-LABEL: global_singlethread_release_store:
847; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
848; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
849; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
850; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
851; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
852; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
853; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
854; GFX90A-TGSPLIT-NEXT:    s_endpgm
855;
856; GFX940-NOTTGSPLIT-LABEL: global_singlethread_release_store:
857; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
858; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
859; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
860; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
861; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
862; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
863; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
864; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
865;
866; GFX940-TGSPLIT-LABEL: global_singlethread_release_store:
867; GFX940-TGSPLIT:       ; %bb.0: ; %entry
868; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
869; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
870; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
871; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
872; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
873; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
874; GFX940-TGSPLIT-NEXT:    s_endpgm
875;
876; GFX11-WGP-LABEL: global_singlethread_release_store:
877; GFX11-WGP:       ; %bb.0: ; %entry
878; GFX11-WGP-NEXT:    s_clause 0x1
879; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x0
880; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
881; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
882; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
883; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s2
884; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[0:1]
885; GFX11-WGP-NEXT:    s_endpgm
886;
887; GFX11-CU-LABEL: global_singlethread_release_store:
888; GFX11-CU:       ; %bb.0: ; %entry
889; GFX11-CU-NEXT:    s_clause 0x1
890; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x0
891; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
892; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
893; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
894; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s2
895; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
896; GFX11-CU-NEXT:    s_endpgm
897    i32 %in, i32 addrspace(1)* %out) {
898entry:
899  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread") release, align 4
900  ret void
901}
902
903define amdgpu_kernel void @global_singlethread_seq_cst_store(
904; GFX6-LABEL: global_singlethread_seq_cst_store:
905; GFX6:       ; %bb.0: ; %entry
906; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
907; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
908; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
909; GFX6-NEXT:    s_mov_b32 s2, -1
910; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
911; GFX6-NEXT:    v_mov_b32_e32 v0, s6
912; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
913; GFX6-NEXT:    s_endpgm
914;
915; GFX7-LABEL: global_singlethread_seq_cst_store:
916; GFX7:       ; %bb.0: ; %entry
917; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
918; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
919; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
920; GFX7-NEXT:    v_mov_b32_e32 v0, s0
921; GFX7-NEXT:    v_mov_b32_e32 v1, s1
922; GFX7-NEXT:    v_mov_b32_e32 v2, s2
923; GFX7-NEXT:    flat_store_dword v[0:1], v2
924; GFX7-NEXT:    s_endpgm
925;
926; GFX10-WGP-LABEL: global_singlethread_seq_cst_store:
927; GFX10-WGP:       ; %bb.0: ; %entry
928; GFX10-WGP-NEXT:    s_clause 0x1
929; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
930; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
931; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
932; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
933; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
934; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
935; GFX10-WGP-NEXT:    s_endpgm
936;
937; GFX10-CU-LABEL: global_singlethread_seq_cst_store:
938; GFX10-CU:       ; %bb.0: ; %entry
939; GFX10-CU-NEXT:    s_clause 0x1
940; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
941; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
942; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
943; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
944; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
945; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
946; GFX10-CU-NEXT:    s_endpgm
947;
948; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_store:
949; SKIP-CACHE-INV:       ; %bb.0: ; %entry
950; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x0
951; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
952; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
953; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
954; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
955; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
956; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
957; SKIP-CACHE-INV-NEXT:    s_endpgm
958;
959; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_store:
960; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
961; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
962; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
963; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
964; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
965; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
966; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
967; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
968;
969; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_store:
970; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
971; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
972; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
973; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
974; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
975; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
976; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
977; GFX90A-TGSPLIT-NEXT:    s_endpgm
978;
979; GFX940-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_store:
980; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
981; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
982; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
983; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
984; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
985; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
986; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
987; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
988;
989; GFX940-TGSPLIT-LABEL: global_singlethread_seq_cst_store:
990; GFX940-TGSPLIT:       ; %bb.0: ; %entry
991; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
992; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
993; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
994; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
995; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
996; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
997; GFX940-TGSPLIT-NEXT:    s_endpgm
998;
999; GFX11-WGP-LABEL: global_singlethread_seq_cst_store:
1000; GFX11-WGP:       ; %bb.0: ; %entry
1001; GFX11-WGP-NEXT:    s_clause 0x1
1002; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x0
1003; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
1004; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
1005; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1006; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s2
1007; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[0:1]
1008; GFX11-WGP-NEXT:    s_endpgm
1009;
1010; GFX11-CU-LABEL: global_singlethread_seq_cst_store:
1011; GFX11-CU:       ; %bb.0: ; %entry
1012; GFX11-CU-NEXT:    s_clause 0x1
1013; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x0
1014; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
1015; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
1016; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1017; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s2
1018; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
1019; GFX11-CU-NEXT:    s_endpgm
1020    i32 %in, i32 addrspace(1)* %out) {
1021entry:
1022  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread") seq_cst, align 4
1023  ret void
1024}
1025
1026define amdgpu_kernel void @global_singlethread_monotonic_atomicrmw(
1027; GFX6-LABEL: global_singlethread_monotonic_atomicrmw:
1028; GFX6:       ; %bb.0: ; %entry
1029; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
1030; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1031; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1032; GFX6-NEXT:    s_mov_b32 s2, -1
1033; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1034; GFX6-NEXT:    v_mov_b32_e32 v0, s6
1035; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
1036; GFX6-NEXT:    s_endpgm
1037;
1038; GFX7-LABEL: global_singlethread_monotonic_atomicrmw:
1039; GFX7:       ; %bb.0: ; %entry
1040; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1041; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1042; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1043; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1044; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1045; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1046; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1047; GFX7-NEXT:    s_endpgm
1048;
1049; GFX10-WGP-LABEL: global_singlethread_monotonic_atomicrmw:
1050; GFX10-WGP:       ; %bb.0: ; %entry
1051; GFX10-WGP-NEXT:    s_clause 0x1
1052; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1053; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1054; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
1055; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1056; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1057; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
1058; GFX10-WGP-NEXT:    s_endpgm
1059;
1060; GFX10-CU-LABEL: global_singlethread_monotonic_atomicrmw:
1061; GFX10-CU:       ; %bb.0: ; %entry
1062; GFX10-CU-NEXT:    s_clause 0x1
1063; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1064; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1065; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
1066; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1067; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1068; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
1069; GFX10-CU-NEXT:    s_endpgm
1070;
1071; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_atomicrmw:
1072; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1073; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
1074; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1075; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
1076; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
1077; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1078; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
1079; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
1080; SKIP-CACHE-INV-NEXT:    s_endpgm
1081;
1082; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_atomicrmw:
1083; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1084; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1085; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1086; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1087; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1088; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1089; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1090; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1091;
1092; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_atomicrmw:
1093; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1094; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1095; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1096; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1097; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1098; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1099; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1100; GFX90A-TGSPLIT-NEXT:    s_endpgm
1101;
1102; GFX940-NOTTGSPLIT-LABEL: global_singlethread_monotonic_atomicrmw:
1103; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1104; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1105; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1106; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1107; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1108; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1109; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
1110; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1111;
1112; GFX940-TGSPLIT-LABEL: global_singlethread_monotonic_atomicrmw:
1113; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1114; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1115; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1116; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1117; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1118; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1119; GFX940-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
1120; GFX940-TGSPLIT-NEXT:    s_endpgm
1121;
1122; GFX11-WGP-LABEL: global_singlethread_monotonic_atomicrmw:
1123; GFX11-WGP:       ; %bb.0: ; %entry
1124; GFX11-WGP-NEXT:    s_clause 0x1
1125; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x8
1126; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1127; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
1128; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1129; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s2
1130; GFX11-WGP-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
1131; GFX11-WGP-NEXT:    s_endpgm
1132;
1133; GFX11-CU-LABEL: global_singlethread_monotonic_atomicrmw:
1134; GFX11-CU:       ; %bb.0: ; %entry
1135; GFX11-CU-NEXT:    s_clause 0x1
1136; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x8
1137; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1138; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
1139; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1140; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s2
1141; GFX11-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
1142; GFX11-CU-NEXT:    s_endpgm
1143    i32 addrspace(1)* %out, i32 %in) {
1144entry:
1145  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") monotonic
1146  ret void
1147}
1148
1149define amdgpu_kernel void @global_singlethread_acquire_atomicrmw(
1150; GFX6-LABEL: global_singlethread_acquire_atomicrmw:
1151; GFX6:       ; %bb.0: ; %entry
1152; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
1153; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1154; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1155; GFX6-NEXT:    s_mov_b32 s2, -1
1156; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1157; GFX6-NEXT:    v_mov_b32_e32 v0, s6
1158; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
1159; GFX6-NEXT:    s_endpgm
1160;
1161; GFX7-LABEL: global_singlethread_acquire_atomicrmw:
1162; GFX7:       ; %bb.0: ; %entry
1163; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1164; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1165; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1166; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1167; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1168; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1169; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1170; GFX7-NEXT:    s_endpgm
1171;
1172; GFX10-WGP-LABEL: global_singlethread_acquire_atomicrmw:
1173; GFX10-WGP:       ; %bb.0: ; %entry
1174; GFX10-WGP-NEXT:    s_clause 0x1
1175; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1176; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1177; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
1178; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1179; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1180; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
1181; GFX10-WGP-NEXT:    s_endpgm
1182;
1183; GFX10-CU-LABEL: global_singlethread_acquire_atomicrmw:
1184; GFX10-CU:       ; %bb.0: ; %entry
1185; GFX10-CU-NEXT:    s_clause 0x1
1186; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1187; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1188; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
1189; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1190; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1191; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
1192; GFX10-CU-NEXT:    s_endpgm
1193;
1194; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_atomicrmw:
1195; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1196; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
1197; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1198; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
1199; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
1200; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1201; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
1202; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
1203; SKIP-CACHE-INV-NEXT:    s_endpgm
1204;
1205; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_atomicrmw:
1206; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1207; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1208; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1209; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1210; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1211; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1212; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1213; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1214;
1215; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_atomicrmw:
1216; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1217; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1218; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1219; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1220; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1221; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1222; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1223; GFX90A-TGSPLIT-NEXT:    s_endpgm
1224;
1225; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acquire_atomicrmw:
1226; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1227; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1228; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1229; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1230; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1231; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1232; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
1233; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1234;
1235; GFX940-TGSPLIT-LABEL: global_singlethread_acquire_atomicrmw:
1236; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1237; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1238; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1239; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1240; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1241; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1242; GFX940-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
1243; GFX940-TGSPLIT-NEXT:    s_endpgm
1244;
1245; GFX11-WGP-LABEL: global_singlethread_acquire_atomicrmw:
1246; GFX11-WGP:       ; %bb.0: ; %entry
1247; GFX11-WGP-NEXT:    s_clause 0x1
1248; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x8
1249; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1250; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
1251; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1252; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s2
1253; GFX11-WGP-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
1254; GFX11-WGP-NEXT:    s_endpgm
1255;
1256; GFX11-CU-LABEL: global_singlethread_acquire_atomicrmw:
1257; GFX11-CU:       ; %bb.0: ; %entry
1258; GFX11-CU-NEXT:    s_clause 0x1
1259; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x8
1260; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1261; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
1262; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1263; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s2
1264; GFX11-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
1265; GFX11-CU-NEXT:    s_endpgm
1266    i32 addrspace(1)* %out, i32 %in) {
1267entry:
1268  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") acquire
1269  ret void
1270}
1271
1272define amdgpu_kernel void @global_singlethread_release_atomicrmw(
1273; GFX6-LABEL: global_singlethread_release_atomicrmw:
1274; GFX6:       ; %bb.0: ; %entry
1275; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
1276; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1277; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1278; GFX6-NEXT:    s_mov_b32 s2, -1
1279; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1280; GFX6-NEXT:    v_mov_b32_e32 v0, s6
1281; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
1282; GFX6-NEXT:    s_endpgm
1283;
1284; GFX7-LABEL: global_singlethread_release_atomicrmw:
1285; GFX7:       ; %bb.0: ; %entry
1286; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1287; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1288; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1289; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1290; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1291; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1292; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1293; GFX7-NEXT:    s_endpgm
1294;
1295; GFX10-WGP-LABEL: global_singlethread_release_atomicrmw:
1296; GFX10-WGP:       ; %bb.0: ; %entry
1297; GFX10-WGP-NEXT:    s_clause 0x1
1298; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1299; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1300; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
1301; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1302; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1303; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
1304; GFX10-WGP-NEXT:    s_endpgm
1305;
1306; GFX10-CU-LABEL: global_singlethread_release_atomicrmw:
1307; GFX10-CU:       ; %bb.0: ; %entry
1308; GFX10-CU-NEXT:    s_clause 0x1
1309; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1310; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1311; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
1312; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1313; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1314; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
1315; GFX10-CU-NEXT:    s_endpgm
1316;
1317; SKIP-CACHE-INV-LABEL: global_singlethread_release_atomicrmw:
1318; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1319; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
1320; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1321; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
1322; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
1323; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1324; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
1325; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
1326; SKIP-CACHE-INV-NEXT:    s_endpgm
1327;
1328; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_release_atomicrmw:
1329; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1330; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1331; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1332; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1333; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1334; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1335; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1336; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1337;
1338; GFX90A-TGSPLIT-LABEL: global_singlethread_release_atomicrmw:
1339; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1340; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1341; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1342; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1343; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1344; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1345; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1346; GFX90A-TGSPLIT-NEXT:    s_endpgm
1347;
1348; GFX940-NOTTGSPLIT-LABEL: global_singlethread_release_atomicrmw:
1349; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1350; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1351; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1352; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1353; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1354; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1355; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
1356; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1357;
1358; GFX940-TGSPLIT-LABEL: global_singlethread_release_atomicrmw:
1359; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1360; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1361; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1362; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1363; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1364; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1365; GFX940-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
1366; GFX940-TGSPLIT-NEXT:    s_endpgm
1367;
1368; GFX11-WGP-LABEL: global_singlethread_release_atomicrmw:
1369; GFX11-WGP:       ; %bb.0: ; %entry
1370; GFX11-WGP-NEXT:    s_clause 0x1
1371; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x8
1372; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1373; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
1374; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1375; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s2
1376; GFX11-WGP-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
1377; GFX11-WGP-NEXT:    s_endpgm
1378;
1379; GFX11-CU-LABEL: global_singlethread_release_atomicrmw:
1380; GFX11-CU:       ; %bb.0: ; %entry
1381; GFX11-CU-NEXT:    s_clause 0x1
1382; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x8
1383; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1384; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
1385; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1386; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s2
1387; GFX11-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
1388; GFX11-CU-NEXT:    s_endpgm
1389    i32 addrspace(1)* %out, i32 %in) {
1390entry:
1391  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") release
1392  ret void
1393}
1394
1395define amdgpu_kernel void @global_singlethread_acq_rel_atomicrmw(
1396; GFX6-LABEL: global_singlethread_acq_rel_atomicrmw:
1397; GFX6:       ; %bb.0: ; %entry
1398; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
1399; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1400; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1401; GFX6-NEXT:    s_mov_b32 s2, -1
1402; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1403; GFX6-NEXT:    v_mov_b32_e32 v0, s6
1404; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
1405; GFX6-NEXT:    s_endpgm
1406;
1407; GFX7-LABEL: global_singlethread_acq_rel_atomicrmw:
1408; GFX7:       ; %bb.0: ; %entry
1409; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1410; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1411; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1412; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1413; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1414; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1415; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1416; GFX7-NEXT:    s_endpgm
1417;
1418; GFX10-WGP-LABEL: global_singlethread_acq_rel_atomicrmw:
1419; GFX10-WGP:       ; %bb.0: ; %entry
1420; GFX10-WGP-NEXT:    s_clause 0x1
1421; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1422; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1423; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
1424; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1425; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1426; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
1427; GFX10-WGP-NEXT:    s_endpgm
1428;
1429; GFX10-CU-LABEL: global_singlethread_acq_rel_atomicrmw:
1430; GFX10-CU:       ; %bb.0: ; %entry
1431; GFX10-CU-NEXT:    s_clause 0x1
1432; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1433; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1434; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
1435; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1436; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1437; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
1438; GFX10-CU-NEXT:    s_endpgm
1439;
1440; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_atomicrmw:
1441; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1442; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
1443; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1444; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
1445; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
1446; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1447; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
1448; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
1449; SKIP-CACHE-INV-NEXT:    s_endpgm
1450;
1451; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_atomicrmw:
1452; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1453; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1454; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1455; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1456; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1457; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1458; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1459; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1460;
1461; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_atomicrmw:
1462; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1463; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1464; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1465; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1466; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1467; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1468; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1469; GFX90A-TGSPLIT-NEXT:    s_endpgm
1470;
1471; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_atomicrmw:
1472; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1473; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1474; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1475; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1476; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1477; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1478; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
1479; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1480;
1481; GFX940-TGSPLIT-LABEL: global_singlethread_acq_rel_atomicrmw:
1482; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1483; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1484; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1485; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1486; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1487; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1488; GFX940-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
1489; GFX940-TGSPLIT-NEXT:    s_endpgm
1490;
1491; GFX11-WGP-LABEL: global_singlethread_acq_rel_atomicrmw:
1492; GFX11-WGP:       ; %bb.0: ; %entry
1493; GFX11-WGP-NEXT:    s_clause 0x1
1494; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x8
1495; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1496; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
1497; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1498; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s2
1499; GFX11-WGP-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
1500; GFX11-WGP-NEXT:    s_endpgm
1501;
1502; GFX11-CU-LABEL: global_singlethread_acq_rel_atomicrmw:
1503; GFX11-CU:       ; %bb.0: ; %entry
1504; GFX11-CU-NEXT:    s_clause 0x1
1505; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x8
1506; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1507; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
1508; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1509; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s2
1510; GFX11-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
1511; GFX11-CU-NEXT:    s_endpgm
1512    i32 addrspace(1)* %out, i32 %in) {
1513entry:
1514  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") acq_rel
1515  ret void
1516}
1517
1518define amdgpu_kernel void @global_singlethread_seq_cst_atomicrmw(
1519; GFX6-LABEL: global_singlethread_seq_cst_atomicrmw:
1520; GFX6:       ; %bb.0: ; %entry
1521; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
1522; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1523; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1524; GFX6-NEXT:    s_mov_b32 s2, -1
1525; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1526; GFX6-NEXT:    v_mov_b32_e32 v0, s6
1527; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
1528; GFX6-NEXT:    s_endpgm
1529;
1530; GFX7-LABEL: global_singlethread_seq_cst_atomicrmw:
1531; GFX7:       ; %bb.0: ; %entry
1532; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1533; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1534; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1535; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1536; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1537; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1538; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1539; GFX7-NEXT:    s_endpgm
1540;
1541; GFX10-WGP-LABEL: global_singlethread_seq_cst_atomicrmw:
1542; GFX10-WGP:       ; %bb.0: ; %entry
1543; GFX10-WGP-NEXT:    s_clause 0x1
1544; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1545; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1546; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
1547; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1548; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1549; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
1550; GFX10-WGP-NEXT:    s_endpgm
1551;
1552; GFX10-CU-LABEL: global_singlethread_seq_cst_atomicrmw:
1553; GFX10-CU:       ; %bb.0: ; %entry
1554; GFX10-CU-NEXT:    s_clause 0x1
1555; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1556; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1557; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
1558; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1559; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1560; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
1561; GFX10-CU-NEXT:    s_endpgm
1562;
1563; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_atomicrmw:
1564; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1565; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
1566; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1567; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
1568; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
1569; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1570; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
1571; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
1572; SKIP-CACHE-INV-NEXT:    s_endpgm
1573;
1574; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_atomicrmw:
1575; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1576; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1577; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1578; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1579; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1580; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1581; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1582; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1583;
1584; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_atomicrmw:
1585; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1586; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1587; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1588; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1589; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1590; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1591; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1592; GFX90A-TGSPLIT-NEXT:    s_endpgm
1593;
1594; GFX940-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_atomicrmw:
1595; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1596; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1597; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1598; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1599; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1600; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1601; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
1602; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1603;
1604; GFX940-TGSPLIT-LABEL: global_singlethread_seq_cst_atomicrmw:
1605; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1606; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1607; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1608; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1609; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1610; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1611; GFX940-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
1612; GFX940-TGSPLIT-NEXT:    s_endpgm
1613;
1614; GFX11-WGP-LABEL: global_singlethread_seq_cst_atomicrmw:
1615; GFX11-WGP:       ; %bb.0: ; %entry
1616; GFX11-WGP-NEXT:    s_clause 0x1
1617; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x8
1618; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1619; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
1620; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1621; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s2
1622; GFX11-WGP-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
1623; GFX11-WGP-NEXT:    s_endpgm
1624;
1625; GFX11-CU-LABEL: global_singlethread_seq_cst_atomicrmw:
1626; GFX11-CU:       ; %bb.0: ; %entry
1627; GFX11-CU-NEXT:    s_clause 0x1
1628; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x8
1629; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1630; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
1631; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1632; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s2
1633; GFX11-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
1634; GFX11-CU-NEXT:    s_endpgm
1635    i32 addrspace(1)* %out, i32 %in) {
1636entry:
1637  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") seq_cst
1638  ret void
1639}
1640
1641define amdgpu_kernel void @global_singlethread_acquire_ret_atomicrmw(
1642; GFX6-LABEL: global_singlethread_acquire_ret_atomicrmw:
1643; GFX6:       ; %bb.0: ; %entry
1644; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
1645; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1646; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1647; GFX6-NEXT:    s_mov_b32 s2, -1
1648; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1649; GFX6-NEXT:    v_mov_b32_e32 v0, s6
1650; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
1651; GFX6-NEXT:    s_waitcnt vmcnt(0)
1652; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1653; GFX6-NEXT:    s_endpgm
1654;
1655; GFX7-LABEL: global_singlethread_acquire_ret_atomicrmw:
1656; GFX7:       ; %bb.0: ; %entry
1657; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1658; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1659; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1660; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1661; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1662; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1663; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1664; GFX7-NEXT:    s_waitcnt vmcnt(0)
1665; GFX7-NEXT:    flat_store_dword v[0:1], v2
1666; GFX7-NEXT:    s_endpgm
1667;
1668; GFX10-WGP-LABEL: global_singlethread_acquire_ret_atomicrmw:
1669; GFX10-WGP:       ; %bb.0: ; %entry
1670; GFX10-WGP-NEXT:    s_clause 0x1
1671; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1672; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1673; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
1674; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1675; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1676; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1677; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
1678; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
1679; GFX10-WGP-NEXT:    s_endpgm
1680;
1681; GFX10-CU-LABEL: global_singlethread_acquire_ret_atomicrmw:
1682; GFX10-CU:       ; %bb.0: ; %entry
1683; GFX10-CU-NEXT:    s_clause 0x1
1684; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1685; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1686; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
1687; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1688; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1689; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1690; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
1691; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
1692; GFX10-CU-NEXT:    s_endpgm
1693;
1694; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_ret_atomicrmw:
1695; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1696; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
1697; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1698; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
1699; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
1700; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1701; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
1702; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
1703; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
1704; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1705; SKIP-CACHE-INV-NEXT:    s_endpgm
1706;
1707; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_ret_atomicrmw:
1708; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1709; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1710; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1711; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1712; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1713; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1714; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1715; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1716; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
1717; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1718;
1719; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_ret_atomicrmw:
1720; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1721; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1722; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1723; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1724; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1725; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1726; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1727; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1728; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
1729; GFX90A-TGSPLIT-NEXT:    s_endpgm
1730;
1731; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acquire_ret_atomicrmw:
1732; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1733; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1734; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1735; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1736; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1737; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1738; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[2:3] sc0
1739; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1740; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
1741; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1742;
1743; GFX940-TGSPLIT-LABEL: global_singlethread_acquire_ret_atomicrmw:
1744; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1745; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1746; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1747; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1748; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1749; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1750; GFX940-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[2:3] sc0
1751; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1752; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
1753; GFX940-TGSPLIT-NEXT:    s_endpgm
1754;
1755; GFX11-WGP-LABEL: global_singlethread_acquire_ret_atomicrmw:
1756; GFX11-WGP:       ; %bb.0: ; %entry
1757; GFX11-WGP-NEXT:    s_clause 0x1
1758; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x8
1759; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1760; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
1761; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1762; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s2
1763; GFX11-WGP-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
1764; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
1765; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[0:1]
1766; GFX11-WGP-NEXT:    s_endpgm
1767;
1768; GFX11-CU-LABEL: global_singlethread_acquire_ret_atomicrmw:
1769; GFX11-CU:       ; %bb.0: ; %entry
1770; GFX11-CU-NEXT:    s_clause 0x1
1771; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x8
1772; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1773; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
1774; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1775; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s2
1776; GFX11-CU-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
1777; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
1778; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
1779; GFX11-CU-NEXT:    s_endpgm
1780    i32 addrspace(1)* %out, i32 %in) {
1781entry:
1782  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") acquire
1783  store i32 %val, i32 addrspace(1)* %out, align 4
1784  ret void
1785}
1786
1787define amdgpu_kernel void @global_singlethread_acq_rel_ret_atomicrmw(
1788; GFX6-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1789; GFX6:       ; %bb.0: ; %entry
1790; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
1791; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1792; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1793; GFX6-NEXT:    s_mov_b32 s2, -1
1794; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1795; GFX6-NEXT:    v_mov_b32_e32 v0, s6
1796; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
1797; GFX6-NEXT:    s_waitcnt vmcnt(0)
1798; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1799; GFX6-NEXT:    s_endpgm
1800;
1801; GFX7-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1802; GFX7:       ; %bb.0: ; %entry
1803; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1804; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1805; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1806; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1807; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1808; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1809; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1810; GFX7-NEXT:    s_waitcnt vmcnt(0)
1811; GFX7-NEXT:    flat_store_dword v[0:1], v2
1812; GFX7-NEXT:    s_endpgm
1813;
1814; GFX10-WGP-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1815; GFX10-WGP:       ; %bb.0: ; %entry
1816; GFX10-WGP-NEXT:    s_clause 0x1
1817; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1818; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1819; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
1820; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1821; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1822; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1823; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
1824; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
1825; GFX10-WGP-NEXT:    s_endpgm
1826;
1827; GFX10-CU-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1828; GFX10-CU:       ; %bb.0: ; %entry
1829; GFX10-CU-NEXT:    s_clause 0x1
1830; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1831; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1832; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
1833; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1834; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1835; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1836; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
1837; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
1838; GFX10-CU-NEXT:    s_endpgm
1839;
1840; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1841; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1842; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
1843; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1844; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
1845; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
1846; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1847; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
1848; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
1849; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
1850; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1851; SKIP-CACHE-INV-NEXT:    s_endpgm
1852;
1853; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1854; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1855; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1856; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1857; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1858; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1859; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1860; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1861; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1862; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
1863; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1864;
1865; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1866; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1867; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1868; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1869; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1870; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1871; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1872; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1873; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1874; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
1875; GFX90A-TGSPLIT-NEXT:    s_endpgm
1876;
1877; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1878; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1879; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1880; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1881; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1882; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1883; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1884; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[2:3] sc0
1885; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1886; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
1887; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1888;
1889; GFX940-TGSPLIT-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1890; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1891; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1892; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1893; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1894; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1895; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
1896; GFX940-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[2:3] sc0
1897; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1898; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
1899; GFX940-TGSPLIT-NEXT:    s_endpgm
1900;
1901; GFX11-WGP-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1902; GFX11-WGP:       ; %bb.0: ; %entry
1903; GFX11-WGP-NEXT:    s_clause 0x1
1904; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x8
1905; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1906; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
1907; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1908; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s2
1909; GFX11-WGP-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
1910; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
1911; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[0:1]
1912; GFX11-WGP-NEXT:    s_endpgm
1913;
1914; GFX11-CU-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
1915; GFX11-CU:       ; %bb.0: ; %entry
1916; GFX11-CU-NEXT:    s_clause 0x1
1917; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x8
1918; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
1919; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
1920; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1921; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s2
1922; GFX11-CU-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
1923; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
1924; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
1925; GFX11-CU-NEXT:    s_endpgm
1926    i32 addrspace(1)* %out, i32 %in) {
1927entry:
1928  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") acq_rel
1929  store i32 %val, i32 addrspace(1)* %out, align 4
1930  ret void
1931}
1932
1933define amdgpu_kernel void @global_singlethread_seq_cst_ret_atomicrmw(
1934; GFX6-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
1935; GFX6:       ; %bb.0: ; %entry
1936; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
1937; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1938; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1939; GFX6-NEXT:    s_mov_b32 s2, -1
1940; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1941; GFX6-NEXT:    v_mov_b32_e32 v0, s6
1942; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
1943; GFX6-NEXT:    s_waitcnt vmcnt(0)
1944; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1945; GFX6-NEXT:    s_endpgm
1946;
1947; GFX7-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
1948; GFX7:       ; %bb.0: ; %entry
1949; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1950; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1951; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1952; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1953; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1954; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1955; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1956; GFX7-NEXT:    s_waitcnt vmcnt(0)
1957; GFX7-NEXT:    flat_store_dword v[0:1], v2
1958; GFX7-NEXT:    s_endpgm
1959;
1960; GFX10-WGP-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
1961; GFX10-WGP:       ; %bb.0: ; %entry
1962; GFX10-WGP-NEXT:    s_clause 0x1
1963; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1964; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1965; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
1966; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1967; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1968; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1969; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
1970; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
1971; GFX10-WGP-NEXT:    s_endpgm
1972;
1973; GFX10-CU-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
1974; GFX10-CU:       ; %bb.0: ; %entry
1975; GFX10-CU-NEXT:    s_clause 0x1
1976; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1977; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1978; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
1979; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1980; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1981; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1982; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
1983; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
1984; GFX10-CU-NEXT:    s_endpgm
1985;
1986; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
1987; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1988; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
1989; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
1990; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
1991; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
1992; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1993; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
1994; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
1995; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
1996; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1997; SKIP-CACHE-INV-NEXT:    s_endpgm
1998;
1999; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
2000; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2001; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
2002; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2003; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
2004; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2005; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2006; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
2007; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2008; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
2009; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2010;
2011; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
2012; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2013; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
2014; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2015; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
2016; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2017; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
2018; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
2019; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2020; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
2021; GFX90A-TGSPLIT-NEXT:    s_endpgm
2022;
2023; GFX940-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
2024; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2025; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
2026; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2027; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
2028; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2029; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
2030; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[2:3] sc0
2031; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2032; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
2033; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2034;
2035; GFX940-TGSPLIT-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
2036; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2037; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
2038; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2039; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
2040; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2041; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
2042; GFX940-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[2:3] sc0
2043; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2044; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
2045; GFX940-TGSPLIT-NEXT:    s_endpgm
2046;
2047; GFX11-WGP-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
2048; GFX11-WGP:       ; %bb.0: ; %entry
2049; GFX11-WGP-NEXT:    s_clause 0x1
2050; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x8
2051; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
2052; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
2053; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2054; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s2
2055; GFX11-WGP-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
2056; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
2057; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[0:1]
2058; GFX11-WGP-NEXT:    s_endpgm
2059;
2060; GFX11-CU-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
2061; GFX11-CU:       ; %bb.0: ; %entry
2062; GFX11-CU-NEXT:    s_clause 0x1
2063; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x8
2064; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
2065; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
2066; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2067; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s2
2068; GFX11-CU-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
2069; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
2070; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
2071; GFX11-CU-NEXT:    s_endpgm
2072    i32 addrspace(1)* %out, i32 %in) {
2073entry:
2074  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") seq_cst
2075  store i32 %val, i32 addrspace(1)* %out, align 4
2076  ret void
2077}
2078
2079define amdgpu_kernel void @global_singlethread_monotonic_monotonic_cmpxchg(
2080; GFX6-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
2081; GFX6:       ; %bb.0: ; %entry
2082; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
2083; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2084; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2085; GFX6-NEXT:    s_mov_b32 s2, -1
2086; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2087; GFX6-NEXT:    v_mov_b32_e32 v0, s6
2088; GFX6-NEXT:    v_mov_b32_e32 v1, s7
2089; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2090; GFX6-NEXT:    s_endpgm
2091;
2092; GFX7-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
2093; GFX7:       ; %bb.0: ; %entry
2094; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2095; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2096; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2097; GFX7-NEXT:    s_add_u32 s0, s0, 16
2098; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2099; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2100; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2101; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2102; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2103; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2104; GFX7-NEXT:    s_endpgm
2105;
2106; GFX10-WGP-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
2107; GFX10-WGP:       ; %bb.0: ; %entry
2108; GFX10-WGP-NEXT:    s_clause 0x1
2109; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2110; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2111; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2112; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2113; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2114; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2115; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2116; GFX10-WGP-NEXT:    s_endpgm
2117;
2118; GFX10-CU-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
2119; GFX10-CU:       ; %bb.0: ; %entry
2120; GFX10-CU-NEXT:    s_clause 0x1
2121; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2122; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2123; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2124; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2125; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2126; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2127; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2128; GFX10-CU-NEXT:    s_endpgm
2129;
2130; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
2131; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2132; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
2133; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
2134; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
2135; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
2136; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2137; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2138; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2139; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2140; SKIP-CACHE-INV-NEXT:    s_endpgm
2141;
2142; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
2143; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2144; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2145; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2146; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2147; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2148; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2149; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2150; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2151;
2152; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
2153; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2154; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2155; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2156; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2157; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2158; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2159; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2160; GFX90A-TGSPLIT-NEXT:    s_endpgm
2161;
2162; GFX940-NOTTGSPLIT-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
2163; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2164; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2165; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2166; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2167; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2168; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2169; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2170; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2171;
2172; GFX940-TGSPLIT-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
2173; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2174; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2175; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2176; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2177; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2178; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2179; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2180; GFX940-TGSPLIT-NEXT:    s_endpgm
2181;
2182; GFX11-WGP-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
2183; GFX11-WGP:       ; %bb.0: ; %entry
2184; GFX11-WGP-NEXT:    s_clause 0x1
2185; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
2186; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
2187; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
2188; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2189; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2190; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2191; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
2192; GFX11-WGP-NEXT:    s_endpgm
2193;
2194; GFX11-CU-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
2195; GFX11-CU:       ; %bb.0: ; %entry
2196; GFX11-CU-NEXT:    s_clause 0x1
2197; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
2198; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
2199; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
2200; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2201; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2202; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2203; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
2204; GFX11-CU-NEXT:    s_endpgm
2205    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2206entry:
2207  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2208  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic monotonic
2209  ret void
2210}
2211
2212define amdgpu_kernel void @global_singlethread_acquire_monotonic_cmpxchg(
2213; GFX6-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
2214; GFX6:       ; %bb.0: ; %entry
2215; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
2216; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2217; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2218; GFX6-NEXT:    s_mov_b32 s2, -1
2219; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2220; GFX6-NEXT:    v_mov_b32_e32 v0, s6
2221; GFX6-NEXT:    v_mov_b32_e32 v1, s7
2222; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2223; GFX6-NEXT:    s_endpgm
2224;
2225; GFX7-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
2226; GFX7:       ; %bb.0: ; %entry
2227; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2228; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2229; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2230; GFX7-NEXT:    s_add_u32 s0, s0, 16
2231; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2232; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2233; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2234; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2235; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2236; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2237; GFX7-NEXT:    s_endpgm
2238;
2239; GFX10-WGP-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
2240; GFX10-WGP:       ; %bb.0: ; %entry
2241; GFX10-WGP-NEXT:    s_clause 0x1
2242; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2243; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2244; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2245; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2246; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2247; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2248; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2249; GFX10-WGP-NEXT:    s_endpgm
2250;
2251; GFX10-CU-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
2252; GFX10-CU:       ; %bb.0: ; %entry
2253; GFX10-CU-NEXT:    s_clause 0x1
2254; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2255; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2256; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2257; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2258; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2259; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2260; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2261; GFX10-CU-NEXT:    s_endpgm
2262;
2263; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
2264; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2265; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
2266; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
2267; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
2268; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
2269; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2270; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2271; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2272; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2273; SKIP-CACHE-INV-NEXT:    s_endpgm
2274;
2275; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
2276; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2277; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2278; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2279; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2280; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2281; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2282; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2283; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2284;
2285; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
2286; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2287; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2288; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2289; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2290; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2291; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2292; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2293; GFX90A-TGSPLIT-NEXT:    s_endpgm
2294;
2295; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
2296; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2297; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2298; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2299; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2300; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2301; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2302; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2303; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2304;
2305; GFX940-TGSPLIT-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
2306; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2307; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2308; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2309; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2310; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2311; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2312; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2313; GFX940-TGSPLIT-NEXT:    s_endpgm
2314;
2315; GFX11-WGP-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
2316; GFX11-WGP:       ; %bb.0: ; %entry
2317; GFX11-WGP-NEXT:    s_clause 0x1
2318; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
2319; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
2320; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
2321; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2322; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2323; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2324; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
2325; GFX11-WGP-NEXT:    s_endpgm
2326;
2327; GFX11-CU-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
2328; GFX11-CU:       ; %bb.0: ; %entry
2329; GFX11-CU-NEXT:    s_clause 0x1
2330; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
2331; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
2332; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
2333; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2334; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2335; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2336; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
2337; GFX11-CU-NEXT:    s_endpgm
2338    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2339entry:
2340  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2341  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire monotonic
2342  ret void
2343}
2344
2345define amdgpu_kernel void @global_singlethread_release_monotonic_cmpxchg(
2346; GFX6-LABEL: global_singlethread_release_monotonic_cmpxchg:
2347; GFX6:       ; %bb.0: ; %entry
2348; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
2349; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2350; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2351; GFX6-NEXT:    s_mov_b32 s2, -1
2352; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2353; GFX6-NEXT:    v_mov_b32_e32 v0, s6
2354; GFX6-NEXT:    v_mov_b32_e32 v1, s7
2355; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2356; GFX6-NEXT:    s_endpgm
2357;
2358; GFX7-LABEL: global_singlethread_release_monotonic_cmpxchg:
2359; GFX7:       ; %bb.0: ; %entry
2360; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2361; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2362; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2363; GFX7-NEXT:    s_add_u32 s0, s0, 16
2364; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2365; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2366; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2367; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2368; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2369; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2370; GFX7-NEXT:    s_endpgm
2371;
2372; GFX10-WGP-LABEL: global_singlethread_release_monotonic_cmpxchg:
2373; GFX10-WGP:       ; %bb.0: ; %entry
2374; GFX10-WGP-NEXT:    s_clause 0x1
2375; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2376; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2377; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2378; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2379; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2380; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2381; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2382; GFX10-WGP-NEXT:    s_endpgm
2383;
2384; GFX10-CU-LABEL: global_singlethread_release_monotonic_cmpxchg:
2385; GFX10-CU:       ; %bb.0: ; %entry
2386; GFX10-CU-NEXT:    s_clause 0x1
2387; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2388; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2389; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2390; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2391; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2392; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2393; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2394; GFX10-CU-NEXT:    s_endpgm
2395;
2396; SKIP-CACHE-INV-LABEL: global_singlethread_release_monotonic_cmpxchg:
2397; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2398; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
2399; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
2400; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
2401; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
2402; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2403; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2404; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2405; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2406; SKIP-CACHE-INV-NEXT:    s_endpgm
2407;
2408; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_release_monotonic_cmpxchg:
2409; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2410; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2411; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2412; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2413; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2414; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2415; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2416; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2417;
2418; GFX90A-TGSPLIT-LABEL: global_singlethread_release_monotonic_cmpxchg:
2419; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2420; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2421; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2422; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2423; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2424; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2425; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2426; GFX90A-TGSPLIT-NEXT:    s_endpgm
2427;
2428; GFX940-NOTTGSPLIT-LABEL: global_singlethread_release_monotonic_cmpxchg:
2429; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2430; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2431; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2432; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2433; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2434; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2435; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2436; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2437;
2438; GFX940-TGSPLIT-LABEL: global_singlethread_release_monotonic_cmpxchg:
2439; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2440; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2441; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2442; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2443; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2444; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2445; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2446; GFX940-TGSPLIT-NEXT:    s_endpgm
2447;
2448; GFX11-WGP-LABEL: global_singlethread_release_monotonic_cmpxchg:
2449; GFX11-WGP:       ; %bb.0: ; %entry
2450; GFX11-WGP-NEXT:    s_clause 0x1
2451; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
2452; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
2453; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
2454; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2455; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2456; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2457; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
2458; GFX11-WGP-NEXT:    s_endpgm
2459;
2460; GFX11-CU-LABEL: global_singlethread_release_monotonic_cmpxchg:
2461; GFX11-CU:       ; %bb.0: ; %entry
2462; GFX11-CU-NEXT:    s_clause 0x1
2463; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
2464; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
2465; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
2466; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2467; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2468; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2469; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
2470; GFX11-CU-NEXT:    s_endpgm
2471    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2472entry:
2473  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2474  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") release monotonic
2475  ret void
2476}
2477
2478define amdgpu_kernel void @global_singlethread_acq_rel_monotonic_cmpxchg(
2479; GFX6-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
2480; GFX6:       ; %bb.0: ; %entry
2481; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
2482; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2483; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2484; GFX6-NEXT:    s_mov_b32 s2, -1
2485; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2486; GFX6-NEXT:    v_mov_b32_e32 v0, s6
2487; GFX6-NEXT:    v_mov_b32_e32 v1, s7
2488; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2489; GFX6-NEXT:    s_endpgm
2490;
2491; GFX7-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
2492; GFX7:       ; %bb.0: ; %entry
2493; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2494; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2495; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2496; GFX7-NEXT:    s_add_u32 s0, s0, 16
2497; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2498; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2499; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2500; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2501; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2502; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2503; GFX7-NEXT:    s_endpgm
2504;
2505; GFX10-WGP-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
2506; GFX10-WGP:       ; %bb.0: ; %entry
2507; GFX10-WGP-NEXT:    s_clause 0x1
2508; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2509; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2510; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2511; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2512; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2513; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2514; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2515; GFX10-WGP-NEXT:    s_endpgm
2516;
2517; GFX10-CU-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
2518; GFX10-CU:       ; %bb.0: ; %entry
2519; GFX10-CU-NEXT:    s_clause 0x1
2520; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2521; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2522; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2523; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2524; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2525; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2526; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2527; GFX10-CU-NEXT:    s_endpgm
2528;
2529; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
2530; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2531; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
2532; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
2533; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
2534; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
2535; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2536; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2537; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2538; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2539; SKIP-CACHE-INV-NEXT:    s_endpgm
2540;
2541; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
2542; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2543; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2544; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2545; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2546; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2547; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2548; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2549; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2550;
2551; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
2552; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2553; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2554; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2555; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2556; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2557; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2558; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2559; GFX90A-TGSPLIT-NEXT:    s_endpgm
2560;
2561; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
2562; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2563; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2564; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2565; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2566; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2567; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2568; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2569; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2570;
2571; GFX940-TGSPLIT-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
2572; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2573; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2574; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2575; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2576; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2577; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2578; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2579; GFX940-TGSPLIT-NEXT:    s_endpgm
2580;
2581; GFX11-WGP-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
2582; GFX11-WGP:       ; %bb.0: ; %entry
2583; GFX11-WGP-NEXT:    s_clause 0x1
2584; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
2585; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
2586; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
2587; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2588; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2589; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2590; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
2591; GFX11-WGP-NEXT:    s_endpgm
2592;
2593; GFX11-CU-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
2594; GFX11-CU:       ; %bb.0: ; %entry
2595; GFX11-CU-NEXT:    s_clause 0x1
2596; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
2597; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
2598; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
2599; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2600; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2601; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2602; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
2603; GFX11-CU-NEXT:    s_endpgm
2604    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2605entry:
2606  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2607  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel monotonic
2608  ret void
2609}
2610
2611define amdgpu_kernel void @global_singlethread_seq_cst_monotonic_cmpxchg(
2612; GFX6-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
2613; GFX6:       ; %bb.0: ; %entry
2614; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
2615; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2616; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2617; GFX6-NEXT:    s_mov_b32 s2, -1
2618; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2619; GFX6-NEXT:    v_mov_b32_e32 v0, s6
2620; GFX6-NEXT:    v_mov_b32_e32 v1, s7
2621; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2622; GFX6-NEXT:    s_endpgm
2623;
2624; GFX7-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
2625; GFX7:       ; %bb.0: ; %entry
2626; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2627; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2628; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2629; GFX7-NEXT:    s_add_u32 s0, s0, 16
2630; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2631; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2632; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2633; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2634; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2635; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2636; GFX7-NEXT:    s_endpgm
2637;
2638; GFX10-WGP-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
2639; GFX10-WGP:       ; %bb.0: ; %entry
2640; GFX10-WGP-NEXT:    s_clause 0x1
2641; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2642; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2643; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2644; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2645; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2646; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2647; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2648; GFX10-WGP-NEXT:    s_endpgm
2649;
2650; GFX10-CU-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
2651; GFX10-CU:       ; %bb.0: ; %entry
2652; GFX10-CU-NEXT:    s_clause 0x1
2653; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2654; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2655; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2656; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2657; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2658; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2659; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2660; GFX10-CU-NEXT:    s_endpgm
2661;
2662; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
2663; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2664; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
2665; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
2666; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
2667; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
2668; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2669; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2670; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2671; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2672; SKIP-CACHE-INV-NEXT:    s_endpgm
2673;
2674; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
2675; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2676; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2677; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2678; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2679; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2680; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2681; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2682; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2683;
2684; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
2685; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2686; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2687; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2688; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2689; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2690; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2691; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2692; GFX90A-TGSPLIT-NEXT:    s_endpgm
2693;
2694; GFX940-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
2695; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2696; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2697; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2698; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2699; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2700; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2701; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2702; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2703;
2704; GFX940-TGSPLIT-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
2705; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2706; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2707; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2708; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2709; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2710; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2711; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2712; GFX940-TGSPLIT-NEXT:    s_endpgm
2713;
2714; GFX11-WGP-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
2715; GFX11-WGP:       ; %bb.0: ; %entry
2716; GFX11-WGP-NEXT:    s_clause 0x1
2717; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
2718; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
2719; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
2720; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2721; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2722; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2723; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
2724; GFX11-WGP-NEXT:    s_endpgm
2725;
2726; GFX11-CU-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
2727; GFX11-CU:       ; %bb.0: ; %entry
2728; GFX11-CU-NEXT:    s_clause 0x1
2729; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
2730; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
2731; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
2732; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2733; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2734; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2735; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
2736; GFX11-CU-NEXT:    s_endpgm
2737    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2738entry:
2739  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2740  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst monotonic
2741  ret void
2742}
2743
2744define amdgpu_kernel void @global_singlethread_monotonic_acquire_cmpxchg(
2745; GFX6-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
2746; GFX6:       ; %bb.0: ; %entry
2747; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
2748; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2749; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2750; GFX6-NEXT:    s_mov_b32 s2, -1
2751; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2752; GFX6-NEXT:    v_mov_b32_e32 v0, s6
2753; GFX6-NEXT:    v_mov_b32_e32 v1, s7
2754; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2755; GFX6-NEXT:    s_endpgm
2756;
2757; GFX7-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
2758; GFX7:       ; %bb.0: ; %entry
2759; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2760; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2761; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2762; GFX7-NEXT:    s_add_u32 s0, s0, 16
2763; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2764; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2765; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2766; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2767; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2768; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2769; GFX7-NEXT:    s_endpgm
2770;
2771; GFX10-WGP-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
2772; GFX10-WGP:       ; %bb.0: ; %entry
2773; GFX10-WGP-NEXT:    s_clause 0x1
2774; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2775; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2776; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2777; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2778; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2779; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2780; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2781; GFX10-WGP-NEXT:    s_endpgm
2782;
2783; GFX10-CU-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
2784; GFX10-CU:       ; %bb.0: ; %entry
2785; GFX10-CU-NEXT:    s_clause 0x1
2786; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2787; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2788; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2789; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2790; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2791; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2792; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2793; GFX10-CU-NEXT:    s_endpgm
2794;
2795; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
2796; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2797; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
2798; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
2799; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
2800; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
2801; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2802; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2803; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2804; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2805; SKIP-CACHE-INV-NEXT:    s_endpgm
2806;
2807; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
2808; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2809; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2810; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2811; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2812; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2813; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2814; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2815; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2816;
2817; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
2818; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2819; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2820; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2821; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2822; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2823; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2824; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2825; GFX90A-TGSPLIT-NEXT:    s_endpgm
2826;
2827; GFX940-NOTTGSPLIT-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
2828; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2829; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2830; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2831; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2832; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2833; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2834; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2835; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2836;
2837; GFX940-TGSPLIT-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
2838; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2839; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2840; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2841; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2842; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2843; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2844; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2845; GFX940-TGSPLIT-NEXT:    s_endpgm
2846;
2847; GFX11-WGP-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
2848; GFX11-WGP:       ; %bb.0: ; %entry
2849; GFX11-WGP-NEXT:    s_clause 0x1
2850; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
2851; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
2852; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
2853; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2854; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2855; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2856; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
2857; GFX11-WGP-NEXT:    s_endpgm
2858;
2859; GFX11-CU-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
2860; GFX11-CU:       ; %bb.0: ; %entry
2861; GFX11-CU-NEXT:    s_clause 0x1
2862; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
2863; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
2864; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
2865; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2866; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2867; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2868; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
2869; GFX11-CU-NEXT:    s_endpgm
2870    i32 addrspace(1)* %out, i32 %in, i32 %old) {
2871entry:
2872  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2873  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic acquire
2874  ret void
2875}
2876
2877define amdgpu_kernel void @global_singlethread_acquire_acquire_cmpxchg(
2878; GFX6-LABEL: global_singlethread_acquire_acquire_cmpxchg:
2879; GFX6:       ; %bb.0: ; %entry
2880; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
2881; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2882; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2883; GFX6-NEXT:    s_mov_b32 s2, -1
2884; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2885; GFX6-NEXT:    v_mov_b32_e32 v0, s6
2886; GFX6-NEXT:    v_mov_b32_e32 v1, s7
2887; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2888; GFX6-NEXT:    s_endpgm
2889;
2890; GFX7-LABEL: global_singlethread_acquire_acquire_cmpxchg:
2891; GFX7:       ; %bb.0: ; %entry
2892; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2893; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2894; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2895; GFX7-NEXT:    s_add_u32 s0, s0, 16
2896; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2897; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2898; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2899; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2900; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2901; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2902; GFX7-NEXT:    s_endpgm
2903;
2904; GFX10-WGP-LABEL: global_singlethread_acquire_acquire_cmpxchg:
2905; GFX10-WGP:       ; %bb.0: ; %entry
2906; GFX10-WGP-NEXT:    s_clause 0x1
2907; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2908; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2909; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2910; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2911; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2912; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2913; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2914; GFX10-WGP-NEXT:    s_endpgm
2915;
2916; GFX10-CU-LABEL: global_singlethread_acquire_acquire_cmpxchg:
2917; GFX10-CU:       ; %bb.0: ; %entry
2918; GFX10-CU-NEXT:    s_clause 0x1
2919; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2920; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2921; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2922; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2923; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2924; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2925; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2926; GFX10-CU-NEXT:    s_endpgm
2927;
2928; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_acquire_cmpxchg:
2929; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2930; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
2931; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
2932; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
2933; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
2934; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2935; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2936; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2937; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2938; SKIP-CACHE-INV-NEXT:    s_endpgm
2939;
2940; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_acquire_cmpxchg:
2941; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2942; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2943; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2944; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2945; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2946; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2947; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2948; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2949;
2950; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_acquire_cmpxchg:
2951; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2952; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2953; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2954; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2955; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2956; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2957; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2958; GFX90A-TGSPLIT-NEXT:    s_endpgm
2959;
2960; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acquire_acquire_cmpxchg:
2961; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2962; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2963; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2964; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2965; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2966; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2967; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2968; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2969;
2970; GFX940-TGSPLIT-LABEL: global_singlethread_acquire_acquire_cmpxchg:
2971; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2972; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
2973; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
2974; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2975; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2976; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2977; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
2978; GFX940-TGSPLIT-NEXT:    s_endpgm
2979;
2980; GFX11-WGP-LABEL: global_singlethread_acquire_acquire_cmpxchg:
2981; GFX11-WGP:       ; %bb.0: ; %entry
2982; GFX11-WGP-NEXT:    s_clause 0x1
2983; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
2984; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
2985; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
2986; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2987; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2988; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2989; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
2990; GFX11-WGP-NEXT:    s_endpgm
2991;
2992; GFX11-CU-LABEL: global_singlethread_acquire_acquire_cmpxchg:
2993; GFX11-CU:       ; %bb.0: ; %entry
2994; GFX11-CU-NEXT:    s_clause 0x1
2995; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
2996; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
2997; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
2998; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2999; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3000; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3001; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
3002; GFX11-CU-NEXT:    s_endpgm
3003    i32 addrspace(1)* %out, i32 %in, i32 %old) {
3004entry:
3005  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3006  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire acquire
3007  ret void
3008}
3009
3010define amdgpu_kernel void @global_singlethread_release_acquire_cmpxchg(
3011; GFX6-LABEL: global_singlethread_release_acquire_cmpxchg:
3012; GFX6:       ; %bb.0: ; %entry
3013; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
3014; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3015; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3016; GFX6-NEXT:    s_mov_b32 s2, -1
3017; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3018; GFX6-NEXT:    v_mov_b32_e32 v0, s6
3019; GFX6-NEXT:    v_mov_b32_e32 v1, s7
3020; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3021; GFX6-NEXT:    s_endpgm
3022;
3023; GFX7-LABEL: global_singlethread_release_acquire_cmpxchg:
3024; GFX7:       ; %bb.0: ; %entry
3025; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3026; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3027; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3028; GFX7-NEXT:    s_add_u32 s0, s0, 16
3029; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3030; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3031; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3032; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3033; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3034; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3035; GFX7-NEXT:    s_endpgm
3036;
3037; GFX10-WGP-LABEL: global_singlethread_release_acquire_cmpxchg:
3038; GFX10-WGP:       ; %bb.0: ; %entry
3039; GFX10-WGP-NEXT:    s_clause 0x1
3040; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3041; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3042; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3043; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3044; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3045; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3046; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3047; GFX10-WGP-NEXT:    s_endpgm
3048;
3049; GFX10-CU-LABEL: global_singlethread_release_acquire_cmpxchg:
3050; GFX10-CU:       ; %bb.0: ; %entry
3051; GFX10-CU-NEXT:    s_clause 0x1
3052; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3053; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3054; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3055; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3056; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3057; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3058; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3059; GFX10-CU-NEXT:    s_endpgm
3060;
3061; SKIP-CACHE-INV-LABEL: global_singlethread_release_acquire_cmpxchg:
3062; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3063; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
3064; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
3065; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3066; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3067; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3068; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3069; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3070; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3071; SKIP-CACHE-INV-NEXT:    s_endpgm
3072;
3073; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_release_acquire_cmpxchg:
3074; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3075; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3076; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3077; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3078; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3079; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3080; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3081; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3082;
3083; GFX90A-TGSPLIT-LABEL: global_singlethread_release_acquire_cmpxchg:
3084; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3085; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3086; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3087; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3088; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3089; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3090; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3091; GFX90A-TGSPLIT-NEXT:    s_endpgm
3092;
3093; GFX940-NOTTGSPLIT-LABEL: global_singlethread_release_acquire_cmpxchg:
3094; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3095; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3096; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3097; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3098; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3099; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3100; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
3101; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3102;
3103; GFX940-TGSPLIT-LABEL: global_singlethread_release_acquire_cmpxchg:
3104; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3105; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3106; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3107; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3108; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3109; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3110; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
3111; GFX940-TGSPLIT-NEXT:    s_endpgm
3112;
3113; GFX11-WGP-LABEL: global_singlethread_release_acquire_cmpxchg:
3114; GFX11-WGP:       ; %bb.0: ; %entry
3115; GFX11-WGP-NEXT:    s_clause 0x1
3116; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
3117; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
3118; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
3119; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3120; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3121; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3122; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
3123; GFX11-WGP-NEXT:    s_endpgm
3124;
3125; GFX11-CU-LABEL: global_singlethread_release_acquire_cmpxchg:
3126; GFX11-CU:       ; %bb.0: ; %entry
3127; GFX11-CU-NEXT:    s_clause 0x1
3128; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
3129; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
3130; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
3131; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3132; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3133; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3134; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
3135; GFX11-CU-NEXT:    s_endpgm
3136    i32 addrspace(1)* %out, i32 %in, i32 %old) {
3137entry:
3138  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3139  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") release acquire
3140  ret void
3141}
3142
3143define amdgpu_kernel void @global_singlethread_acq_rel_acquire_cmpxchg(
3144; GFX6-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
3145; GFX6:       ; %bb.0: ; %entry
3146; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
3147; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3148; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3149; GFX6-NEXT:    s_mov_b32 s2, -1
3150; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3151; GFX6-NEXT:    v_mov_b32_e32 v0, s6
3152; GFX6-NEXT:    v_mov_b32_e32 v1, s7
3153; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3154; GFX6-NEXT:    s_endpgm
3155;
3156; GFX7-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
3157; GFX7:       ; %bb.0: ; %entry
3158; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3159; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3160; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3161; GFX7-NEXT:    s_add_u32 s0, s0, 16
3162; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3163; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3164; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3165; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3166; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3167; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3168; GFX7-NEXT:    s_endpgm
3169;
3170; GFX10-WGP-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
3171; GFX10-WGP:       ; %bb.0: ; %entry
3172; GFX10-WGP-NEXT:    s_clause 0x1
3173; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3174; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3175; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3176; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3177; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3178; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3179; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3180; GFX10-WGP-NEXT:    s_endpgm
3181;
3182; GFX10-CU-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
3183; GFX10-CU:       ; %bb.0: ; %entry
3184; GFX10-CU-NEXT:    s_clause 0x1
3185; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3186; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3187; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3188; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3189; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3190; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3191; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3192; GFX10-CU-NEXT:    s_endpgm
3193;
3194; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
3195; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3196; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
3197; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
3198; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3199; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3200; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3201; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3202; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3203; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3204; SKIP-CACHE-INV-NEXT:    s_endpgm
3205;
3206; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
3207; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3208; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3209; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3210; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3211; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3212; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3213; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3214; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3215;
3216; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
3217; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3218; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3219; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3220; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3221; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3222; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3223; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3224; GFX90A-TGSPLIT-NEXT:    s_endpgm
3225;
3226; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
3227; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3228; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3229; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3230; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3231; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3232; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3233; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
3234; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3235;
3236; GFX940-TGSPLIT-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
3237; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3238; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3239; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3240; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3241; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3242; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3243; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
3244; GFX940-TGSPLIT-NEXT:    s_endpgm
3245;
3246; GFX11-WGP-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
3247; GFX11-WGP:       ; %bb.0: ; %entry
3248; GFX11-WGP-NEXT:    s_clause 0x1
3249; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
3250; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
3251; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
3252; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3253; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3254; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3255; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
3256; GFX11-WGP-NEXT:    s_endpgm
3257;
3258; GFX11-CU-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
3259; GFX11-CU:       ; %bb.0: ; %entry
3260; GFX11-CU-NEXT:    s_clause 0x1
3261; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
3262; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
3263; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
3264; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3265; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3266; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3267; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
3268; GFX11-CU-NEXT:    s_endpgm
3269    i32 addrspace(1)* %out, i32 %in, i32 %old) {
3270entry:
3271  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3272  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel acquire
3273  ret void
3274}
3275
3276define amdgpu_kernel void @global_singlethread_seq_cst_acquire_cmpxchg(
3277; GFX6-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
3278; GFX6:       ; %bb.0: ; %entry
3279; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
3280; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3281; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3282; GFX6-NEXT:    s_mov_b32 s2, -1
3283; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3284; GFX6-NEXT:    v_mov_b32_e32 v0, s6
3285; GFX6-NEXT:    v_mov_b32_e32 v1, s7
3286; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3287; GFX6-NEXT:    s_endpgm
3288;
3289; GFX7-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
3290; GFX7:       ; %bb.0: ; %entry
3291; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3292; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3293; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3294; GFX7-NEXT:    s_add_u32 s0, s0, 16
3295; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3296; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3297; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3298; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3299; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3300; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3301; GFX7-NEXT:    s_endpgm
3302;
3303; GFX10-WGP-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
3304; GFX10-WGP:       ; %bb.0: ; %entry
3305; GFX10-WGP-NEXT:    s_clause 0x1
3306; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3307; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3308; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3309; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3310; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3311; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3312; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3313; GFX10-WGP-NEXT:    s_endpgm
3314;
3315; GFX10-CU-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
3316; GFX10-CU:       ; %bb.0: ; %entry
3317; GFX10-CU-NEXT:    s_clause 0x1
3318; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3319; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3320; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3321; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3322; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3323; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3324; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3325; GFX10-CU-NEXT:    s_endpgm
3326;
3327; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
3328; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3329; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
3330; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
3331; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3332; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3333; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3334; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3335; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3336; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3337; SKIP-CACHE-INV-NEXT:    s_endpgm
3338;
3339; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
3340; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3341; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3342; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3343; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3344; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3345; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3346; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3347; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3348;
3349; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
3350; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3351; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3352; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3353; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3354; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3355; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3356; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3357; GFX90A-TGSPLIT-NEXT:    s_endpgm
3358;
3359; GFX940-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
3360; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3361; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3362; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3363; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3364; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3365; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3366; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
3367; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3368;
3369; GFX940-TGSPLIT-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
3370; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3371; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3372; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3373; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3374; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3375; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3376; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
3377; GFX940-TGSPLIT-NEXT:    s_endpgm
3378;
3379; GFX11-WGP-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
3380; GFX11-WGP:       ; %bb.0: ; %entry
3381; GFX11-WGP-NEXT:    s_clause 0x1
3382; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
3383; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
3384; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
3385; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3386; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3387; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3388; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
3389; GFX11-WGP-NEXT:    s_endpgm
3390;
3391; GFX11-CU-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
3392; GFX11-CU:       ; %bb.0: ; %entry
3393; GFX11-CU-NEXT:    s_clause 0x1
3394; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
3395; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
3396; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
3397; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3398; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3399; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3400; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
3401; GFX11-CU-NEXT:    s_endpgm
3402    i32 addrspace(1)* %out, i32 %in, i32 %old) {
3403entry:
3404  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3405  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst acquire
3406  ret void
3407}
3408
3409define amdgpu_kernel void @global_singlethread_monotonic_seq_cst_cmpxchg(
3410; GFX6-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
3411; GFX6:       ; %bb.0: ; %entry
3412; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
3413; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3414; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3415; GFX6-NEXT:    s_mov_b32 s2, -1
3416; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3417; GFX6-NEXT:    v_mov_b32_e32 v0, s6
3418; GFX6-NEXT:    v_mov_b32_e32 v1, s7
3419; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3420; GFX6-NEXT:    s_endpgm
3421;
3422; GFX7-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
3423; GFX7:       ; %bb.0: ; %entry
3424; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3425; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3426; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3427; GFX7-NEXT:    s_add_u32 s0, s0, 16
3428; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3429; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3430; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3431; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3432; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3433; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3434; GFX7-NEXT:    s_endpgm
3435;
3436; GFX10-WGP-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
3437; GFX10-WGP:       ; %bb.0: ; %entry
3438; GFX10-WGP-NEXT:    s_clause 0x1
3439; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3440; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3441; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3442; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3443; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3444; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3445; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3446; GFX10-WGP-NEXT:    s_endpgm
3447;
3448; GFX10-CU-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
3449; GFX10-CU:       ; %bb.0: ; %entry
3450; GFX10-CU-NEXT:    s_clause 0x1
3451; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3452; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3453; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3454; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3455; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3456; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3457; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3458; GFX10-CU-NEXT:    s_endpgm
3459;
3460; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
3461; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3462; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
3463; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
3464; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3465; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3466; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3467; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3468; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3469; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3470; SKIP-CACHE-INV-NEXT:    s_endpgm
3471;
3472; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
3473; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3474; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3475; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3476; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3477; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3478; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3479; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3480; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3481;
3482; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
3483; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3484; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3485; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3486; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3487; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3488; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3489; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3490; GFX90A-TGSPLIT-NEXT:    s_endpgm
3491;
3492; GFX940-NOTTGSPLIT-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
3493; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3494; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3495; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3496; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3497; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3498; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3499; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
3500; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3501;
3502; GFX940-TGSPLIT-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
3503; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3504; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3505; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3506; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3507; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3508; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3509; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
3510; GFX940-TGSPLIT-NEXT:    s_endpgm
3511;
3512; GFX11-WGP-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
3513; GFX11-WGP:       ; %bb.0: ; %entry
3514; GFX11-WGP-NEXT:    s_clause 0x1
3515; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
3516; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
3517; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
3518; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3519; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3520; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3521; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
3522; GFX11-WGP-NEXT:    s_endpgm
3523;
3524; GFX11-CU-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
3525; GFX11-CU:       ; %bb.0: ; %entry
3526; GFX11-CU-NEXT:    s_clause 0x1
3527; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
3528; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
3529; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
3530; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3531; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3532; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3533; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
3534; GFX11-CU-NEXT:    s_endpgm
3535    i32 addrspace(1)* %out, i32 %in, i32 %old) {
3536entry:
3537  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3538  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic seq_cst
3539  ret void
3540}
3541
3542define amdgpu_kernel void @global_singlethread_acquire_seq_cst_cmpxchg(
3543; GFX6-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
3544; GFX6:       ; %bb.0: ; %entry
3545; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
3546; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3547; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3548; GFX6-NEXT:    s_mov_b32 s2, -1
3549; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3550; GFX6-NEXT:    v_mov_b32_e32 v0, s6
3551; GFX6-NEXT:    v_mov_b32_e32 v1, s7
3552; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3553; GFX6-NEXT:    s_endpgm
3554;
3555; GFX7-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
3556; GFX7:       ; %bb.0: ; %entry
3557; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3558; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3559; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3560; GFX7-NEXT:    s_add_u32 s0, s0, 16
3561; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3562; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3563; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3564; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3565; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3566; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3567; GFX7-NEXT:    s_endpgm
3568;
3569; GFX10-WGP-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
3570; GFX10-WGP:       ; %bb.0: ; %entry
3571; GFX10-WGP-NEXT:    s_clause 0x1
3572; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3573; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3574; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3575; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3576; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3577; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3578; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3579; GFX10-WGP-NEXT:    s_endpgm
3580;
3581; GFX10-CU-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
3582; GFX10-CU:       ; %bb.0: ; %entry
3583; GFX10-CU-NEXT:    s_clause 0x1
3584; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3585; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3586; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3587; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3588; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3589; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3590; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3591; GFX10-CU-NEXT:    s_endpgm
3592;
3593; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
3594; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3595; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
3596; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
3597; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3598; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3599; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3600; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3601; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3602; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3603; SKIP-CACHE-INV-NEXT:    s_endpgm
3604;
3605; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
3606; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3607; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3608; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3609; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3610; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3611; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3612; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3613; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3614;
3615; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
3616; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3617; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3618; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3619; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3620; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3621; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3622; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3623; GFX90A-TGSPLIT-NEXT:    s_endpgm
3624;
3625; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
3626; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3627; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3628; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3629; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3630; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3631; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3632; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
3633; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3634;
3635; GFX940-TGSPLIT-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
3636; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3637; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3638; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3639; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3640; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3641; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3642; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
3643; GFX940-TGSPLIT-NEXT:    s_endpgm
3644;
3645; GFX11-WGP-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
3646; GFX11-WGP:       ; %bb.0: ; %entry
3647; GFX11-WGP-NEXT:    s_clause 0x1
3648; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
3649; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
3650; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
3651; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3652; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3653; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3654; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
3655; GFX11-WGP-NEXT:    s_endpgm
3656;
3657; GFX11-CU-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
3658; GFX11-CU:       ; %bb.0: ; %entry
3659; GFX11-CU-NEXT:    s_clause 0x1
3660; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
3661; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
3662; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
3663; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3664; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3665; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3666; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
3667; GFX11-CU-NEXT:    s_endpgm
3668    i32 addrspace(1)* %out, i32 %in, i32 %old) {
3669entry:
3670  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3671  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire seq_cst
3672  ret void
3673}
3674
3675define amdgpu_kernel void @global_singlethread_release_seq_cst_cmpxchg(
3676; GFX6-LABEL: global_singlethread_release_seq_cst_cmpxchg:
3677; GFX6:       ; %bb.0: ; %entry
3678; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
3679; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3680; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3681; GFX6-NEXT:    s_mov_b32 s2, -1
3682; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3683; GFX6-NEXT:    v_mov_b32_e32 v0, s6
3684; GFX6-NEXT:    v_mov_b32_e32 v1, s7
3685; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3686; GFX6-NEXT:    s_endpgm
3687;
3688; GFX7-LABEL: global_singlethread_release_seq_cst_cmpxchg:
3689; GFX7:       ; %bb.0: ; %entry
3690; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3691; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3692; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3693; GFX7-NEXT:    s_add_u32 s0, s0, 16
3694; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3695; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3696; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3697; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3698; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3699; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3700; GFX7-NEXT:    s_endpgm
3701;
3702; GFX10-WGP-LABEL: global_singlethread_release_seq_cst_cmpxchg:
3703; GFX10-WGP:       ; %bb.0: ; %entry
3704; GFX10-WGP-NEXT:    s_clause 0x1
3705; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3706; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3707; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3708; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3709; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3710; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3711; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3712; GFX10-WGP-NEXT:    s_endpgm
3713;
3714; GFX10-CU-LABEL: global_singlethread_release_seq_cst_cmpxchg:
3715; GFX10-CU:       ; %bb.0: ; %entry
3716; GFX10-CU-NEXT:    s_clause 0x1
3717; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3718; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3719; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3720; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3721; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3722; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3723; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3724; GFX10-CU-NEXT:    s_endpgm
3725;
3726; SKIP-CACHE-INV-LABEL: global_singlethread_release_seq_cst_cmpxchg:
3727; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3728; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
3729; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
3730; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3731; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3732; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3733; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3734; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3735; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3736; SKIP-CACHE-INV-NEXT:    s_endpgm
3737;
3738; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_release_seq_cst_cmpxchg:
3739; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3740; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3741; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3742; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3743; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3744; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3745; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3746; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3747;
3748; GFX90A-TGSPLIT-LABEL: global_singlethread_release_seq_cst_cmpxchg:
3749; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3750; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3751; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3752; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3753; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3754; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3755; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3756; GFX90A-TGSPLIT-NEXT:    s_endpgm
3757;
3758; GFX940-NOTTGSPLIT-LABEL: global_singlethread_release_seq_cst_cmpxchg:
3759; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3760; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3761; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3762; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3763; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3764; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3765; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
3766; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3767;
3768; GFX940-TGSPLIT-LABEL: global_singlethread_release_seq_cst_cmpxchg:
3769; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3770; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3771; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3772; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3773; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3774; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3775; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
3776; GFX940-TGSPLIT-NEXT:    s_endpgm
3777;
3778; GFX11-WGP-LABEL: global_singlethread_release_seq_cst_cmpxchg:
3779; GFX11-WGP:       ; %bb.0: ; %entry
3780; GFX11-WGP-NEXT:    s_clause 0x1
3781; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
3782; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
3783; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
3784; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3785; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3786; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3787; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
3788; GFX11-WGP-NEXT:    s_endpgm
3789;
3790; GFX11-CU-LABEL: global_singlethread_release_seq_cst_cmpxchg:
3791; GFX11-CU:       ; %bb.0: ; %entry
3792; GFX11-CU-NEXT:    s_clause 0x1
3793; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
3794; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
3795; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
3796; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3797; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3798; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3799; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
3800; GFX11-CU-NEXT:    s_endpgm
3801    i32 addrspace(1)* %out, i32 %in, i32 %old) {
3802entry:
3803  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3804  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") release seq_cst
3805  ret void
3806}
3807
3808define amdgpu_kernel void @global_singlethread_acq_rel_seq_cst_cmpxchg(
3809; GFX6-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
3810; GFX6:       ; %bb.0: ; %entry
3811; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
3812; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3813; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3814; GFX6-NEXT:    s_mov_b32 s2, -1
3815; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3816; GFX6-NEXT:    v_mov_b32_e32 v0, s6
3817; GFX6-NEXT:    v_mov_b32_e32 v1, s7
3818; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3819; GFX6-NEXT:    s_endpgm
3820;
3821; GFX7-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
3822; GFX7:       ; %bb.0: ; %entry
3823; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3824; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3825; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3826; GFX7-NEXT:    s_add_u32 s0, s0, 16
3827; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3828; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3829; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3830; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3831; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3832; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3833; GFX7-NEXT:    s_endpgm
3834;
3835; GFX10-WGP-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
3836; GFX10-WGP:       ; %bb.0: ; %entry
3837; GFX10-WGP-NEXT:    s_clause 0x1
3838; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3839; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3840; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3841; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3842; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3843; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3844; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3845; GFX10-WGP-NEXT:    s_endpgm
3846;
3847; GFX10-CU-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
3848; GFX10-CU:       ; %bb.0: ; %entry
3849; GFX10-CU-NEXT:    s_clause 0x1
3850; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3851; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3852; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3853; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3854; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3855; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3856; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3857; GFX10-CU-NEXT:    s_endpgm
3858;
3859; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
3860; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3861; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
3862; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
3863; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3864; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3865; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3866; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3867; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3868; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3869; SKIP-CACHE-INV-NEXT:    s_endpgm
3870;
3871; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
3872; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3873; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3874; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3875; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3876; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3877; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3878; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3879; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3880;
3881; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
3882; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3883; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3884; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3885; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3886; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3887; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3888; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3889; GFX90A-TGSPLIT-NEXT:    s_endpgm
3890;
3891; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
3892; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3893; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3894; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3895; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3896; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3897; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3898; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
3899; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3900;
3901; GFX940-TGSPLIT-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
3902; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3903; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
3904; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
3905; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3906; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3907; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3908; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
3909; GFX940-TGSPLIT-NEXT:    s_endpgm
3910;
3911; GFX11-WGP-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
3912; GFX11-WGP:       ; %bb.0: ; %entry
3913; GFX11-WGP-NEXT:    s_clause 0x1
3914; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
3915; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
3916; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
3917; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3918; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3919; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3920; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
3921; GFX11-WGP-NEXT:    s_endpgm
3922;
3923; GFX11-CU-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
3924; GFX11-CU:       ; %bb.0: ; %entry
3925; GFX11-CU-NEXT:    s_clause 0x1
3926; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
3927; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
3928; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
3929; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3930; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3931; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3932; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
3933; GFX11-CU-NEXT:    s_endpgm
3934    i32 addrspace(1)* %out, i32 %in, i32 %old) {
3935entry:
3936  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3937  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel seq_cst
3938  ret void
3939}
3940
3941define amdgpu_kernel void @global_singlethread_seq_cst_seq_cst_cmpxchg(
3942; GFX6-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
3943; GFX6:       ; %bb.0: ; %entry
3944; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
3945; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3946; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3947; GFX6-NEXT:    s_mov_b32 s2, -1
3948; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3949; GFX6-NEXT:    v_mov_b32_e32 v0, s6
3950; GFX6-NEXT:    v_mov_b32_e32 v1, s7
3951; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3952; GFX6-NEXT:    s_endpgm
3953;
3954; GFX7-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
3955; GFX7:       ; %bb.0: ; %entry
3956; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3957; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3958; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3959; GFX7-NEXT:    s_add_u32 s0, s0, 16
3960; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3961; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3962; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3963; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3964; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3965; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3966; GFX7-NEXT:    s_endpgm
3967;
3968; GFX10-WGP-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
3969; GFX10-WGP:       ; %bb.0: ; %entry
3970; GFX10-WGP-NEXT:    s_clause 0x1
3971; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3972; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3973; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3974; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3975; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3976; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3977; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3978; GFX10-WGP-NEXT:    s_endpgm
3979;
3980; GFX10-CU-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
3981; GFX10-CU:       ; %bb.0: ; %entry
3982; GFX10-CU-NEXT:    s_clause 0x1
3983; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3984; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3985; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3986; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3987; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3988; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3989; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
3990; GFX10-CU-NEXT:    s_endpgm
3991;
3992; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
3993; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3994; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
3995; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
3996; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
3997; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
3998; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3999; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4000; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4001; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
4002; SKIP-CACHE-INV-NEXT:    s_endpgm
4003;
4004; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
4005; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4006; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4007; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4008; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4009; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4010; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4011; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
4012; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4013;
4014; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
4015; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4016; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4017; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4018; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4019; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4020; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4021; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
4022; GFX90A-TGSPLIT-NEXT:    s_endpgm
4023;
4024; GFX940-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
4025; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4026; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4027; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4028; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4029; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4030; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4031; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
4032; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4033;
4034; GFX940-TGSPLIT-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
4035; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4036; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4037; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4038; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4039; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4040; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4041; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
4042; GFX940-TGSPLIT-NEXT:    s_endpgm
4043;
4044; GFX11-WGP-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
4045; GFX11-WGP:       ; %bb.0: ; %entry
4046; GFX11-WGP-NEXT:    s_clause 0x1
4047; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
4048; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
4049; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
4050; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4051; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
4052; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
4053; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
4054; GFX11-WGP-NEXT:    s_endpgm
4055;
4056; GFX11-CU-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
4057; GFX11-CU:       ; %bb.0: ; %entry
4058; GFX11-CU-NEXT:    s_clause 0x1
4059; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
4060; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
4061; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
4062; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4063; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
4064; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
4065; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
4066; GFX11-CU-NEXT:    s_endpgm
4067    i32 addrspace(1)* %out, i32 %in, i32 %old) {
4068entry:
4069  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
4070  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst seq_cst
4071  ret void
4072}
4073
4074define amdgpu_kernel void @global_singlethread_monotonic_monotonic_ret_cmpxchg(
4075; GFX6-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
4076; GFX6:       ; %bb.0: ; %entry
4077; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
4078; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4079; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4080; GFX6-NEXT:    s_mov_b32 s2, -1
4081; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4082; GFX6-NEXT:    v_mov_b32_e32 v0, s6
4083; GFX6-NEXT:    v_mov_b32_e32 v1, s7
4084; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4085; GFX6-NEXT:    s_waitcnt vmcnt(0)
4086; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4087; GFX6-NEXT:    s_endpgm
4088;
4089; GFX7-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
4090; GFX7:       ; %bb.0: ; %entry
4091; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4092; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4093; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4094; GFX7-NEXT:    s_add_u32 s4, s0, 16
4095; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4096; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4097; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4098; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4099; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4100; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4101; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4102; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4103; GFX7-NEXT:    s_waitcnt vmcnt(0)
4104; GFX7-NEXT:    flat_store_dword v[0:1], v2
4105; GFX7-NEXT:    s_endpgm
4106;
4107; GFX10-WGP-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
4108; GFX10-WGP:       ; %bb.0: ; %entry
4109; GFX10-WGP-NEXT:    s_clause 0x1
4110; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4111; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4112; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
4113; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4114; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4115; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4116; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4117; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4118; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
4119; GFX10-WGP-NEXT:    s_endpgm
4120;
4121; GFX10-CU-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
4122; GFX10-CU:       ; %bb.0: ; %entry
4123; GFX10-CU-NEXT:    s_clause 0x1
4124; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4125; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4126; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
4127; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4128; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4129; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4130; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4131; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4132; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
4133; GFX10-CU-NEXT:    s_endpgm
4134;
4135; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
4136; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4137; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
4138; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
4139; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
4140; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
4141; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4142; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4143; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4144; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4145; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4146; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4147; SKIP-CACHE-INV-NEXT:    s_endpgm
4148;
4149; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
4150; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4151; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4152; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4153; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4154; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4155; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4156; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4157; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4158; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4159; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4160;
4161; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
4162; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4163; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4164; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4165; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4166; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4167; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4168; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4169; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4170; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4171; GFX90A-TGSPLIT-NEXT:    s_endpgm
4172;
4173; GFX940-NOTTGSPLIT-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
4174; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4175; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4176; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4177; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4178; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4179; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4180; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4181; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4182; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4183; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4184;
4185; GFX940-TGSPLIT-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
4186; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4187; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4188; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4189; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4190; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4191; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4192; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4193; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4194; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4195; GFX940-TGSPLIT-NEXT:    s_endpgm
4196;
4197; GFX11-WGP-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
4198; GFX11-WGP:       ; %bb.0: ; %entry
4199; GFX11-WGP-NEXT:    s_clause 0x1
4200; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
4201; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
4202; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
4203; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4204; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
4205; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
4206; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
4207; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
4208; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
4209; GFX11-WGP-NEXT:    s_endpgm
4210;
4211; GFX11-CU-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
4212; GFX11-CU:       ; %bb.0: ; %entry
4213; GFX11-CU-NEXT:    s_clause 0x1
4214; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
4215; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
4216; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
4217; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4218; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
4219; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
4220; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
4221; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
4222; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
4223; GFX11-CU-NEXT:    s_endpgm
4224    i32 addrspace(1)* %out, i32 %in, i32 %old) {
4225entry:
4226  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
4227  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic monotonic
4228  %val0 = extractvalue { i32, i1 } %val, 0
4229  store i32 %val0, i32 addrspace(1)* %out, align 4
4230  ret void
4231}
4232
4233define amdgpu_kernel void @global_singlethread_acquire_monotonic_ret_cmpxchg(
4234; GFX6-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
4235; GFX6:       ; %bb.0: ; %entry
4236; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
4237; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4238; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4239; GFX6-NEXT:    s_mov_b32 s2, -1
4240; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4241; GFX6-NEXT:    v_mov_b32_e32 v0, s6
4242; GFX6-NEXT:    v_mov_b32_e32 v1, s7
4243; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4244; GFX6-NEXT:    s_waitcnt vmcnt(0)
4245; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4246; GFX6-NEXT:    s_endpgm
4247;
4248; GFX7-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
4249; GFX7:       ; %bb.0: ; %entry
4250; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4251; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4252; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4253; GFX7-NEXT:    s_add_u32 s4, s0, 16
4254; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4255; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4256; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4257; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4258; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4259; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4260; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4261; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4262; GFX7-NEXT:    s_waitcnt vmcnt(0)
4263; GFX7-NEXT:    flat_store_dword v[0:1], v2
4264; GFX7-NEXT:    s_endpgm
4265;
4266; GFX10-WGP-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
4267; GFX10-WGP:       ; %bb.0: ; %entry
4268; GFX10-WGP-NEXT:    s_clause 0x1
4269; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4270; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4271; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
4272; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4273; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4274; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4275; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4276; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4277; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
4278; GFX10-WGP-NEXT:    s_endpgm
4279;
4280; GFX10-CU-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
4281; GFX10-CU:       ; %bb.0: ; %entry
4282; GFX10-CU-NEXT:    s_clause 0x1
4283; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4284; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4285; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
4286; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4287; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4288; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4289; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4290; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4291; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
4292; GFX10-CU-NEXT:    s_endpgm
4293;
4294; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
4295; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4296; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
4297; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
4298; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
4299; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
4300; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4301; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4302; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4303; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4304; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4305; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4306; SKIP-CACHE-INV-NEXT:    s_endpgm
4307;
4308; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
4309; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4310; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4311; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4312; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4313; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4314; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4315; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4316; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4317; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4318; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4319;
4320; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
4321; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4322; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4323; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4324; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4325; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4326; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4327; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4328; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4329; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4330; GFX90A-TGSPLIT-NEXT:    s_endpgm
4331;
4332; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
4333; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4334; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4335; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4336; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4337; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4338; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4339; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4340; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4341; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4342; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4343;
4344; GFX940-TGSPLIT-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
4345; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4346; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4347; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4348; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4349; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4350; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4351; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4352; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4353; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4354; GFX940-TGSPLIT-NEXT:    s_endpgm
4355;
4356; GFX11-WGP-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
4357; GFX11-WGP:       ; %bb.0: ; %entry
4358; GFX11-WGP-NEXT:    s_clause 0x1
4359; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
4360; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
4361; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
4362; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4363; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
4364; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
4365; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
4366; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
4367; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
4368; GFX11-WGP-NEXT:    s_endpgm
4369;
4370; GFX11-CU-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
4371; GFX11-CU:       ; %bb.0: ; %entry
4372; GFX11-CU-NEXT:    s_clause 0x1
4373; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
4374; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
4375; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
4376; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4377; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
4378; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
4379; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
4380; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
4381; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
4382; GFX11-CU-NEXT:    s_endpgm
4383    i32 addrspace(1)* %out, i32 %in, i32 %old) {
4384entry:
4385  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
4386  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire monotonic
4387  %val0 = extractvalue { i32, i1 } %val, 0
4388  store i32 %val0, i32 addrspace(1)* %out, align 4
4389  ret void
4390}
4391
4392define amdgpu_kernel void @global_singlethread_release_monotonic_ret_cmpxchg(
4393; GFX6-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
4394; GFX6:       ; %bb.0: ; %entry
4395; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
4396; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4397; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4398; GFX6-NEXT:    s_mov_b32 s2, -1
4399; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4400; GFX6-NEXT:    v_mov_b32_e32 v0, s6
4401; GFX6-NEXT:    v_mov_b32_e32 v1, s7
4402; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4403; GFX6-NEXT:    s_waitcnt vmcnt(0)
4404; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4405; GFX6-NEXT:    s_endpgm
4406;
4407; GFX7-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
4408; GFX7:       ; %bb.0: ; %entry
4409; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4410; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4411; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4412; GFX7-NEXT:    s_add_u32 s4, s0, 16
4413; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4414; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4415; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4416; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4417; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4418; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4419; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4420; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4421; GFX7-NEXT:    s_waitcnt vmcnt(0)
4422; GFX7-NEXT:    flat_store_dword v[0:1], v2
4423; GFX7-NEXT:    s_endpgm
4424;
4425; GFX10-WGP-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
4426; GFX10-WGP:       ; %bb.0: ; %entry
4427; GFX10-WGP-NEXT:    s_clause 0x1
4428; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4429; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4430; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
4431; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4432; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4433; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4434; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4435; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4436; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
4437; GFX10-WGP-NEXT:    s_endpgm
4438;
4439; GFX10-CU-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
4440; GFX10-CU:       ; %bb.0: ; %entry
4441; GFX10-CU-NEXT:    s_clause 0x1
4442; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4443; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4444; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
4445; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4446; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4447; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4448; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4449; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4450; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
4451; GFX10-CU-NEXT:    s_endpgm
4452;
4453; SKIP-CACHE-INV-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
4454; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4455; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
4456; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
4457; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
4458; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
4459; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4460; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4461; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4462; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4463; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4464; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4465; SKIP-CACHE-INV-NEXT:    s_endpgm
4466;
4467; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
4468; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4469; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4470; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4471; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4472; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4473; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4474; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4475; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4476; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4477; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4478;
4479; GFX90A-TGSPLIT-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
4480; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4481; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4482; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4483; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4484; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4485; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4486; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4487; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4488; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4489; GFX90A-TGSPLIT-NEXT:    s_endpgm
4490;
4491; GFX940-NOTTGSPLIT-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
4492; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4493; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4494; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4495; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4496; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4497; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4498; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4499; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4500; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4501; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4502;
4503; GFX940-TGSPLIT-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
4504; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4505; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4506; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4507; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4508; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4509; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4510; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4511; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4512; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4513; GFX940-TGSPLIT-NEXT:    s_endpgm
4514;
4515; GFX11-WGP-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
4516; GFX11-WGP:       ; %bb.0: ; %entry
4517; GFX11-WGP-NEXT:    s_clause 0x1
4518; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
4519; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
4520; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
4521; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4522; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
4523; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
4524; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
4525; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
4526; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
4527; GFX11-WGP-NEXT:    s_endpgm
4528;
4529; GFX11-CU-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
4530; GFX11-CU:       ; %bb.0: ; %entry
4531; GFX11-CU-NEXT:    s_clause 0x1
4532; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
4533; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
4534; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
4535; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4536; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
4537; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
4538; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
4539; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
4540; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
4541; GFX11-CU-NEXT:    s_endpgm
4542    i32 addrspace(1)* %out, i32 %in, i32 %old) {
4543entry:
4544  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
4545  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") release monotonic
4546  %val0 = extractvalue { i32, i1 } %val, 0
4547  store i32 %val0, i32 addrspace(1)* %out, align 4
4548  ret void
4549}
4550
4551define amdgpu_kernel void @global_singlethread_acq_rel_monotonic_ret_cmpxchg(
4552; GFX6-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
4553; GFX6:       ; %bb.0: ; %entry
4554; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
4555; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4556; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4557; GFX6-NEXT:    s_mov_b32 s2, -1
4558; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4559; GFX6-NEXT:    v_mov_b32_e32 v0, s6
4560; GFX6-NEXT:    v_mov_b32_e32 v1, s7
4561; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4562; GFX6-NEXT:    s_waitcnt vmcnt(0)
4563; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4564; GFX6-NEXT:    s_endpgm
4565;
4566; GFX7-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
4567; GFX7:       ; %bb.0: ; %entry
4568; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4569; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4570; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4571; GFX7-NEXT:    s_add_u32 s4, s0, 16
4572; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4573; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4574; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4575; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4576; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4577; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4578; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4579; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4580; GFX7-NEXT:    s_waitcnt vmcnt(0)
4581; GFX7-NEXT:    flat_store_dword v[0:1], v2
4582; GFX7-NEXT:    s_endpgm
4583;
4584; GFX10-WGP-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
4585; GFX10-WGP:       ; %bb.0: ; %entry
4586; GFX10-WGP-NEXT:    s_clause 0x1
4587; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4588; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4589; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
4590; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4591; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4592; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4593; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4594; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4595; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
4596; GFX10-WGP-NEXT:    s_endpgm
4597;
4598; GFX10-CU-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
4599; GFX10-CU:       ; %bb.0: ; %entry
4600; GFX10-CU-NEXT:    s_clause 0x1
4601; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4602; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4603; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
4604; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4605; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4606; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4607; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4608; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4609; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
4610; GFX10-CU-NEXT:    s_endpgm
4611;
4612; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
4613; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4614; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
4615; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
4616; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
4617; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
4618; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4619; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4620; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4621; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4622; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4623; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4624; SKIP-CACHE-INV-NEXT:    s_endpgm
4625;
4626; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
4627; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4628; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4629; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4630; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4631; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4632; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4633; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4634; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4635; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4636; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4637;
4638; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
4639; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4640; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4641; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4642; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4643; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4644; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4645; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4646; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4647; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4648; GFX90A-TGSPLIT-NEXT:    s_endpgm
4649;
4650; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
4651; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4652; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4653; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4654; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4655; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4656; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4657; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4658; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4659; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4660; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4661;
4662; GFX940-TGSPLIT-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
4663; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4664; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4665; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4666; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4667; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4668; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4669; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4670; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4671; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4672; GFX940-TGSPLIT-NEXT:    s_endpgm
4673;
4674; GFX11-WGP-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
4675; GFX11-WGP:       ; %bb.0: ; %entry
4676; GFX11-WGP-NEXT:    s_clause 0x1
4677; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
4678; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
4679; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
4680; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4681; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
4682; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
4683; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
4684; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
4685; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
4686; GFX11-WGP-NEXT:    s_endpgm
4687;
4688; GFX11-CU-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
4689; GFX11-CU:       ; %bb.0: ; %entry
4690; GFX11-CU-NEXT:    s_clause 0x1
4691; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
4692; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
4693; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
4694; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4695; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
4696; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
4697; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
4698; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
4699; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
4700; GFX11-CU-NEXT:    s_endpgm
4701    i32 addrspace(1)* %out, i32 %in, i32 %old) {
4702entry:
4703  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
4704  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel monotonic
4705  %val0 = extractvalue { i32, i1 } %val, 0
4706  store i32 %val0, i32 addrspace(1)* %out, align 4
4707  ret void
4708}
4709
4710define amdgpu_kernel void @global_singlethread_seq_cst_monotonic_ret_cmpxchg(
4711; GFX6-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
4712; GFX6:       ; %bb.0: ; %entry
4713; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
4714; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4715; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4716; GFX6-NEXT:    s_mov_b32 s2, -1
4717; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4718; GFX6-NEXT:    v_mov_b32_e32 v0, s6
4719; GFX6-NEXT:    v_mov_b32_e32 v1, s7
4720; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4721; GFX6-NEXT:    s_waitcnt vmcnt(0)
4722; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4723; GFX6-NEXT:    s_endpgm
4724;
4725; GFX7-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
4726; GFX7:       ; %bb.0: ; %entry
4727; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4728; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4729; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4730; GFX7-NEXT:    s_add_u32 s4, s0, 16
4731; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4732; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4733; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4734; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4735; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4736; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4737; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4738; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4739; GFX7-NEXT:    s_waitcnt vmcnt(0)
4740; GFX7-NEXT:    flat_store_dword v[0:1], v2
4741; GFX7-NEXT:    s_endpgm
4742;
4743; GFX10-WGP-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
4744; GFX10-WGP:       ; %bb.0: ; %entry
4745; GFX10-WGP-NEXT:    s_clause 0x1
4746; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4747; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4748; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
4749; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4750; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4751; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4752; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4753; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4754; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
4755; GFX10-WGP-NEXT:    s_endpgm
4756;
4757; GFX10-CU-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
4758; GFX10-CU:       ; %bb.0: ; %entry
4759; GFX10-CU-NEXT:    s_clause 0x1
4760; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4761; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4762; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
4763; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4764; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4765; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4766; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4767; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4768; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
4769; GFX10-CU-NEXT:    s_endpgm
4770;
4771; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
4772; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4773; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
4774; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
4775; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
4776; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
4777; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4778; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4779; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4780; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4781; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4782; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4783; SKIP-CACHE-INV-NEXT:    s_endpgm
4784;
4785; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
4786; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4787; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4788; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4789; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4790; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4791; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4792; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4793; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4794; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4795; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4796;
4797; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
4798; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4799; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4800; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4801; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4802; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4803; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4804; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4805; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4806; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4807; GFX90A-TGSPLIT-NEXT:    s_endpgm
4808;
4809; GFX940-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
4810; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4811; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4812; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4813; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4814; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4815; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4816; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4817; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4818; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4819; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4820;
4821; GFX940-TGSPLIT-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
4822; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4823; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4824; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4825; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4826; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4827; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4828; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4829; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4830; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4831; GFX940-TGSPLIT-NEXT:    s_endpgm
4832;
4833; GFX11-WGP-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
4834; GFX11-WGP:       ; %bb.0: ; %entry
4835; GFX11-WGP-NEXT:    s_clause 0x1
4836; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
4837; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
4838; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
4839; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4840; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
4841; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
4842; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
4843; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
4844; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
4845; GFX11-WGP-NEXT:    s_endpgm
4846;
4847; GFX11-CU-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
4848; GFX11-CU:       ; %bb.0: ; %entry
4849; GFX11-CU-NEXT:    s_clause 0x1
4850; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
4851; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
4852; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
4853; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4854; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
4855; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
4856; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
4857; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
4858; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
4859; GFX11-CU-NEXT:    s_endpgm
4860    i32 addrspace(1)* %out, i32 %in, i32 %old) {
4861entry:
4862  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
4863  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst monotonic
4864  %val0 = extractvalue { i32, i1 } %val, 0
4865  store i32 %val0, i32 addrspace(1)* %out, align 4
4866  ret void
4867}
4868
4869define amdgpu_kernel void @global_singlethread_monotonic_acquire_ret_cmpxchg(
4870; GFX6-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
4871; GFX6:       ; %bb.0: ; %entry
4872; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
4873; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4874; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4875; GFX6-NEXT:    s_mov_b32 s2, -1
4876; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4877; GFX6-NEXT:    v_mov_b32_e32 v0, s6
4878; GFX6-NEXT:    v_mov_b32_e32 v1, s7
4879; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4880; GFX6-NEXT:    s_waitcnt vmcnt(0)
4881; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4882; GFX6-NEXT:    s_endpgm
4883;
4884; GFX7-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
4885; GFX7:       ; %bb.0: ; %entry
4886; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4887; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4888; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4889; GFX7-NEXT:    s_add_u32 s4, s0, 16
4890; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4891; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4892; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4893; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4894; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4895; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4896; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4897; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4898; GFX7-NEXT:    s_waitcnt vmcnt(0)
4899; GFX7-NEXT:    flat_store_dword v[0:1], v2
4900; GFX7-NEXT:    s_endpgm
4901;
4902; GFX10-WGP-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
4903; GFX10-WGP:       ; %bb.0: ; %entry
4904; GFX10-WGP-NEXT:    s_clause 0x1
4905; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4906; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4907; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
4908; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4909; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4910; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4911; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4912; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4913; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
4914; GFX10-WGP-NEXT:    s_endpgm
4915;
4916; GFX10-CU-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
4917; GFX10-CU:       ; %bb.0: ; %entry
4918; GFX10-CU-NEXT:    s_clause 0x1
4919; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4920; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4921; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
4922; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4923; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4924; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4925; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4926; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4927; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
4928; GFX10-CU-NEXT:    s_endpgm
4929;
4930; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
4931; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4932; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
4933; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
4934; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
4935; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
4936; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4937; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4938; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4939; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4940; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4941; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4942; SKIP-CACHE-INV-NEXT:    s_endpgm
4943;
4944; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
4945; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4946; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4947; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4948; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4949; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4950; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4951; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4952; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4953; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4954; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4955;
4956; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
4957; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4958; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4959; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4960; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4961; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4962; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4963; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4964; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4965; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
4966; GFX90A-TGSPLIT-NEXT:    s_endpgm
4967;
4968; GFX940-NOTTGSPLIT-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
4969; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4970; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4971; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4972; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4973; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4974; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4975; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4976; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4977; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4978; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4979;
4980; GFX940-TGSPLIT-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
4981; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4982; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
4983; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
4984; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4985; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4986; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4987; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
4988; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4989; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
4990; GFX940-TGSPLIT-NEXT:    s_endpgm
4991;
4992; GFX11-WGP-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
4993; GFX11-WGP:       ; %bb.0: ; %entry
4994; GFX11-WGP-NEXT:    s_clause 0x1
4995; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
4996; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
4997; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
4998; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4999; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
5000; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
5001; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
5002; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
5003; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
5004; GFX11-WGP-NEXT:    s_endpgm
5005;
5006; GFX11-CU-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
5007; GFX11-CU:       ; %bb.0: ; %entry
5008; GFX11-CU-NEXT:    s_clause 0x1
5009; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
5010; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
5011; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
5012; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5013; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
5014; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
5015; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
5016; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
5017; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
5018; GFX11-CU-NEXT:    s_endpgm
5019    i32 addrspace(1)* %out, i32 %in, i32 %old) {
5020entry:
5021  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
5022  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic acquire
5023  %val0 = extractvalue { i32, i1 } %val, 0
5024  store i32 %val0, i32 addrspace(1)* %out, align 4
5025  ret void
5026}
5027
5028define amdgpu_kernel void @global_singlethread_acquire_acquire_ret_cmpxchg(
5029; GFX6-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
5030; GFX6:       ; %bb.0: ; %entry
5031; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
5032; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5033; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5034; GFX6-NEXT:    s_mov_b32 s2, -1
5035; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5036; GFX6-NEXT:    v_mov_b32_e32 v0, s6
5037; GFX6-NEXT:    v_mov_b32_e32 v1, s7
5038; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
5039; GFX6-NEXT:    s_waitcnt vmcnt(0)
5040; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5041; GFX6-NEXT:    s_endpgm
5042;
5043; GFX7-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
5044; GFX7:       ; %bb.0: ; %entry
5045; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5046; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5047; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5048; GFX7-NEXT:    s_add_u32 s4, s0, 16
5049; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5050; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5051; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5052; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5053; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5054; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5055; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5056; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5057; GFX7-NEXT:    s_waitcnt vmcnt(0)
5058; GFX7-NEXT:    flat_store_dword v[0:1], v2
5059; GFX7-NEXT:    s_endpgm
5060;
5061; GFX10-WGP-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
5062; GFX10-WGP:       ; %bb.0: ; %entry
5063; GFX10-WGP-NEXT:    s_clause 0x1
5064; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5065; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5066; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
5067; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5068; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5069; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5070; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5071; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5072; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
5073; GFX10-WGP-NEXT:    s_endpgm
5074;
5075; GFX10-CU-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
5076; GFX10-CU:       ; %bb.0: ; %entry
5077; GFX10-CU-NEXT:    s_clause 0x1
5078; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5079; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5080; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
5081; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5082; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5083; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5084; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5085; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5086; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
5087; GFX10-CU-NEXT:    s_endpgm
5088;
5089; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
5090; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5091; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
5092; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
5093; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
5094; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
5095; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5096; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5097; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5098; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
5099; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5100; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5101; SKIP-CACHE-INV-NEXT:    s_endpgm
5102;
5103; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
5104; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5105; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5106; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5107; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5108; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5109; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5110; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5111; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5112; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
5113; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5114;
5115; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
5116; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5117; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5118; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5119; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5120; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5121; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5122; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5123; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5124; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
5125; GFX90A-TGSPLIT-NEXT:    s_endpgm
5126;
5127; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
5128; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5129; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5130; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
5131; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5132; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5133; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5134; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
5135; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5136; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
5137; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5138;
5139; GFX940-TGSPLIT-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
5140; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5141; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5142; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
5143; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5144; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5145; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5146; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
5147; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5148; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
5149; GFX940-TGSPLIT-NEXT:    s_endpgm
5150;
5151; GFX11-WGP-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
5152; GFX11-WGP:       ; %bb.0: ; %entry
5153; GFX11-WGP-NEXT:    s_clause 0x1
5154; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
5155; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
5156; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
5157; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5158; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
5159; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
5160; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
5161; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
5162; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
5163; GFX11-WGP-NEXT:    s_endpgm
5164;
5165; GFX11-CU-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
5166; GFX11-CU:       ; %bb.0: ; %entry
5167; GFX11-CU-NEXT:    s_clause 0x1
5168; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
5169; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
5170; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
5171; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5172; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
5173; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
5174; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
5175; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
5176; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
5177; GFX11-CU-NEXT:    s_endpgm
5178    i32 addrspace(1)* %out, i32 %in, i32 %old) {
5179entry:
5180  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
5181  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire acquire
5182  %val0 = extractvalue { i32, i1 } %val, 0
5183  store i32 %val0, i32 addrspace(1)* %out, align 4
5184  ret void
5185}
5186
5187define amdgpu_kernel void @global_singlethread_release_acquire_ret_cmpxchg(
5188; GFX6-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
5189; GFX6:       ; %bb.0: ; %entry
5190; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
5191; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5192; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5193; GFX6-NEXT:    s_mov_b32 s2, -1
5194; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5195; GFX6-NEXT:    v_mov_b32_e32 v0, s6
5196; GFX6-NEXT:    v_mov_b32_e32 v1, s7
5197; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
5198; GFX6-NEXT:    s_waitcnt vmcnt(0)
5199; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5200; GFX6-NEXT:    s_endpgm
5201;
5202; GFX7-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
5203; GFX7:       ; %bb.0: ; %entry
5204; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5205; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5206; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5207; GFX7-NEXT:    s_add_u32 s4, s0, 16
5208; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5209; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5210; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5211; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5212; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5213; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5214; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5215; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5216; GFX7-NEXT:    s_waitcnt vmcnt(0)
5217; GFX7-NEXT:    flat_store_dword v[0:1], v2
5218; GFX7-NEXT:    s_endpgm
5219;
5220; GFX10-WGP-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
5221; GFX10-WGP:       ; %bb.0: ; %entry
5222; GFX10-WGP-NEXT:    s_clause 0x1
5223; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5224; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5225; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
5226; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5227; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5228; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5229; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5230; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5231; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
5232; GFX10-WGP-NEXT:    s_endpgm
5233;
5234; GFX10-CU-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
5235; GFX10-CU:       ; %bb.0: ; %entry
5236; GFX10-CU-NEXT:    s_clause 0x1
5237; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5238; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5239; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
5240; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5241; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5242; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5243; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5244; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5245; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
5246; GFX10-CU-NEXT:    s_endpgm
5247;
5248; SKIP-CACHE-INV-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
5249; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5250; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
5251; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
5252; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
5253; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
5254; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5255; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5256; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5257; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
5258; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5259; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5260; SKIP-CACHE-INV-NEXT:    s_endpgm
5261;
5262; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
5263; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5264; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5265; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5266; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5267; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5268; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5269; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5270; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5271; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
5272; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5273;
5274; GFX90A-TGSPLIT-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
5275; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5276; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5277; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5278; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5279; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5280; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5281; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5282; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5283; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
5284; GFX90A-TGSPLIT-NEXT:    s_endpgm
5285;
5286; GFX940-NOTTGSPLIT-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
5287; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5288; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5289; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
5290; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5291; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5292; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5293; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
5294; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5295; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
5296; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5297;
5298; GFX940-TGSPLIT-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
5299; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5300; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5301; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
5302; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5303; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5304; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5305; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
5306; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5307; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
5308; GFX940-TGSPLIT-NEXT:    s_endpgm
5309;
5310; GFX11-WGP-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
5311; GFX11-WGP:       ; %bb.0: ; %entry
5312; GFX11-WGP-NEXT:    s_clause 0x1
5313; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
5314; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
5315; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
5316; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5317; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
5318; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
5319; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
5320; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
5321; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
5322; GFX11-WGP-NEXT:    s_endpgm
5323;
5324; GFX11-CU-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
5325; GFX11-CU:       ; %bb.0: ; %entry
5326; GFX11-CU-NEXT:    s_clause 0x1
5327; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
5328; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
5329; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
5330; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5331; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
5332; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
5333; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
5334; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
5335; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
5336; GFX11-CU-NEXT:    s_endpgm
5337    i32 addrspace(1)* %out, i32 %in, i32 %old) {
5338entry:
5339  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
5340  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") release acquire
5341  %val0 = extractvalue { i32, i1 } %val, 0
5342  store i32 %val0, i32 addrspace(1)* %out, align 4
5343  ret void
5344}
5345
5346define amdgpu_kernel void @global_singlethread_acq_rel_acquire_ret_cmpxchg(
5347; GFX6-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
5348; GFX6:       ; %bb.0: ; %entry
5349; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
5350; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5351; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5352; GFX6-NEXT:    s_mov_b32 s2, -1
5353; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5354; GFX6-NEXT:    v_mov_b32_e32 v0, s6
5355; GFX6-NEXT:    v_mov_b32_e32 v1, s7
5356; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
5357; GFX6-NEXT:    s_waitcnt vmcnt(0)
5358; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5359; GFX6-NEXT:    s_endpgm
5360;
5361; GFX7-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
5362; GFX7:       ; %bb.0: ; %entry
5363; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5364; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5365; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5366; GFX7-NEXT:    s_add_u32 s4, s0, 16
5367; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5368; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5369; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5370; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5371; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5372; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5373; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5374; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5375; GFX7-NEXT:    s_waitcnt vmcnt(0)
5376; GFX7-NEXT:    flat_store_dword v[0:1], v2
5377; GFX7-NEXT:    s_endpgm
5378;
5379; GFX10-WGP-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
5380; GFX10-WGP:       ; %bb.0: ; %entry
5381; GFX10-WGP-NEXT:    s_clause 0x1
5382; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5383; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5384; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
5385; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5386; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5387; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5388; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5389; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5390; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
5391; GFX10-WGP-NEXT:    s_endpgm
5392;
5393; GFX10-CU-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
5394; GFX10-CU:       ; %bb.0: ; %entry
5395; GFX10-CU-NEXT:    s_clause 0x1
5396; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5397; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5398; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
5399; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5400; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5401; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5402; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5403; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5404; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
5405; GFX10-CU-NEXT:    s_endpgm
5406;
5407; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
5408; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5409; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
5410; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
5411; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
5412; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
5413; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5414; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5415; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5416; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
5417; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5418; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5419; SKIP-CACHE-INV-NEXT:    s_endpgm
5420;
5421; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
5422; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5423; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5424; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5425; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5426; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5427; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5428; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5429; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5430; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
5431; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5432;
5433; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
5434; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5435; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5436; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5437; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5438; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5439; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5440; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5441; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5442; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
5443; GFX90A-TGSPLIT-NEXT:    s_endpgm
5444;
5445; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
5446; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5447; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5448; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
5449; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5450; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5451; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5452; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
5453; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5454; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
5455; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5456;
5457; GFX940-TGSPLIT-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
5458; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5459; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5460; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
5461; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5462; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5463; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5464; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
5465; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5466; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
5467; GFX940-TGSPLIT-NEXT:    s_endpgm
5468;
5469; GFX11-WGP-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
5470; GFX11-WGP:       ; %bb.0: ; %entry
5471; GFX11-WGP-NEXT:    s_clause 0x1
5472; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
5473; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
5474; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
5475; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5476; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
5477; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
5478; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
5479; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
5480; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
5481; GFX11-WGP-NEXT:    s_endpgm
5482;
5483; GFX11-CU-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
5484; GFX11-CU:       ; %bb.0: ; %entry
5485; GFX11-CU-NEXT:    s_clause 0x1
5486; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
5487; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
5488; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
5489; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5490; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
5491; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
5492; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
5493; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
5494; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
5495; GFX11-CU-NEXT:    s_endpgm
5496    i32 addrspace(1)* %out, i32 %in, i32 %old) {
5497entry:
5498  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
5499  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel acquire
5500  %val0 = extractvalue { i32, i1 } %val, 0
5501  store i32 %val0, i32 addrspace(1)* %out, align 4
5502  ret void
5503}
5504
5505define amdgpu_kernel void @global_singlethread_seq_cst_acquire_ret_cmpxchg(
5506; GFX6-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
5507; GFX6:       ; %bb.0: ; %entry
5508; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
5509; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5510; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5511; GFX6-NEXT:    s_mov_b32 s2, -1
5512; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5513; GFX6-NEXT:    v_mov_b32_e32 v0, s6
5514; GFX6-NEXT:    v_mov_b32_e32 v1, s7
5515; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
5516; GFX6-NEXT:    s_waitcnt vmcnt(0)
5517; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5518; GFX6-NEXT:    s_endpgm
5519;
5520; GFX7-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
5521; GFX7:       ; %bb.0: ; %entry
5522; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5523; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5524; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5525; GFX7-NEXT:    s_add_u32 s4, s0, 16
5526; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5527; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5528; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5529; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5530; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5531; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5532; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5533; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5534; GFX7-NEXT:    s_waitcnt vmcnt(0)
5535; GFX7-NEXT:    flat_store_dword v[0:1], v2
5536; GFX7-NEXT:    s_endpgm
5537;
5538; GFX10-WGP-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
5539; GFX10-WGP:       ; %bb.0: ; %entry
5540; GFX10-WGP-NEXT:    s_clause 0x1
5541; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5542; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5543; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
5544; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5545; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5546; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5547; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5548; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5549; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
5550; GFX10-WGP-NEXT:    s_endpgm
5551;
5552; GFX10-CU-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
5553; GFX10-CU:       ; %bb.0: ; %entry
5554; GFX10-CU-NEXT:    s_clause 0x1
5555; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5556; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5557; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
5558; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5559; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5560; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5561; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5562; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5563; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
5564; GFX10-CU-NEXT:    s_endpgm
5565;
5566; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
5567; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5568; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
5569; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
5570; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
5571; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
5572; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5573; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5574; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5575; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
5576; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5577; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5578; SKIP-CACHE-INV-NEXT:    s_endpgm
5579;
5580; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
5581; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5582; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5583; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5584; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5585; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5586; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5587; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5588; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5589; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
5590; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5591;
5592; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
5593; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5594; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5595; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5596; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5597; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5598; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5599; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5600; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5601; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
5602; GFX90A-TGSPLIT-NEXT:    s_endpgm
5603;
5604; GFX940-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
5605; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5606; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5607; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
5608; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5609; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5610; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5611; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
5612; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5613; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
5614; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5615;
5616; GFX940-TGSPLIT-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
5617; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5618; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5619; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
5620; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5621; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5622; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5623; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
5624; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5625; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
5626; GFX940-TGSPLIT-NEXT:    s_endpgm
5627;
5628; GFX11-WGP-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
5629; GFX11-WGP:       ; %bb.0: ; %entry
5630; GFX11-WGP-NEXT:    s_clause 0x1
5631; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
5632; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
5633; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
5634; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5635; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
5636; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
5637; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
5638; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
5639; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
5640; GFX11-WGP-NEXT:    s_endpgm
5641;
5642; GFX11-CU-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
5643; GFX11-CU:       ; %bb.0: ; %entry
5644; GFX11-CU-NEXT:    s_clause 0x1
5645; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
5646; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
5647; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
5648; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5649; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
5650; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
5651; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
5652; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
5653; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
5654; GFX11-CU-NEXT:    s_endpgm
5655    i32 addrspace(1)* %out, i32 %in, i32 %old) {
5656entry:
5657  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
5658  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst acquire
5659  %val0 = extractvalue { i32, i1 } %val, 0
5660  store i32 %val0, i32 addrspace(1)* %out, align 4
5661  ret void
5662}
5663
5664define amdgpu_kernel void @global_singlethread_monotonic_seq_cst_ret_cmpxchg(
5665; GFX6-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
5666; GFX6:       ; %bb.0: ; %entry
5667; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
5668; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5669; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5670; GFX6-NEXT:    s_mov_b32 s2, -1
5671; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5672; GFX6-NEXT:    v_mov_b32_e32 v0, s6
5673; GFX6-NEXT:    v_mov_b32_e32 v1, s7
5674; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
5675; GFX6-NEXT:    s_waitcnt vmcnt(0)
5676; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5677; GFX6-NEXT:    s_endpgm
5678;
5679; GFX7-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
5680; GFX7:       ; %bb.0: ; %entry
5681; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5682; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5683; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5684; GFX7-NEXT:    s_add_u32 s4, s0, 16
5685; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5686; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5687; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5688; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5689; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5690; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5691; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5692; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5693; GFX7-NEXT:    s_waitcnt vmcnt(0)
5694; GFX7-NEXT:    flat_store_dword v[0:1], v2
5695; GFX7-NEXT:    s_endpgm
5696;
5697; GFX10-WGP-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
5698; GFX10-WGP:       ; %bb.0: ; %entry
5699; GFX10-WGP-NEXT:    s_clause 0x1
5700; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5701; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5702; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
5703; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5704; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5705; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5706; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5707; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5708; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
5709; GFX10-WGP-NEXT:    s_endpgm
5710;
5711; GFX10-CU-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
5712; GFX10-CU:       ; %bb.0: ; %entry
5713; GFX10-CU-NEXT:    s_clause 0x1
5714; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5715; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5716; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
5717; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5718; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5719; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5720; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5721; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5722; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
5723; GFX10-CU-NEXT:    s_endpgm
5724;
5725; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
5726; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5727; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
5728; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
5729; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
5730; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
5731; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5732; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5733; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5734; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
5735; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5736; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5737; SKIP-CACHE-INV-NEXT:    s_endpgm
5738;
5739; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
5740; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5741; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5742; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5743; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5744; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5745; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5746; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5747; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5748; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
5749; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5750;
5751; GFX90A-TGSPLIT-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
5752; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5753; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5754; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5755; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5756; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5757; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5758; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5759; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5760; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
5761; GFX90A-TGSPLIT-NEXT:    s_endpgm
5762;
5763; GFX940-NOTTGSPLIT-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
5764; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5765; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5766; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
5767; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5768; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5769; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5770; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
5771; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5772; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
5773; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5774;
5775; GFX940-TGSPLIT-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
5776; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5777; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5778; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
5779; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5780; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5781; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5782; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
5783; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5784; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
5785; GFX940-TGSPLIT-NEXT:    s_endpgm
5786;
5787; GFX11-WGP-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
5788; GFX11-WGP:       ; %bb.0: ; %entry
5789; GFX11-WGP-NEXT:    s_clause 0x1
5790; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
5791; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
5792; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
5793; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5794; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
5795; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
5796; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
5797; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
5798; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
5799; GFX11-WGP-NEXT:    s_endpgm
5800;
5801; GFX11-CU-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
5802; GFX11-CU:       ; %bb.0: ; %entry
5803; GFX11-CU-NEXT:    s_clause 0x1
5804; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
5805; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
5806; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
5807; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5808; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
5809; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
5810; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
5811; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
5812; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
5813; GFX11-CU-NEXT:    s_endpgm
5814    i32 addrspace(1)* %out, i32 %in, i32 %old) {
5815entry:
5816  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
5817  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic seq_cst
5818  %val0 = extractvalue { i32, i1 } %val, 0
5819  store i32 %val0, i32 addrspace(1)* %out, align 4
5820  ret void
5821}
5822
5823define amdgpu_kernel void @global_singlethread_acquire_seq_cst_ret_cmpxchg(
5824; GFX6-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
5825; GFX6:       ; %bb.0: ; %entry
5826; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
5827; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5828; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5829; GFX6-NEXT:    s_mov_b32 s2, -1
5830; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5831; GFX6-NEXT:    v_mov_b32_e32 v0, s6
5832; GFX6-NEXT:    v_mov_b32_e32 v1, s7
5833; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
5834; GFX6-NEXT:    s_waitcnt vmcnt(0)
5835; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5836; GFX6-NEXT:    s_endpgm
5837;
5838; GFX7-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
5839; GFX7:       ; %bb.0: ; %entry
5840; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5841; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5842; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5843; GFX7-NEXT:    s_add_u32 s4, s0, 16
5844; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5845; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5846; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5847; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5848; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5849; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5850; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5851; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5852; GFX7-NEXT:    s_waitcnt vmcnt(0)
5853; GFX7-NEXT:    flat_store_dword v[0:1], v2
5854; GFX7-NEXT:    s_endpgm
5855;
5856; GFX10-WGP-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
5857; GFX10-WGP:       ; %bb.0: ; %entry
5858; GFX10-WGP-NEXT:    s_clause 0x1
5859; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5860; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5861; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
5862; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5863; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5864; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5865; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5866; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5867; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
5868; GFX10-WGP-NEXT:    s_endpgm
5869;
5870; GFX10-CU-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
5871; GFX10-CU:       ; %bb.0: ; %entry
5872; GFX10-CU-NEXT:    s_clause 0x1
5873; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5874; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5875; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
5876; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5877; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5878; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5879; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5880; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5881; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
5882; GFX10-CU-NEXT:    s_endpgm
5883;
5884; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
5885; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5886; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
5887; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
5888; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
5889; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
5890; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5891; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5892; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5893; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
5894; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5895; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5896; SKIP-CACHE-INV-NEXT:    s_endpgm
5897;
5898; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
5899; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5900; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5901; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5902; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5903; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5904; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5905; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5906; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5907; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
5908; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5909;
5910; GFX90A-TGSPLIT-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
5911; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5912; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5913; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
5914; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5915; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5916; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5917; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
5918; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5919; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
5920; GFX90A-TGSPLIT-NEXT:    s_endpgm
5921;
5922; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
5923; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5924; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5925; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
5926; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5927; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5928; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5929; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
5930; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5931; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
5932; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5933;
5934; GFX940-TGSPLIT-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
5935; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5936; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5937; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
5938; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
5939; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5940; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5941; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
5942; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5943; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
5944; GFX940-TGSPLIT-NEXT:    s_endpgm
5945;
5946; GFX11-WGP-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
5947; GFX11-WGP:       ; %bb.0: ; %entry
5948; GFX11-WGP-NEXT:    s_clause 0x1
5949; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
5950; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
5951; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
5952; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5953; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
5954; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
5955; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
5956; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
5957; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
5958; GFX11-WGP-NEXT:    s_endpgm
5959;
5960; GFX11-CU-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
5961; GFX11-CU:       ; %bb.0: ; %entry
5962; GFX11-CU-NEXT:    s_clause 0x1
5963; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
5964; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
5965; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
5966; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5967; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
5968; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
5969; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
5970; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
5971; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
5972; GFX11-CU-NEXT:    s_endpgm
5973    i32 addrspace(1)* %out, i32 %in, i32 %old) {
5974entry:
5975  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
5976  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire seq_cst
5977  %val0 = extractvalue { i32, i1 } %val, 0
5978  store i32 %val0, i32 addrspace(1)* %out, align 4
5979  ret void
5980}
5981
5982define amdgpu_kernel void @global_singlethread_release_seq_cst_ret_cmpxchg(
5983; GFX6-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
5984; GFX6:       ; %bb.0: ; %entry
5985; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
5986; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5987; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5988; GFX6-NEXT:    s_mov_b32 s2, -1
5989; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5990; GFX6-NEXT:    v_mov_b32_e32 v0, s6
5991; GFX6-NEXT:    v_mov_b32_e32 v1, s7
5992; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
5993; GFX6-NEXT:    s_waitcnt vmcnt(0)
5994; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5995; GFX6-NEXT:    s_endpgm
5996;
5997; GFX7-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
5998; GFX7:       ; %bb.0: ; %entry
5999; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6000; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6001; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6002; GFX7-NEXT:    s_add_u32 s4, s0, 16
6003; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6004; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6005; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6006; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6007; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6008; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6009; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6010; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6011; GFX7-NEXT:    s_waitcnt vmcnt(0)
6012; GFX7-NEXT:    flat_store_dword v[0:1], v2
6013; GFX7-NEXT:    s_endpgm
6014;
6015; GFX10-WGP-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
6016; GFX10-WGP:       ; %bb.0: ; %entry
6017; GFX10-WGP-NEXT:    s_clause 0x1
6018; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6019; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6020; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
6021; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6022; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6023; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6024; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
6025; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6026; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
6027; GFX10-WGP-NEXT:    s_endpgm
6028;
6029; GFX10-CU-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
6030; GFX10-CU:       ; %bb.0: ; %entry
6031; GFX10-CU-NEXT:    s_clause 0x1
6032; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6033; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6034; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
6035; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6036; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6037; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6038; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
6039; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6040; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
6041; GFX10-CU-NEXT:    s_endpgm
6042;
6043; SKIP-CACHE-INV-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
6044; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6045; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
6046; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6047; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
6048; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
6049; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6050; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6051; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6052; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
6053; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6054; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
6055; SKIP-CACHE-INV-NEXT:    s_endpgm
6056;
6057; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
6058; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6059; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6060; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6061; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6062; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6063; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6064; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
6065; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6066; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
6067; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6068;
6069; GFX90A-TGSPLIT-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
6070; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6071; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6072; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6073; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6074; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6075; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6076; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
6077; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6078; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
6079; GFX90A-TGSPLIT-NEXT:    s_endpgm
6080;
6081; GFX940-NOTTGSPLIT-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
6082; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6083; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
6084; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
6085; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6086; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6087; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6088; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
6089; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6090; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
6091; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6092;
6093; GFX940-TGSPLIT-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
6094; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6095; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
6096; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
6097; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6098; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6099; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6100; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
6101; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6102; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
6103; GFX940-TGSPLIT-NEXT:    s_endpgm
6104;
6105; GFX11-WGP-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
6106; GFX11-WGP:       ; %bb.0: ; %entry
6107; GFX11-WGP-NEXT:    s_clause 0x1
6108; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
6109; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6110; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
6111; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6112; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
6113; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
6114; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
6115; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
6116; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
6117; GFX11-WGP-NEXT:    s_endpgm
6118;
6119; GFX11-CU-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
6120; GFX11-CU:       ; %bb.0: ; %entry
6121; GFX11-CU-NEXT:    s_clause 0x1
6122; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
6123; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6124; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
6125; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6126; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
6127; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
6128; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
6129; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
6130; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
6131; GFX11-CU-NEXT:    s_endpgm
6132    i32 addrspace(1)* %out, i32 %in, i32 %old) {
6133entry:
6134  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
6135  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") release seq_cst
6136  %val0 = extractvalue { i32, i1 } %val, 0
6137  store i32 %val0, i32 addrspace(1)* %out, align 4
6138  ret void
6139}
6140
6141define amdgpu_kernel void @global_singlethread_acq_rel_seq_cst_ret_cmpxchg(
6142; GFX6-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6143; GFX6:       ; %bb.0: ; %entry
6144; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
6145; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6146; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
6147; GFX6-NEXT:    s_mov_b32 s2, -1
6148; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6149; GFX6-NEXT:    v_mov_b32_e32 v0, s6
6150; GFX6-NEXT:    v_mov_b32_e32 v1, s7
6151; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
6152; GFX6-NEXT:    s_waitcnt vmcnt(0)
6153; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
6154; GFX6-NEXT:    s_endpgm
6155;
6156; GFX7-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6157; GFX7:       ; %bb.0: ; %entry
6158; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6159; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6160; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6161; GFX7-NEXT:    s_add_u32 s4, s0, 16
6162; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6163; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6164; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6165; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6166; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6167; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6168; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6169; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6170; GFX7-NEXT:    s_waitcnt vmcnt(0)
6171; GFX7-NEXT:    flat_store_dword v[0:1], v2
6172; GFX7-NEXT:    s_endpgm
6173;
6174; GFX10-WGP-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6175; GFX10-WGP:       ; %bb.0: ; %entry
6176; GFX10-WGP-NEXT:    s_clause 0x1
6177; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6178; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6179; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
6180; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6181; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6182; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6183; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
6184; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6185; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
6186; GFX10-WGP-NEXT:    s_endpgm
6187;
6188; GFX10-CU-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6189; GFX10-CU:       ; %bb.0: ; %entry
6190; GFX10-CU-NEXT:    s_clause 0x1
6191; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6192; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6193; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
6194; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6195; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6196; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6197; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
6198; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6199; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
6200; GFX10-CU-NEXT:    s_endpgm
6201;
6202; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6203; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6204; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
6205; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6206; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
6207; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
6208; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6209; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6210; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6211; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
6212; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6213; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
6214; SKIP-CACHE-INV-NEXT:    s_endpgm
6215;
6216; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6217; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6218; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6219; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6220; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6221; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6222; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6223; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
6224; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6225; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
6226; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6227;
6228; GFX90A-TGSPLIT-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6229; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6230; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6231; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6232; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6233; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6234; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6235; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
6236; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6237; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
6238; GFX90A-TGSPLIT-NEXT:    s_endpgm
6239;
6240; GFX940-NOTTGSPLIT-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6241; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6242; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
6243; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
6244; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6245; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6246; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6247; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
6248; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6249; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
6250; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6251;
6252; GFX940-TGSPLIT-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6253; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6254; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
6255; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
6256; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6257; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6258; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6259; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
6260; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6261; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
6262; GFX940-TGSPLIT-NEXT:    s_endpgm
6263;
6264; GFX11-WGP-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6265; GFX11-WGP:       ; %bb.0: ; %entry
6266; GFX11-WGP-NEXT:    s_clause 0x1
6267; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
6268; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6269; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
6270; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6271; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
6272; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
6273; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
6274; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
6275; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
6276; GFX11-WGP-NEXT:    s_endpgm
6277;
6278; GFX11-CU-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
6279; GFX11-CU:       ; %bb.0: ; %entry
6280; GFX11-CU-NEXT:    s_clause 0x1
6281; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
6282; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6283; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
6284; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6285; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
6286; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
6287; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
6288; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
6289; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
6290; GFX11-CU-NEXT:    s_endpgm
6291    i32 addrspace(1)* %out, i32 %in, i32 %old) {
6292entry:
6293  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
6294  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel seq_cst
6295  %val0 = extractvalue { i32, i1 } %val, 0
6296  store i32 %val0, i32 addrspace(1)* %out, align 4
6297  ret void
6298}
6299
6300define amdgpu_kernel void @global_singlethread_seq_cst_seq_cst_ret_cmpxchg(
6301; GFX6-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6302; GFX6:       ; %bb.0: ; %entry
6303; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
6304; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6305; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
6306; GFX6-NEXT:    s_mov_b32 s2, -1
6307; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6308; GFX6-NEXT:    v_mov_b32_e32 v0, s6
6309; GFX6-NEXT:    v_mov_b32_e32 v1, s7
6310; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
6311; GFX6-NEXT:    s_waitcnt vmcnt(0)
6312; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
6313; GFX6-NEXT:    s_endpgm
6314;
6315; GFX7-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6316; GFX7:       ; %bb.0: ; %entry
6317; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6318; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6319; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6320; GFX7-NEXT:    s_add_u32 s4, s0, 16
6321; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6322; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6323; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6324; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6325; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6326; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6327; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6328; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6329; GFX7-NEXT:    s_waitcnt vmcnt(0)
6330; GFX7-NEXT:    flat_store_dword v[0:1], v2
6331; GFX7-NEXT:    s_endpgm
6332;
6333; GFX10-WGP-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6334; GFX10-WGP:       ; %bb.0: ; %entry
6335; GFX10-WGP-NEXT:    s_clause 0x1
6336; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6337; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6338; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
6339; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6340; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6341; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6342; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
6343; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6344; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
6345; GFX10-WGP-NEXT:    s_endpgm
6346;
6347; GFX10-CU-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6348; GFX10-CU:       ; %bb.0: ; %entry
6349; GFX10-CU-NEXT:    s_clause 0x1
6350; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6351; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6352; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
6353; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6354; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6355; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6356; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
6357; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6358; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
6359; GFX10-CU-NEXT:    s_endpgm
6360;
6361; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6362; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6363; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
6364; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
6365; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
6366; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
6367; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6368; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6369; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6370; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
6371; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6372; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
6373; SKIP-CACHE-INV-NEXT:    s_endpgm
6374;
6375; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6376; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6377; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6378; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6379; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6380; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6381; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6382; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
6383; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6384; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
6385; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6386;
6387; GFX90A-TGSPLIT-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6388; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6389; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6390; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6391; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6392; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6393; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6394; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
6395; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6396; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
6397; GFX90A-TGSPLIT-NEXT:    s_endpgm
6398;
6399; GFX940-NOTTGSPLIT-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6400; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6401; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
6402; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
6403; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6404; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6405; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6406; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
6407; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6408; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
6409; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6410;
6411; GFX940-TGSPLIT-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6412; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6413; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
6414; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
6415; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6416; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6417; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6418; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
6419; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6420; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
6421; GFX940-TGSPLIT-NEXT:    s_endpgm
6422;
6423; GFX11-WGP-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6424; GFX11-WGP:       ; %bb.0: ; %entry
6425; GFX11-WGP-NEXT:    s_clause 0x1
6426; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
6427; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6428; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
6429; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6430; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
6431; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
6432; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
6433; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
6434; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
6435; GFX11-WGP-NEXT:    s_endpgm
6436;
6437; GFX11-CU-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
6438; GFX11-CU:       ; %bb.0: ; %entry
6439; GFX11-CU-NEXT:    s_clause 0x1
6440; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
6441; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
6442; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
6443; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6444; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
6445; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
6446; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
6447; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
6448; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
6449; GFX11-CU-NEXT:    s_endpgm
6450    i32 addrspace(1)* %out, i32 %in, i32 %old) {
6451entry:
6452  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
6453  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst seq_cst
6454  %val0 = extractvalue { i32, i1 } %val, 0
6455  store i32 %val0, i32 addrspace(1)* %out, align 4
6456  ret void
6457}
6458
6459define amdgpu_kernel void @global_singlethread_one_as_unordered_load(
6460; GFX6-LABEL: global_singlethread_one_as_unordered_load:
6461; GFX6:       ; %bb.0: ; %entry
6462; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6463; GFX6-NEXT:    s_mov_b32 s7, 0x100f000
6464; GFX6-NEXT:    s_mov_b32 s6, -1
6465; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6466; GFX6-NEXT:    s_mov_b32 s4, s0
6467; GFX6-NEXT:    s_mov_b32 s5, s1
6468; GFX6-NEXT:    buffer_load_dword v0, off, s[4:7], 0
6469; GFX6-NEXT:    s_mov_b32 s4, s2
6470; GFX6-NEXT:    s_mov_b32 s5, s3
6471; GFX6-NEXT:    s_waitcnt vmcnt(0)
6472; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
6473; GFX6-NEXT:    s_endpgm
6474;
6475; GFX7-LABEL: global_singlethread_one_as_unordered_load:
6476; GFX7:       ; %bb.0: ; %entry
6477; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6478; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6479; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6480; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6481; GFX7-NEXT:    flat_load_dword v2, v[0:1]
6482; GFX7-NEXT:    v_mov_b32_e32 v0, s2
6483; GFX7-NEXT:    v_mov_b32_e32 v1, s3
6484; GFX7-NEXT:    s_waitcnt vmcnt(0)
6485; GFX7-NEXT:    flat_store_dword v[0:1], v2
6486; GFX7-NEXT:    s_endpgm
6487;
6488; GFX10-WGP-LABEL: global_singlethread_one_as_unordered_load:
6489; GFX10-WGP:       ; %bb.0: ; %entry
6490; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6491; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
6492; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6493; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
6494; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6495; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
6496; GFX10-WGP-NEXT:    s_endpgm
6497;
6498; GFX10-CU-LABEL: global_singlethread_one_as_unordered_load:
6499; GFX10-CU:       ; %bb.0: ; %entry
6500; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6501; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
6502; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6503; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
6504; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6505; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
6506; GFX10-CU-NEXT:    s_endpgm
6507;
6508; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_unordered_load:
6509; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6510; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6511; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
6512; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
6513; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6514; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s0
6515; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s1
6516; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[4:7], 0
6517; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s2
6518; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s3
6519; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6520; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
6521; SKIP-CACHE-INV-NEXT:    s_endpgm
6522;
6523; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_unordered_load:
6524; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6525; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6526; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6527; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6528; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
6529; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6530; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6531; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6532;
6533; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_unordered_load:
6534; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6535; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6536; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6537; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6538; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
6539; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6540; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6541; GFX90A-TGSPLIT-NEXT:    s_endpgm
6542;
6543; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_unordered_load:
6544; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6545; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6546; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6547; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6548; GFX940-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
6549; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6550; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6551; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6552;
6553; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_unordered_load:
6554; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6555; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6556; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6557; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6558; GFX940-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
6559; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6560; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6561; GFX940-TGSPLIT-NEXT:    s_endpgm
6562;
6563; GFX11-WGP-LABEL: global_singlethread_one_as_unordered_load:
6564; GFX11-WGP:       ; %bb.0: ; %entry
6565; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6566; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
6567; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6568; GFX11-WGP-NEXT:    global_load_b32 v1, v0, s[0:1]
6569; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
6570; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[2:3]
6571; GFX11-WGP-NEXT:    s_endpgm
6572;
6573; GFX11-CU-LABEL: global_singlethread_one_as_unordered_load:
6574; GFX11-CU:       ; %bb.0: ; %entry
6575; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6576; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
6577; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6578; GFX11-CU-NEXT:    global_load_b32 v1, v0, s[0:1]
6579; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
6580; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[2:3]
6581; GFX11-CU-NEXT:    s_endpgm
6582    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
6583entry:
6584  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread-one-as") unordered, align 4
6585  store i32 %val, i32 addrspace(1)* %out
6586  ret void
6587}
6588
6589define amdgpu_kernel void @global_singlethread_one_as_monotonic_load(
6590; GFX6-LABEL: global_singlethread_one_as_monotonic_load:
6591; GFX6:       ; %bb.0: ; %entry
6592; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6593; GFX6-NEXT:    s_mov_b32 s7, 0x100f000
6594; GFX6-NEXT:    s_mov_b32 s6, -1
6595; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6596; GFX6-NEXT:    s_mov_b32 s4, s0
6597; GFX6-NEXT:    s_mov_b32 s5, s1
6598; GFX6-NEXT:    buffer_load_dword v0, off, s[4:7], 0
6599; GFX6-NEXT:    s_mov_b32 s4, s2
6600; GFX6-NEXT:    s_mov_b32 s5, s3
6601; GFX6-NEXT:    s_waitcnt vmcnt(0)
6602; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
6603; GFX6-NEXT:    s_endpgm
6604;
6605; GFX7-LABEL: global_singlethread_one_as_monotonic_load:
6606; GFX7:       ; %bb.0: ; %entry
6607; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6608; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6609; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6610; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6611; GFX7-NEXT:    flat_load_dword v2, v[0:1]
6612; GFX7-NEXT:    v_mov_b32_e32 v0, s2
6613; GFX7-NEXT:    v_mov_b32_e32 v1, s3
6614; GFX7-NEXT:    s_waitcnt vmcnt(0)
6615; GFX7-NEXT:    flat_store_dword v[0:1], v2
6616; GFX7-NEXT:    s_endpgm
6617;
6618; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_load:
6619; GFX10-WGP:       ; %bb.0: ; %entry
6620; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6621; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
6622; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6623; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
6624; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6625; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
6626; GFX10-WGP-NEXT:    s_endpgm
6627;
6628; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_load:
6629; GFX10-CU:       ; %bb.0: ; %entry
6630; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6631; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
6632; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6633; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
6634; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6635; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
6636; GFX10-CU-NEXT:    s_endpgm
6637;
6638; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_load:
6639; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6640; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6641; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
6642; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
6643; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6644; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s0
6645; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s1
6646; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[4:7], 0
6647; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s2
6648; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s3
6649; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6650; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
6651; SKIP-CACHE-INV-NEXT:    s_endpgm
6652;
6653; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_load:
6654; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6655; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6656; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6657; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6658; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
6659; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6660; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6661; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6662;
6663; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_load:
6664; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6665; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6666; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6667; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6668; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
6669; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6670; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6671; GFX90A-TGSPLIT-NEXT:    s_endpgm
6672;
6673; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_load:
6674; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6675; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6676; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6677; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6678; GFX940-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
6679; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6680; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6681; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6682;
6683; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_load:
6684; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6685; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6686; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6687; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6688; GFX940-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
6689; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6690; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6691; GFX940-TGSPLIT-NEXT:    s_endpgm
6692;
6693; GFX11-WGP-LABEL: global_singlethread_one_as_monotonic_load:
6694; GFX11-WGP:       ; %bb.0: ; %entry
6695; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6696; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
6697; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6698; GFX11-WGP-NEXT:    global_load_b32 v1, v0, s[0:1]
6699; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
6700; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[2:3]
6701; GFX11-WGP-NEXT:    s_endpgm
6702;
6703; GFX11-CU-LABEL: global_singlethread_one_as_monotonic_load:
6704; GFX11-CU:       ; %bb.0: ; %entry
6705; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6706; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
6707; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6708; GFX11-CU-NEXT:    global_load_b32 v1, v0, s[0:1]
6709; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
6710; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[2:3]
6711; GFX11-CU-NEXT:    s_endpgm
6712    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
6713entry:
6714  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread-one-as") monotonic, align 4
6715  store i32 %val, i32 addrspace(1)* %out
6716  ret void
6717}
6718
6719define amdgpu_kernel void @global_singlethread_one_as_acquire_load(
6720; GFX6-LABEL: global_singlethread_one_as_acquire_load:
6721; GFX6:       ; %bb.0: ; %entry
6722; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6723; GFX6-NEXT:    s_mov_b32 s7, 0x100f000
6724; GFX6-NEXT:    s_mov_b32 s6, -1
6725; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6726; GFX6-NEXT:    s_mov_b32 s4, s0
6727; GFX6-NEXT:    s_mov_b32 s5, s1
6728; GFX6-NEXT:    buffer_load_dword v0, off, s[4:7], 0
6729; GFX6-NEXT:    s_mov_b32 s4, s2
6730; GFX6-NEXT:    s_mov_b32 s5, s3
6731; GFX6-NEXT:    s_waitcnt vmcnt(0)
6732; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
6733; GFX6-NEXT:    s_endpgm
6734;
6735; GFX7-LABEL: global_singlethread_one_as_acquire_load:
6736; GFX7:       ; %bb.0: ; %entry
6737; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6738; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6739; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6740; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6741; GFX7-NEXT:    flat_load_dword v2, v[0:1]
6742; GFX7-NEXT:    v_mov_b32_e32 v0, s2
6743; GFX7-NEXT:    v_mov_b32_e32 v1, s3
6744; GFX7-NEXT:    s_waitcnt vmcnt(0)
6745; GFX7-NEXT:    flat_store_dword v[0:1], v2
6746; GFX7-NEXT:    s_endpgm
6747;
6748; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_load:
6749; GFX10-WGP:       ; %bb.0: ; %entry
6750; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6751; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
6752; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6753; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
6754; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6755; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
6756; GFX10-WGP-NEXT:    s_endpgm
6757;
6758; GFX10-CU-LABEL: global_singlethread_one_as_acquire_load:
6759; GFX10-CU:       ; %bb.0: ; %entry
6760; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6761; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
6762; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6763; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
6764; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6765; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
6766; GFX10-CU-NEXT:    s_endpgm
6767;
6768; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_load:
6769; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6770; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6771; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
6772; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
6773; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6774; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s0
6775; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s1
6776; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[4:7], 0
6777; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s2
6778; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s3
6779; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6780; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
6781; SKIP-CACHE-INV-NEXT:    s_endpgm
6782;
6783; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_load:
6784; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6785; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6786; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6787; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6788; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
6789; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6790; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6791; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6792;
6793; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_load:
6794; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6795; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6796; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6797; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6798; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
6799; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6800; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6801; GFX90A-TGSPLIT-NEXT:    s_endpgm
6802;
6803; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_load:
6804; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6805; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6806; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6807; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6808; GFX940-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
6809; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6810; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6811; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6812;
6813; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acquire_load:
6814; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6815; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6816; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6817; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6818; GFX940-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
6819; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6820; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6821; GFX940-TGSPLIT-NEXT:    s_endpgm
6822;
6823; GFX11-WGP-LABEL: global_singlethread_one_as_acquire_load:
6824; GFX11-WGP:       ; %bb.0: ; %entry
6825; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6826; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
6827; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6828; GFX11-WGP-NEXT:    global_load_b32 v1, v0, s[0:1]
6829; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
6830; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[2:3]
6831; GFX11-WGP-NEXT:    s_endpgm
6832;
6833; GFX11-CU-LABEL: global_singlethread_one_as_acquire_load:
6834; GFX11-CU:       ; %bb.0: ; %entry
6835; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6836; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
6837; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6838; GFX11-CU-NEXT:    global_load_b32 v1, v0, s[0:1]
6839; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
6840; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[2:3]
6841; GFX11-CU-NEXT:    s_endpgm
6842    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
6843entry:
6844  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread-one-as") acquire, align 4
6845  store i32 %val, i32 addrspace(1)* %out
6846  ret void
6847}
6848
6849define amdgpu_kernel void @global_singlethread_one_as_seq_cst_load(
6850; GFX6-LABEL: global_singlethread_one_as_seq_cst_load:
6851; GFX6:       ; %bb.0: ; %entry
6852; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6853; GFX6-NEXT:    s_mov_b32 s7, 0x100f000
6854; GFX6-NEXT:    s_mov_b32 s6, -1
6855; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6856; GFX6-NEXT:    s_mov_b32 s4, s0
6857; GFX6-NEXT:    s_mov_b32 s5, s1
6858; GFX6-NEXT:    buffer_load_dword v0, off, s[4:7], 0
6859; GFX6-NEXT:    s_mov_b32 s4, s2
6860; GFX6-NEXT:    s_mov_b32 s5, s3
6861; GFX6-NEXT:    s_waitcnt vmcnt(0)
6862; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
6863; GFX6-NEXT:    s_endpgm
6864;
6865; GFX7-LABEL: global_singlethread_one_as_seq_cst_load:
6866; GFX7:       ; %bb.0: ; %entry
6867; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6868; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6869; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6870; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6871; GFX7-NEXT:    flat_load_dword v2, v[0:1]
6872; GFX7-NEXT:    v_mov_b32_e32 v0, s2
6873; GFX7-NEXT:    v_mov_b32_e32 v1, s3
6874; GFX7-NEXT:    s_waitcnt vmcnt(0)
6875; GFX7-NEXT:    flat_store_dword v[0:1], v2
6876; GFX7-NEXT:    s_endpgm
6877;
6878; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_load:
6879; GFX10-WGP:       ; %bb.0: ; %entry
6880; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6881; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
6882; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6883; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
6884; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6885; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
6886; GFX10-WGP-NEXT:    s_endpgm
6887;
6888; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_load:
6889; GFX10-CU:       ; %bb.0: ; %entry
6890; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6891; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
6892; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6893; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
6894; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6895; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
6896; GFX10-CU-NEXT:    s_endpgm
6897;
6898; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_load:
6899; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6900; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6901; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
6902; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
6903; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6904; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s0
6905; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s1
6906; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[4:7], 0
6907; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s2
6908; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s3
6909; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6910; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
6911; SKIP-CACHE-INV-NEXT:    s_endpgm
6912;
6913; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_load:
6914; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6915; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6916; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6917; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6918; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
6919; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6920; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6921; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6922;
6923; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_load:
6924; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6925; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6926; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6927; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6928; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
6929; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6930; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6931; GFX90A-TGSPLIT-NEXT:    s_endpgm
6932;
6933; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_load:
6934; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6935; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6936; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6937; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6938; GFX940-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
6939; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6940; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6941; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6942;
6943; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_load:
6944; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6945; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6946; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6947; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6948; GFX940-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
6949; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6950; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
6951; GFX940-TGSPLIT-NEXT:    s_endpgm
6952;
6953; GFX11-WGP-LABEL: global_singlethread_one_as_seq_cst_load:
6954; GFX11-WGP:       ; %bb.0: ; %entry
6955; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6956; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
6957; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6958; GFX11-WGP-NEXT:    global_load_b32 v1, v0, s[0:1]
6959; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
6960; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[2:3]
6961; GFX11-WGP-NEXT:    s_endpgm
6962;
6963; GFX11-CU-LABEL: global_singlethread_one_as_seq_cst_load:
6964; GFX11-CU:       ; %bb.0: ; %entry
6965; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
6966; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
6967; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6968; GFX11-CU-NEXT:    global_load_b32 v1, v0, s[0:1]
6969; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
6970; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[2:3]
6971; GFX11-CU-NEXT:    s_endpgm
6972    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
6973entry:
6974  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread-one-as") seq_cst, align 4
6975  store i32 %val, i32 addrspace(1)* %out
6976  ret void
6977}
6978
6979define amdgpu_kernel void @global_singlethread_one_as_unordered_store(
6980; GFX6-LABEL: global_singlethread_one_as_unordered_store:
6981; GFX6:       ; %bb.0: ; %entry
6982; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
6983; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
6984; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
6985; GFX6-NEXT:    s_mov_b32 s2, -1
6986; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6987; GFX6-NEXT:    v_mov_b32_e32 v0, s6
6988; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
6989; GFX6-NEXT:    s_endpgm
6990;
6991; GFX7-LABEL: global_singlethread_one_as_unordered_store:
6992; GFX7:       ; %bb.0: ; %entry
6993; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
6994; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
6995; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6996; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6997; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6998; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6999; GFX7-NEXT:    flat_store_dword v[0:1], v2
7000; GFX7-NEXT:    s_endpgm
7001;
7002; GFX10-WGP-LABEL: global_singlethread_one_as_unordered_store:
7003; GFX10-WGP:       ; %bb.0: ; %entry
7004; GFX10-WGP-NEXT:    s_clause 0x1
7005; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
7006; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7007; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
7008; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7009; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
7010; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
7011; GFX10-WGP-NEXT:    s_endpgm
7012;
7013; GFX10-CU-LABEL: global_singlethread_one_as_unordered_store:
7014; GFX10-CU:       ; %bb.0: ; %entry
7015; GFX10-CU-NEXT:    s_clause 0x1
7016; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
7017; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7018; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
7019; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7020; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
7021; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
7022; GFX10-CU-NEXT:    s_endpgm
7023;
7024; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_unordered_store:
7025; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7026; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x0
7027; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7028; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
7029; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
7030; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7031; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7032; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
7033; SKIP-CACHE-INV-NEXT:    s_endpgm
7034;
7035; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_unordered_store:
7036; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7037; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7038; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7039; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7040; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7041; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7042; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
7043; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7044;
7045; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_unordered_store:
7046; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7047; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7048; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7049; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7050; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7051; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7052; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
7053; GFX90A-TGSPLIT-NEXT:    s_endpgm
7054;
7055; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_unordered_store:
7056; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7057; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7058; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7059; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7060; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7061; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
7062; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
7063; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7064;
7065; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_unordered_store:
7066; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7067; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7068; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7069; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7070; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7071; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
7072; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
7073; GFX940-TGSPLIT-NEXT:    s_endpgm
7074;
7075; GFX11-WGP-LABEL: global_singlethread_one_as_unordered_store:
7076; GFX11-WGP:       ; %bb.0: ; %entry
7077; GFX11-WGP-NEXT:    s_clause 0x1
7078; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x0
7079; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7080; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
7081; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7082; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s2
7083; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[0:1]
7084; GFX11-WGP-NEXT:    s_endpgm
7085;
7086; GFX11-CU-LABEL: global_singlethread_one_as_unordered_store:
7087; GFX11-CU:       ; %bb.0: ; %entry
7088; GFX11-CU-NEXT:    s_clause 0x1
7089; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x0
7090; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7091; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
7092; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7093; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s2
7094; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
7095; GFX11-CU-NEXT:    s_endpgm
7096    i32 %in, i32 addrspace(1)* %out) {
7097entry:
7098  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread-one-as") unordered, align 4
7099  ret void
7100}
7101
7102define amdgpu_kernel void @global_singlethread_one_as_monotonic_store(
7103; GFX6-LABEL: global_singlethread_one_as_monotonic_store:
7104; GFX6:       ; %bb.0: ; %entry
7105; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
7106; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
7107; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7108; GFX6-NEXT:    s_mov_b32 s2, -1
7109; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7110; GFX6-NEXT:    v_mov_b32_e32 v0, s6
7111; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
7112; GFX6-NEXT:    s_endpgm
7113;
7114; GFX7-LABEL: global_singlethread_one_as_monotonic_store:
7115; GFX7:       ; %bb.0: ; %entry
7116; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
7117; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
7118; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7119; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7120; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7121; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7122; GFX7-NEXT:    flat_store_dword v[0:1], v2
7123; GFX7-NEXT:    s_endpgm
7124;
7125; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_store:
7126; GFX10-WGP:       ; %bb.0: ; %entry
7127; GFX10-WGP-NEXT:    s_clause 0x1
7128; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
7129; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7130; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
7131; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7132; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
7133; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
7134; GFX10-WGP-NEXT:    s_endpgm
7135;
7136; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_store:
7137; GFX10-CU:       ; %bb.0: ; %entry
7138; GFX10-CU-NEXT:    s_clause 0x1
7139; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
7140; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7141; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
7142; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7143; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
7144; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
7145; GFX10-CU-NEXT:    s_endpgm
7146;
7147; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_store:
7148; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7149; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x0
7150; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7151; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
7152; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
7153; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7154; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7155; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
7156; SKIP-CACHE-INV-NEXT:    s_endpgm
7157;
7158; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_store:
7159; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7160; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7161; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7162; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7163; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7164; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7165; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
7166; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7167;
7168; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_store:
7169; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7170; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7171; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7172; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7173; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7174; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7175; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
7176; GFX90A-TGSPLIT-NEXT:    s_endpgm
7177;
7178; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_store:
7179; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7180; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7181; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7182; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7183; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7184; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
7185; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
7186; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7187;
7188; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_store:
7189; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7190; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7191; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7192; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7193; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7194; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
7195; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
7196; GFX940-TGSPLIT-NEXT:    s_endpgm
7197;
7198; GFX11-WGP-LABEL: global_singlethread_one_as_monotonic_store:
7199; GFX11-WGP:       ; %bb.0: ; %entry
7200; GFX11-WGP-NEXT:    s_clause 0x1
7201; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x0
7202; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7203; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
7204; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7205; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s2
7206; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[0:1]
7207; GFX11-WGP-NEXT:    s_endpgm
7208;
7209; GFX11-CU-LABEL: global_singlethread_one_as_monotonic_store:
7210; GFX11-CU:       ; %bb.0: ; %entry
7211; GFX11-CU-NEXT:    s_clause 0x1
7212; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x0
7213; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7214; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
7215; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7216; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s2
7217; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
7218; GFX11-CU-NEXT:    s_endpgm
7219    i32 %in, i32 addrspace(1)* %out) {
7220entry:
7221  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread-one-as") monotonic, align 4
7222  ret void
7223}
7224
7225define amdgpu_kernel void @global_singlethread_one_as_release_store(
7226; GFX6-LABEL: global_singlethread_one_as_release_store:
7227; GFX6:       ; %bb.0: ; %entry
7228; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
7229; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
7230; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7231; GFX6-NEXT:    s_mov_b32 s2, -1
7232; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7233; GFX6-NEXT:    v_mov_b32_e32 v0, s6
7234; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
7235; GFX6-NEXT:    s_endpgm
7236;
7237; GFX7-LABEL: global_singlethread_one_as_release_store:
7238; GFX7:       ; %bb.0: ; %entry
7239; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
7240; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
7241; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7242; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7243; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7244; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7245; GFX7-NEXT:    flat_store_dword v[0:1], v2
7246; GFX7-NEXT:    s_endpgm
7247;
7248; GFX10-WGP-LABEL: global_singlethread_one_as_release_store:
7249; GFX10-WGP:       ; %bb.0: ; %entry
7250; GFX10-WGP-NEXT:    s_clause 0x1
7251; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
7252; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7253; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
7254; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7255; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
7256; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
7257; GFX10-WGP-NEXT:    s_endpgm
7258;
7259; GFX10-CU-LABEL: global_singlethread_one_as_release_store:
7260; GFX10-CU:       ; %bb.0: ; %entry
7261; GFX10-CU-NEXT:    s_clause 0x1
7262; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
7263; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7264; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
7265; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7266; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
7267; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
7268; GFX10-CU-NEXT:    s_endpgm
7269;
7270; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_store:
7271; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7272; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x0
7273; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7274; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
7275; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
7276; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7277; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7278; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
7279; SKIP-CACHE-INV-NEXT:    s_endpgm
7280;
7281; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_store:
7282; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7283; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7284; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7285; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7286; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7287; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7288; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
7289; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7290;
7291; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_release_store:
7292; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7293; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7294; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7295; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7296; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7297; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7298; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
7299; GFX90A-TGSPLIT-NEXT:    s_endpgm
7300;
7301; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_store:
7302; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7303; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7304; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7305; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7306; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7307; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
7308; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
7309; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7310;
7311; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_release_store:
7312; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7313; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7314; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7315; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7316; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7317; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
7318; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
7319; GFX940-TGSPLIT-NEXT:    s_endpgm
7320;
7321; GFX11-WGP-LABEL: global_singlethread_one_as_release_store:
7322; GFX11-WGP:       ; %bb.0: ; %entry
7323; GFX11-WGP-NEXT:    s_clause 0x1
7324; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x0
7325; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7326; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
7327; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7328; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s2
7329; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[0:1]
7330; GFX11-WGP-NEXT:    s_endpgm
7331;
7332; GFX11-CU-LABEL: global_singlethread_one_as_release_store:
7333; GFX11-CU:       ; %bb.0: ; %entry
7334; GFX11-CU-NEXT:    s_clause 0x1
7335; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x0
7336; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7337; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
7338; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7339; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s2
7340; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
7341; GFX11-CU-NEXT:    s_endpgm
7342    i32 %in, i32 addrspace(1)* %out) {
7343entry:
7344  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread-one-as") release, align 4
7345  ret void
7346}
7347
7348define amdgpu_kernel void @global_singlethread_one_as_seq_cst_store(
7349; GFX6-LABEL: global_singlethread_one_as_seq_cst_store:
7350; GFX6:       ; %bb.0: ; %entry
7351; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
7352; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
7353; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7354; GFX6-NEXT:    s_mov_b32 s2, -1
7355; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7356; GFX6-NEXT:    v_mov_b32_e32 v0, s6
7357; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
7358; GFX6-NEXT:    s_endpgm
7359;
7360; GFX7-LABEL: global_singlethread_one_as_seq_cst_store:
7361; GFX7:       ; %bb.0: ; %entry
7362; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
7363; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
7364; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7365; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7366; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7367; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7368; GFX7-NEXT:    flat_store_dword v[0:1], v2
7369; GFX7-NEXT:    s_endpgm
7370;
7371; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_store:
7372; GFX10-WGP:       ; %bb.0: ; %entry
7373; GFX10-WGP-NEXT:    s_clause 0x1
7374; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
7375; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7376; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
7377; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7378; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
7379; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
7380; GFX10-WGP-NEXT:    s_endpgm
7381;
7382; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_store:
7383; GFX10-CU:       ; %bb.0: ; %entry
7384; GFX10-CU-NEXT:    s_clause 0x1
7385; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
7386; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7387; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
7388; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7389; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
7390; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
7391; GFX10-CU-NEXT:    s_endpgm
7392;
7393; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_store:
7394; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7395; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x0
7396; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7397; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
7398; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
7399; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7400; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7401; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
7402; SKIP-CACHE-INV-NEXT:    s_endpgm
7403;
7404; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_store:
7405; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7406; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7407; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7408; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7409; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7410; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7411; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
7412; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7413;
7414; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_store:
7415; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7416; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
7417; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7418; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7419; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7420; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7421; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
7422; GFX90A-TGSPLIT-NEXT:    s_endpgm
7423;
7424; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_store:
7425; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7426; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7427; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7428; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7429; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7430; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
7431; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
7432; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7433;
7434; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_store:
7435; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7436; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
7437; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
7438; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7439; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7440; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
7441; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
7442; GFX940-TGSPLIT-NEXT:    s_endpgm
7443;
7444; GFX11-WGP-LABEL: global_singlethread_one_as_seq_cst_store:
7445; GFX11-WGP:       ; %bb.0: ; %entry
7446; GFX11-WGP-NEXT:    s_clause 0x1
7447; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x0
7448; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7449; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
7450; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7451; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s2
7452; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[0:1]
7453; GFX11-WGP-NEXT:    s_endpgm
7454;
7455; GFX11-CU-LABEL: global_singlethread_one_as_seq_cst_store:
7456; GFX11-CU:       ; %bb.0: ; %entry
7457; GFX11-CU-NEXT:    s_clause 0x1
7458; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x0
7459; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7460; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
7461; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7462; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s2
7463; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
7464; GFX11-CU-NEXT:    s_endpgm
7465    i32 %in, i32 addrspace(1)* %out) {
7466entry:
7467  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread-one-as") seq_cst, align 4
7468  ret void
7469}
7470
7471define amdgpu_kernel void @global_singlethread_one_as_monotonic_atomicrmw(
7472; GFX6-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
7473; GFX6:       ; %bb.0: ; %entry
7474; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
7475; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7476; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7477; GFX6-NEXT:    s_mov_b32 s2, -1
7478; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7479; GFX6-NEXT:    v_mov_b32_e32 v0, s6
7480; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
7481; GFX6-NEXT:    s_endpgm
7482;
7483; GFX7-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
7484; GFX7:       ; %bb.0: ; %entry
7485; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7486; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7487; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7488; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7489; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7490; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7491; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
7492; GFX7-NEXT:    s_endpgm
7493;
7494; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
7495; GFX10-WGP:       ; %bb.0: ; %entry
7496; GFX10-WGP-NEXT:    s_clause 0x1
7497; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7498; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7499; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
7500; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7501; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
7502; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
7503; GFX10-WGP-NEXT:    s_endpgm
7504;
7505; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
7506; GFX10-CU:       ; %bb.0: ; %entry
7507; GFX10-CU-NEXT:    s_clause 0x1
7508; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7509; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7510; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
7511; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7512; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
7513; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
7514; GFX10-CU-NEXT:    s_endpgm
7515;
7516; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
7517; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7518; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
7519; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7520; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
7521; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
7522; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7523; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7524; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
7525; SKIP-CACHE-INV-NEXT:    s_endpgm
7526;
7527; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
7528; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7529; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7530; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7531; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7532; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7533; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7534; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
7535; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7536;
7537; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
7538; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7539; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7540; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7541; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7542; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7543; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7544; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
7545; GFX90A-TGSPLIT-NEXT:    s_endpgm
7546;
7547; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
7548; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7549; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7550; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7551; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7552; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7553; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
7554; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
7555; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7556;
7557; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
7558; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7559; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7560; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7561; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7562; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7563; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
7564; GFX940-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
7565; GFX940-TGSPLIT-NEXT:    s_endpgm
7566;
7567; GFX11-WGP-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
7568; GFX11-WGP:       ; %bb.0: ; %entry
7569; GFX11-WGP-NEXT:    s_clause 0x1
7570; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x8
7571; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7572; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
7573; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7574; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s2
7575; GFX11-WGP-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
7576; GFX11-WGP-NEXT:    s_endpgm
7577;
7578; GFX11-CU-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
7579; GFX11-CU:       ; %bb.0: ; %entry
7580; GFX11-CU-NEXT:    s_clause 0x1
7581; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x8
7582; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7583; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
7584; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7585; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s2
7586; GFX11-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
7587; GFX11-CU-NEXT:    s_endpgm
7588    i32 addrspace(1)* %out, i32 %in) {
7589entry:
7590  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") monotonic
7591  ret void
7592}
7593
7594define amdgpu_kernel void @global_singlethread_one_as_acquire_atomicrmw(
7595; GFX6-LABEL: global_singlethread_one_as_acquire_atomicrmw:
7596; GFX6:       ; %bb.0: ; %entry
7597; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
7598; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7599; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7600; GFX6-NEXT:    s_mov_b32 s2, -1
7601; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7602; GFX6-NEXT:    v_mov_b32_e32 v0, s6
7603; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
7604; GFX6-NEXT:    s_endpgm
7605;
7606; GFX7-LABEL: global_singlethread_one_as_acquire_atomicrmw:
7607; GFX7:       ; %bb.0: ; %entry
7608; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7609; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7610; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7611; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7612; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7613; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7614; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
7615; GFX7-NEXT:    s_endpgm
7616;
7617; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_atomicrmw:
7618; GFX10-WGP:       ; %bb.0: ; %entry
7619; GFX10-WGP-NEXT:    s_clause 0x1
7620; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7621; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7622; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
7623; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7624; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
7625; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
7626; GFX10-WGP-NEXT:    s_endpgm
7627;
7628; GFX10-CU-LABEL: global_singlethread_one_as_acquire_atomicrmw:
7629; GFX10-CU:       ; %bb.0: ; %entry
7630; GFX10-CU-NEXT:    s_clause 0x1
7631; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7632; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7633; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
7634; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7635; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
7636; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
7637; GFX10-CU-NEXT:    s_endpgm
7638;
7639; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_atomicrmw:
7640; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7641; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
7642; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7643; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
7644; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
7645; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7646; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7647; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
7648; SKIP-CACHE-INV-NEXT:    s_endpgm
7649;
7650; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_atomicrmw:
7651; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7652; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7653; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7654; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7655; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7656; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7657; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
7658; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7659;
7660; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_atomicrmw:
7661; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7662; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7663; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7664; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7665; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7666; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7667; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
7668; GFX90A-TGSPLIT-NEXT:    s_endpgm
7669;
7670; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_atomicrmw:
7671; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7672; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7673; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7674; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7675; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7676; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
7677; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
7678; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7679;
7680; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acquire_atomicrmw:
7681; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7682; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7683; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7684; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7685; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7686; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
7687; GFX940-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
7688; GFX940-TGSPLIT-NEXT:    s_endpgm
7689;
7690; GFX11-WGP-LABEL: global_singlethread_one_as_acquire_atomicrmw:
7691; GFX11-WGP:       ; %bb.0: ; %entry
7692; GFX11-WGP-NEXT:    s_clause 0x1
7693; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x8
7694; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7695; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
7696; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7697; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s2
7698; GFX11-WGP-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
7699; GFX11-WGP-NEXT:    s_endpgm
7700;
7701; GFX11-CU-LABEL: global_singlethread_one_as_acquire_atomicrmw:
7702; GFX11-CU:       ; %bb.0: ; %entry
7703; GFX11-CU-NEXT:    s_clause 0x1
7704; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x8
7705; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7706; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
7707; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7708; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s2
7709; GFX11-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
7710; GFX11-CU-NEXT:    s_endpgm
7711    i32 addrspace(1)* %out, i32 %in) {
7712entry:
7713  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") acquire
7714  ret void
7715}
7716
7717define amdgpu_kernel void @global_singlethread_one_as_release_atomicrmw(
7718; GFX6-LABEL: global_singlethread_one_as_release_atomicrmw:
7719; GFX6:       ; %bb.0: ; %entry
7720; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
7721; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7722; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7723; GFX6-NEXT:    s_mov_b32 s2, -1
7724; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7725; GFX6-NEXT:    v_mov_b32_e32 v0, s6
7726; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
7727; GFX6-NEXT:    s_endpgm
7728;
7729; GFX7-LABEL: global_singlethread_one_as_release_atomicrmw:
7730; GFX7:       ; %bb.0: ; %entry
7731; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7732; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7733; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7734; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7735; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7736; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7737; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
7738; GFX7-NEXT:    s_endpgm
7739;
7740; GFX10-WGP-LABEL: global_singlethread_one_as_release_atomicrmw:
7741; GFX10-WGP:       ; %bb.0: ; %entry
7742; GFX10-WGP-NEXT:    s_clause 0x1
7743; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7744; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7745; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
7746; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7747; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
7748; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
7749; GFX10-WGP-NEXT:    s_endpgm
7750;
7751; GFX10-CU-LABEL: global_singlethread_one_as_release_atomicrmw:
7752; GFX10-CU:       ; %bb.0: ; %entry
7753; GFX10-CU-NEXT:    s_clause 0x1
7754; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7755; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7756; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
7757; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7758; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
7759; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
7760; GFX10-CU-NEXT:    s_endpgm
7761;
7762; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_atomicrmw:
7763; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7764; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
7765; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7766; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
7767; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
7768; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7769; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7770; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
7771; SKIP-CACHE-INV-NEXT:    s_endpgm
7772;
7773; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_atomicrmw:
7774; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7775; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7776; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7777; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7778; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7779; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7780; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
7781; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7782;
7783; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_release_atomicrmw:
7784; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7785; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7786; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7787; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7788; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7789; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7790; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
7791; GFX90A-TGSPLIT-NEXT:    s_endpgm
7792;
7793; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_atomicrmw:
7794; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7795; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7796; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7797; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7798; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7799; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
7800; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
7801; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7802;
7803; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_release_atomicrmw:
7804; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7805; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7806; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7807; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7808; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7809; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
7810; GFX940-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
7811; GFX940-TGSPLIT-NEXT:    s_endpgm
7812;
7813; GFX11-WGP-LABEL: global_singlethread_one_as_release_atomicrmw:
7814; GFX11-WGP:       ; %bb.0: ; %entry
7815; GFX11-WGP-NEXT:    s_clause 0x1
7816; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x8
7817; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7818; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
7819; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7820; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s2
7821; GFX11-WGP-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
7822; GFX11-WGP-NEXT:    s_endpgm
7823;
7824; GFX11-CU-LABEL: global_singlethread_one_as_release_atomicrmw:
7825; GFX11-CU:       ; %bb.0: ; %entry
7826; GFX11-CU-NEXT:    s_clause 0x1
7827; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x8
7828; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7829; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
7830; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7831; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s2
7832; GFX11-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
7833; GFX11-CU-NEXT:    s_endpgm
7834    i32 addrspace(1)* %out, i32 %in) {
7835entry:
7836  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") release
7837  ret void
7838}
7839
7840define amdgpu_kernel void @global_singlethread_one_as_acq_rel_atomicrmw(
7841; GFX6-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
7842; GFX6:       ; %bb.0: ; %entry
7843; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
7844; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7845; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7846; GFX6-NEXT:    s_mov_b32 s2, -1
7847; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7848; GFX6-NEXT:    v_mov_b32_e32 v0, s6
7849; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
7850; GFX6-NEXT:    s_endpgm
7851;
7852; GFX7-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
7853; GFX7:       ; %bb.0: ; %entry
7854; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7855; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7856; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7857; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7858; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7859; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7860; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
7861; GFX7-NEXT:    s_endpgm
7862;
7863; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
7864; GFX10-WGP:       ; %bb.0: ; %entry
7865; GFX10-WGP-NEXT:    s_clause 0x1
7866; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7867; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7868; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
7869; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7870; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
7871; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
7872; GFX10-WGP-NEXT:    s_endpgm
7873;
7874; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
7875; GFX10-CU:       ; %bb.0: ; %entry
7876; GFX10-CU-NEXT:    s_clause 0x1
7877; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7878; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7879; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
7880; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7881; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
7882; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
7883; GFX10-CU-NEXT:    s_endpgm
7884;
7885; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
7886; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7887; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
7888; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
7889; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
7890; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
7891; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7892; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7893; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
7894; SKIP-CACHE-INV-NEXT:    s_endpgm
7895;
7896; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
7897; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7898; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7899; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7900; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7901; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7902; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7903; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
7904; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7905;
7906; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
7907; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7908; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7909; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7910; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7911; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7912; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
7913; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
7914; GFX90A-TGSPLIT-NEXT:    s_endpgm
7915;
7916; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
7917; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7918; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7919; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7920; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7921; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7922; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
7923; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
7924; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7925;
7926; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
7927; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7928; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7929; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7930; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
7931; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7932; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
7933; GFX940-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
7934; GFX940-TGSPLIT-NEXT:    s_endpgm
7935;
7936; GFX11-WGP-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
7937; GFX11-WGP:       ; %bb.0: ; %entry
7938; GFX11-WGP-NEXT:    s_clause 0x1
7939; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x8
7940; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7941; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
7942; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7943; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s2
7944; GFX11-WGP-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
7945; GFX11-WGP-NEXT:    s_endpgm
7946;
7947; GFX11-CU-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
7948; GFX11-CU:       ; %bb.0: ; %entry
7949; GFX11-CU-NEXT:    s_clause 0x1
7950; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x8
7951; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
7952; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
7953; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7954; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s2
7955; GFX11-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
7956; GFX11-CU-NEXT:    s_endpgm
7957    i32 addrspace(1)* %out, i32 %in) {
7958entry:
7959  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") acq_rel
7960  ret void
7961}
7962
7963define amdgpu_kernel void @global_singlethread_one_as_seq_cst_atomicrmw(
7964; GFX6-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
7965; GFX6:       ; %bb.0: ; %entry
7966; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
7967; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7968; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7969; GFX6-NEXT:    s_mov_b32 s2, -1
7970; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7971; GFX6-NEXT:    v_mov_b32_e32 v0, s6
7972; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
7973; GFX6-NEXT:    s_endpgm
7974;
7975; GFX7-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
7976; GFX7:       ; %bb.0: ; %entry
7977; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7978; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7979; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7980; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7981; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7982; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7983; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
7984; GFX7-NEXT:    s_endpgm
7985;
7986; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
7987; GFX10-WGP:       ; %bb.0: ; %entry
7988; GFX10-WGP-NEXT:    s_clause 0x1
7989; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7990; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7991; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
7992; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7993; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
7994; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
7995; GFX10-WGP-NEXT:    s_endpgm
7996;
7997; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
7998; GFX10-CU:       ; %bb.0: ; %entry
7999; GFX10-CU-NEXT:    s_clause 0x1
8000; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8001; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8002; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
8003; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8004; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
8005; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
8006; GFX10-CU-NEXT:    s_endpgm
8007;
8008; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
8009; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8010; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
8011; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
8012; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
8013; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
8014; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8015; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8016; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
8017; SKIP-CACHE-INV-NEXT:    s_endpgm
8018;
8019; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
8020; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8021; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8022; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8023; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
8024; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8025; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8026; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
8027; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8028;
8029; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
8030; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8031; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8032; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8033; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
8034; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8035; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8036; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
8037; GFX90A-TGSPLIT-NEXT:    s_endpgm
8038;
8039; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
8040; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8041; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8042; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8043; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
8044; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8045; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
8046; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
8047; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8048;
8049; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
8050; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8051; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8052; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8053; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
8054; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8055; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
8056; GFX940-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[2:3]
8057; GFX940-TGSPLIT-NEXT:    s_endpgm
8058;
8059; GFX11-WGP-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
8060; GFX11-WGP:       ; %bb.0: ; %entry
8061; GFX11-WGP-NEXT:    s_clause 0x1
8062; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x8
8063; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
8064; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
8065; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8066; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s2
8067; GFX11-WGP-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
8068; GFX11-WGP-NEXT:    s_endpgm
8069;
8070; GFX11-CU-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
8071; GFX11-CU:       ; %bb.0: ; %entry
8072; GFX11-CU-NEXT:    s_clause 0x1
8073; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x8
8074; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
8075; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
8076; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8077; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s2
8078; GFX11-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
8079; GFX11-CU-NEXT:    s_endpgm
8080    i32 addrspace(1)* %out, i32 %in) {
8081entry:
8082  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") seq_cst
8083  ret void
8084}
8085
8086define amdgpu_kernel void @global_singlethread_one_as_acquire_ret_atomicrmw(
8087; GFX6-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
8088; GFX6:       ; %bb.0: ; %entry
8089; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
8090; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8091; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
8092; GFX6-NEXT:    s_mov_b32 s2, -1
8093; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8094; GFX6-NEXT:    v_mov_b32_e32 v0, s6
8095; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
8096; GFX6-NEXT:    s_waitcnt vmcnt(0)
8097; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
8098; GFX6-NEXT:    s_endpgm
8099;
8100; GFX7-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
8101; GFX7:       ; %bb.0: ; %entry
8102; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8103; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
8104; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8105; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8106; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8107; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8108; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8109; GFX7-NEXT:    s_waitcnt vmcnt(0)
8110; GFX7-NEXT:    flat_store_dword v[0:1], v2
8111; GFX7-NEXT:    s_endpgm
8112;
8113; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
8114; GFX10-WGP:       ; %bb.0: ; %entry
8115; GFX10-WGP-NEXT:    s_clause 0x1
8116; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
8117; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8118; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
8119; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8120; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
8121; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
8122; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8123; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
8124; GFX10-WGP-NEXT:    s_endpgm
8125;
8126; GFX10-CU-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
8127; GFX10-CU:       ; %bb.0: ; %entry
8128; GFX10-CU-NEXT:    s_clause 0x1
8129; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8130; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8131; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
8132; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8133; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
8134; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
8135; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8136; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
8137; GFX10-CU-NEXT:    s_endpgm
8138;
8139; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
8140; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8141; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
8142; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
8143; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
8144; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
8145; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8146; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8147; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
8148; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8149; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
8150; SKIP-CACHE-INV-NEXT:    s_endpgm
8151;
8152; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
8153; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8154; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8155; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8156; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
8157; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8158; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8159; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
8160; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8161; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
8162; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8163;
8164; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
8165; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8166; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8167; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8168; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
8169; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8170; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8171; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
8172; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8173; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
8174; GFX90A-TGSPLIT-NEXT:    s_endpgm
8175;
8176; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
8177; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8178; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8179; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8180; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
8181; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8182; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
8183; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[2:3] sc0
8184; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8185; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
8186; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8187;
8188; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
8189; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8190; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8191; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8192; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
8193; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8194; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
8195; GFX940-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[2:3] sc0
8196; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8197; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
8198; GFX940-TGSPLIT-NEXT:    s_endpgm
8199;
8200; GFX11-WGP-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
8201; GFX11-WGP:       ; %bb.0: ; %entry
8202; GFX11-WGP-NEXT:    s_clause 0x1
8203; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x8
8204; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
8205; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
8206; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8207; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s2
8208; GFX11-WGP-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
8209; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8210; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[0:1]
8211; GFX11-WGP-NEXT:    s_endpgm
8212;
8213; GFX11-CU-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
8214; GFX11-CU:       ; %bb.0: ; %entry
8215; GFX11-CU-NEXT:    s_clause 0x1
8216; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x8
8217; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
8218; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
8219; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8220; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s2
8221; GFX11-CU-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
8222; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
8223; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
8224; GFX11-CU-NEXT:    s_endpgm
8225    i32 addrspace(1)* %out, i32 %in) {
8226entry:
8227  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") acquire
8228  store i32 %val, i32 addrspace(1)* %out, align 4
8229  ret void
8230}
8231
8232define amdgpu_kernel void @global_singlethread_one_as_acq_rel_ret_atomicrmw(
8233; GFX6-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
8234; GFX6:       ; %bb.0: ; %entry
8235; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
8236; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8237; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
8238; GFX6-NEXT:    s_mov_b32 s2, -1
8239; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8240; GFX6-NEXT:    v_mov_b32_e32 v0, s6
8241; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
8242; GFX6-NEXT:    s_waitcnt vmcnt(0)
8243; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
8244; GFX6-NEXT:    s_endpgm
8245;
8246; GFX7-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
8247; GFX7:       ; %bb.0: ; %entry
8248; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8249; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
8250; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8251; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8252; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8253; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8254; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8255; GFX7-NEXT:    s_waitcnt vmcnt(0)
8256; GFX7-NEXT:    flat_store_dword v[0:1], v2
8257; GFX7-NEXT:    s_endpgm
8258;
8259; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
8260; GFX10-WGP:       ; %bb.0: ; %entry
8261; GFX10-WGP-NEXT:    s_clause 0x1
8262; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
8263; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8264; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
8265; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8266; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
8267; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
8268; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8269; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
8270; GFX10-WGP-NEXT:    s_endpgm
8271;
8272; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
8273; GFX10-CU:       ; %bb.0: ; %entry
8274; GFX10-CU-NEXT:    s_clause 0x1
8275; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8276; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8277; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
8278; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8279; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
8280; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
8281; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8282; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
8283; GFX10-CU-NEXT:    s_endpgm
8284;
8285; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
8286; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8287; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
8288; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
8289; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
8290; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
8291; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8292; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8293; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
8294; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8295; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
8296; SKIP-CACHE-INV-NEXT:    s_endpgm
8297;
8298; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
8299; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8300; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8301; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8302; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
8303; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8304; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8305; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
8306; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8307; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
8308; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8309;
8310; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
8311; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8312; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8313; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8314; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
8315; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8316; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8317; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
8318; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8319; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
8320; GFX90A-TGSPLIT-NEXT:    s_endpgm
8321;
8322; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
8323; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8324; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8325; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8326; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
8327; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8328; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
8329; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[2:3] sc0
8330; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8331; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
8332; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8333;
8334; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
8335; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8336; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8337; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8338; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
8339; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8340; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
8341; GFX940-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[2:3] sc0
8342; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8343; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
8344; GFX940-TGSPLIT-NEXT:    s_endpgm
8345;
8346; GFX11-WGP-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
8347; GFX11-WGP:       ; %bb.0: ; %entry
8348; GFX11-WGP-NEXT:    s_clause 0x1
8349; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x8
8350; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
8351; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
8352; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8353; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s2
8354; GFX11-WGP-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
8355; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8356; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[0:1]
8357; GFX11-WGP-NEXT:    s_endpgm
8358;
8359; GFX11-CU-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
8360; GFX11-CU:       ; %bb.0: ; %entry
8361; GFX11-CU-NEXT:    s_clause 0x1
8362; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x8
8363; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
8364; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
8365; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8366; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s2
8367; GFX11-CU-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
8368; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
8369; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
8370; GFX11-CU-NEXT:    s_endpgm
8371    i32 addrspace(1)* %out, i32 %in) {
8372entry:
8373  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") acq_rel
8374  store i32 %val, i32 addrspace(1)* %out, align 4
8375  ret void
8376}
8377
8378define amdgpu_kernel void @global_singlethread_one_as_seq_cst_ret_atomicrmw(
8379; GFX6-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
8380; GFX6:       ; %bb.0: ; %entry
8381; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x2
8382; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8383; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
8384; GFX6-NEXT:    s_mov_b32 s2, -1
8385; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8386; GFX6-NEXT:    v_mov_b32_e32 v0, s6
8387; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
8388; GFX6-NEXT:    s_waitcnt vmcnt(0)
8389; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
8390; GFX6-NEXT:    s_endpgm
8391;
8392; GFX7-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
8393; GFX7:       ; %bb.0: ; %entry
8394; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8395; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
8396; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8397; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8398; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8399; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8400; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
8401; GFX7-NEXT:    s_waitcnt vmcnt(0)
8402; GFX7-NEXT:    flat_store_dword v[0:1], v2
8403; GFX7-NEXT:    s_endpgm
8404;
8405; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
8406; GFX10-WGP:       ; %bb.0: ; %entry
8407; GFX10-WGP-NEXT:    s_clause 0x1
8408; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
8409; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8410; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
8411; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8412; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
8413; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
8414; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8415; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
8416; GFX10-WGP-NEXT:    s_endpgm
8417;
8418; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
8419; GFX10-CU:       ; %bb.0: ; %entry
8420; GFX10-CU-NEXT:    s_clause 0x1
8421; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8422; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8423; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
8424; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8425; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
8426; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
8427; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8428; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
8429; GFX10-CU-NEXT:    s_endpgm
8430;
8431; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
8432; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8433; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x2
8434; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
8435; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
8436; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
8437; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8438; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8439; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
8440; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8441; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
8442; SKIP-CACHE-INV-NEXT:    s_endpgm
8443;
8444; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
8445; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8446; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8447; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8448; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
8449; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8450; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8451; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
8452; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8453; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
8454; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8455;
8456; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
8457; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8458; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8459; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8460; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
8461; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8462; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
8463; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
8464; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8465; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
8466; GFX90A-TGSPLIT-NEXT:    s_endpgm
8467;
8468; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
8469; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8470; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8471; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8472; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
8473; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8474; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
8475; GFX940-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[2:3] sc0
8476; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8477; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
8478; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8479;
8480; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
8481; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8482; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8483; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8484; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
8485; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8486; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s4
8487; GFX940-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[2:3] sc0
8488; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8489; GFX940-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
8490; GFX940-TGSPLIT-NEXT:    s_endpgm
8491;
8492; GFX11-WGP-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
8493; GFX11-WGP:       ; %bb.0: ; %entry
8494; GFX11-WGP-NEXT:    s_clause 0x1
8495; GFX11-WGP-NEXT:    s_load_b32 s2, s[0:1], 0x8
8496; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
8497; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, 0
8498; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8499; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s2
8500; GFX11-WGP-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
8501; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8502; GFX11-WGP-NEXT:    global_store_b32 v0, v1, s[0:1]
8503; GFX11-WGP-NEXT:    s_endpgm
8504;
8505; GFX11-CU-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
8506; GFX11-CU:       ; %bb.0: ; %entry
8507; GFX11-CU-NEXT:    s_clause 0x1
8508; GFX11-CU-NEXT:    s_load_b32 s2, s[0:1], 0x8
8509; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
8510; GFX11-CU-NEXT:    v_mov_b32_e32 v0, 0
8511; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8512; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s2
8513; GFX11-CU-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
8514; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
8515; GFX11-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
8516; GFX11-CU-NEXT:    s_endpgm
8517    i32 addrspace(1)* %out, i32 %in) {
8518entry:
8519  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") seq_cst
8520  store i32 %val, i32 addrspace(1)* %out, align 4
8521  ret void
8522}
8523
8524define amdgpu_kernel void @global_singlethread_one_as_monotonic_monotonic_cmpxchg(
8525; GFX6-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
8526; GFX6:       ; %bb.0: ; %entry
8527; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
8528; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8529; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
8530; GFX6-NEXT:    s_mov_b32 s2, -1
8531; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8532; GFX6-NEXT:    v_mov_b32_e32 v0, s6
8533; GFX6-NEXT:    v_mov_b32_e32 v1, s7
8534; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
8535; GFX6-NEXT:    s_endpgm
8536;
8537; GFX7-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
8538; GFX7:       ; %bb.0: ; %entry
8539; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8540; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8541; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8542; GFX7-NEXT:    s_add_u32 s0, s0, 16
8543; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8544; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8545; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8546; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8547; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8548; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8549; GFX7-NEXT:    s_endpgm
8550;
8551; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
8552; GFX10-WGP:       ; %bb.0: ; %entry
8553; GFX10-WGP-NEXT:    s_clause 0x1
8554; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8555; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8556; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
8557; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8558; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8559; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8560; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8561; GFX10-WGP-NEXT:    s_endpgm
8562;
8563; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
8564; GFX10-CU:       ; %bb.0: ; %entry
8565; GFX10-CU-NEXT:    s_clause 0x1
8566; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8567; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8568; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
8569; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8570; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8571; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8572; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8573; GFX10-CU-NEXT:    s_endpgm
8574;
8575; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
8576; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8577; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
8578; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
8579; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
8580; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
8581; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8582; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8583; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8584; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
8585; SKIP-CACHE-INV-NEXT:    s_endpgm
8586;
8587; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
8588; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8589; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8590; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8591; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8592; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8593; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8594; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8595; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8596;
8597; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
8598; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8599; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8600; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8601; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8602; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8603; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8604; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8605; GFX90A-TGSPLIT-NEXT:    s_endpgm
8606;
8607; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
8608; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8609; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8610; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
8611; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8612; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8613; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8614; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
8615; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8616;
8617; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
8618; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8619; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8620; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
8621; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8622; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8623; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8624; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
8625; GFX940-TGSPLIT-NEXT:    s_endpgm
8626;
8627; GFX11-WGP-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
8628; GFX11-WGP:       ; %bb.0: ; %entry
8629; GFX11-WGP-NEXT:    s_clause 0x1
8630; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8631; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
8632; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
8633; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8634; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8635; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8636; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
8637; GFX11-WGP-NEXT:    s_endpgm
8638;
8639; GFX11-CU-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
8640; GFX11-CU:       ; %bb.0: ; %entry
8641; GFX11-CU-NEXT:    s_clause 0x1
8642; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8643; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
8644; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
8645; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8646; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8647; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8648; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
8649; GFX11-CU-NEXT:    s_endpgm
8650    i32 addrspace(1)* %out, i32 %in, i32 %old) {
8651entry:
8652  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
8653  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic monotonic
8654  ret void
8655}
8656
8657define amdgpu_kernel void @global_singlethread_one_as_acquire_monotonic_cmpxchg(
8658; GFX6-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
8659; GFX6:       ; %bb.0: ; %entry
8660; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
8661; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8662; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
8663; GFX6-NEXT:    s_mov_b32 s2, -1
8664; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8665; GFX6-NEXT:    v_mov_b32_e32 v0, s6
8666; GFX6-NEXT:    v_mov_b32_e32 v1, s7
8667; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
8668; GFX6-NEXT:    s_endpgm
8669;
8670; GFX7-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
8671; GFX7:       ; %bb.0: ; %entry
8672; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8673; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8674; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8675; GFX7-NEXT:    s_add_u32 s0, s0, 16
8676; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8677; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8678; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8679; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8680; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8681; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8682; GFX7-NEXT:    s_endpgm
8683;
8684; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
8685; GFX10-WGP:       ; %bb.0: ; %entry
8686; GFX10-WGP-NEXT:    s_clause 0x1
8687; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8688; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8689; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
8690; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8691; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8692; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8693; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8694; GFX10-WGP-NEXT:    s_endpgm
8695;
8696; GFX10-CU-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
8697; GFX10-CU:       ; %bb.0: ; %entry
8698; GFX10-CU-NEXT:    s_clause 0x1
8699; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8700; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8701; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
8702; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8703; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8704; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8705; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8706; GFX10-CU-NEXT:    s_endpgm
8707;
8708; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
8709; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8710; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
8711; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
8712; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
8713; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
8714; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8715; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8716; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8717; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
8718; SKIP-CACHE-INV-NEXT:    s_endpgm
8719;
8720; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
8721; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8722; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8723; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8724; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8725; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8726; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8727; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8728; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8729;
8730; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
8731; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8732; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8733; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8734; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8735; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8736; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8737; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8738; GFX90A-TGSPLIT-NEXT:    s_endpgm
8739;
8740; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
8741; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8742; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8743; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
8744; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8745; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8746; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8747; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
8748; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8749;
8750; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
8751; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8752; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8753; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
8754; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8755; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8756; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8757; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
8758; GFX940-TGSPLIT-NEXT:    s_endpgm
8759;
8760; GFX11-WGP-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
8761; GFX11-WGP:       ; %bb.0: ; %entry
8762; GFX11-WGP-NEXT:    s_clause 0x1
8763; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8764; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
8765; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
8766; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8767; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8768; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8769; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
8770; GFX11-WGP-NEXT:    s_endpgm
8771;
8772; GFX11-CU-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
8773; GFX11-CU:       ; %bb.0: ; %entry
8774; GFX11-CU-NEXT:    s_clause 0x1
8775; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8776; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
8777; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
8778; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8779; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8780; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8781; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
8782; GFX11-CU-NEXT:    s_endpgm
8783    i32 addrspace(1)* %out, i32 %in, i32 %old) {
8784entry:
8785  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
8786  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire monotonic
8787  ret void
8788}
8789
8790define amdgpu_kernel void @global_singlethread_one_as_release_monotonic_cmpxchg(
8791; GFX6-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
8792; GFX6:       ; %bb.0: ; %entry
8793; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
8794; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8795; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
8796; GFX6-NEXT:    s_mov_b32 s2, -1
8797; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8798; GFX6-NEXT:    v_mov_b32_e32 v0, s6
8799; GFX6-NEXT:    v_mov_b32_e32 v1, s7
8800; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
8801; GFX6-NEXT:    s_endpgm
8802;
8803; GFX7-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
8804; GFX7:       ; %bb.0: ; %entry
8805; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8806; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8807; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8808; GFX7-NEXT:    s_add_u32 s0, s0, 16
8809; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8810; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8811; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8812; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8813; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8814; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8815; GFX7-NEXT:    s_endpgm
8816;
8817; GFX10-WGP-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
8818; GFX10-WGP:       ; %bb.0: ; %entry
8819; GFX10-WGP-NEXT:    s_clause 0x1
8820; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8821; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8822; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
8823; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8824; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8825; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8826; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8827; GFX10-WGP-NEXT:    s_endpgm
8828;
8829; GFX10-CU-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
8830; GFX10-CU:       ; %bb.0: ; %entry
8831; GFX10-CU-NEXT:    s_clause 0x1
8832; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8833; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8834; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
8835; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8836; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8837; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8838; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8839; GFX10-CU-NEXT:    s_endpgm
8840;
8841; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
8842; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8843; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
8844; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
8845; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
8846; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
8847; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8848; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8849; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8850; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
8851; SKIP-CACHE-INV-NEXT:    s_endpgm
8852;
8853; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
8854; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8855; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8856; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8857; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8858; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8859; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8860; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8861; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8862;
8863; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
8864; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8865; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8866; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8867; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8868; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8869; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8870; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8871; GFX90A-TGSPLIT-NEXT:    s_endpgm
8872;
8873; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
8874; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8875; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8876; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
8877; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8878; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8879; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8880; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
8881; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8882;
8883; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
8884; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8885; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8886; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
8887; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8888; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8889; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8890; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
8891; GFX940-TGSPLIT-NEXT:    s_endpgm
8892;
8893; GFX11-WGP-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
8894; GFX11-WGP:       ; %bb.0: ; %entry
8895; GFX11-WGP-NEXT:    s_clause 0x1
8896; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8897; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
8898; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
8899; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8900; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8901; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8902; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
8903; GFX11-WGP-NEXT:    s_endpgm
8904;
8905; GFX11-CU-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
8906; GFX11-CU:       ; %bb.0: ; %entry
8907; GFX11-CU-NEXT:    s_clause 0x1
8908; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8909; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
8910; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
8911; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8912; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8913; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8914; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
8915; GFX11-CU-NEXT:    s_endpgm
8916    i32 addrspace(1)* %out, i32 %in, i32 %old) {
8917entry:
8918  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
8919  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release monotonic
8920  ret void
8921}
8922
8923define amdgpu_kernel void @global_singlethread_one_as_acq_rel_monotonic_cmpxchg(
8924; GFX6-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8925; GFX6:       ; %bb.0: ; %entry
8926; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
8927; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8928; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
8929; GFX6-NEXT:    s_mov_b32 s2, -1
8930; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8931; GFX6-NEXT:    v_mov_b32_e32 v0, s6
8932; GFX6-NEXT:    v_mov_b32_e32 v1, s7
8933; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
8934; GFX6-NEXT:    s_endpgm
8935;
8936; GFX7-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8937; GFX7:       ; %bb.0: ; %entry
8938; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8939; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8940; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8941; GFX7-NEXT:    s_add_u32 s0, s0, 16
8942; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8943; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8944; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8945; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8946; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8947; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8948; GFX7-NEXT:    s_endpgm
8949;
8950; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8951; GFX10-WGP:       ; %bb.0: ; %entry
8952; GFX10-WGP-NEXT:    s_clause 0x1
8953; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8954; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8955; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
8956; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8957; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8958; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8959; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8960; GFX10-WGP-NEXT:    s_endpgm
8961;
8962; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8963; GFX10-CU:       ; %bb.0: ; %entry
8964; GFX10-CU-NEXT:    s_clause 0x1
8965; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8966; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8967; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
8968; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8969; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8970; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8971; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8972; GFX10-CU-NEXT:    s_endpgm
8973;
8974; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8975; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8976; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
8977; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
8978; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
8979; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
8980; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8981; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8982; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8983; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
8984; SKIP-CACHE-INV-NEXT:    s_endpgm
8985;
8986; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8987; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8988; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8989; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8990; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8991; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8992; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8993; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
8994; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8995;
8996; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
8997; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8998; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8999; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9000; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9001; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9002; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9003; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9004; GFX90A-TGSPLIT-NEXT:    s_endpgm
9005;
9006; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
9007; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9008; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9009; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9010; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9011; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9012; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9013; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
9014; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9015;
9016; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
9017; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9018; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9019; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9020; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9021; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9022; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9023; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
9024; GFX940-TGSPLIT-NEXT:    s_endpgm
9025;
9026; GFX11-WGP-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
9027; GFX11-WGP:       ; %bb.0: ; %entry
9028; GFX11-WGP-NEXT:    s_clause 0x1
9029; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
9030; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
9031; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
9032; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9033; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9034; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9035; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
9036; GFX11-WGP-NEXT:    s_endpgm
9037;
9038; GFX11-CU-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
9039; GFX11-CU:       ; %bb.0: ; %entry
9040; GFX11-CU-NEXT:    s_clause 0x1
9041; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
9042; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
9043; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
9044; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9045; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9046; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9047; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
9048; GFX11-CU-NEXT:    s_endpgm
9049    i32 addrspace(1)* %out, i32 %in, i32 %old) {
9050entry:
9051  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
9052  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel monotonic
9053  ret void
9054}
9055
9056define amdgpu_kernel void @global_singlethread_one_as_seq_cst_monotonic_cmpxchg(
9057; GFX6-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
9058; GFX6:       ; %bb.0: ; %entry
9059; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
9060; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9061; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
9062; GFX6-NEXT:    s_mov_b32 s2, -1
9063; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9064; GFX6-NEXT:    v_mov_b32_e32 v0, s6
9065; GFX6-NEXT:    v_mov_b32_e32 v1, s7
9066; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
9067; GFX6-NEXT:    s_endpgm
9068;
9069; GFX7-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
9070; GFX7:       ; %bb.0: ; %entry
9071; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9072; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9073; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9074; GFX7-NEXT:    s_add_u32 s0, s0, 16
9075; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9076; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9077; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9078; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9079; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9080; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9081; GFX7-NEXT:    s_endpgm
9082;
9083; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
9084; GFX10-WGP:       ; %bb.0: ; %entry
9085; GFX10-WGP-NEXT:    s_clause 0x1
9086; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9087; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9088; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
9089; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9090; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9091; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9092; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9093; GFX10-WGP-NEXT:    s_endpgm
9094;
9095; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
9096; GFX10-CU:       ; %bb.0: ; %entry
9097; GFX10-CU-NEXT:    s_clause 0x1
9098; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9099; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9100; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
9101; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9102; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9103; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9104; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9105; GFX10-CU-NEXT:    s_endpgm
9106;
9107; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
9108; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9109; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
9110; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
9111; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
9112; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
9113; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9114; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
9115; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
9116; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
9117; SKIP-CACHE-INV-NEXT:    s_endpgm
9118;
9119; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
9120; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9121; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9122; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9123; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9124; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9125; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9126; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9127; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9128;
9129; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
9130; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9131; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9132; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9133; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9134; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9135; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9136; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9137; GFX90A-TGSPLIT-NEXT:    s_endpgm
9138;
9139; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
9140; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9141; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9142; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9143; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9144; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9145; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9146; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
9147; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9148;
9149; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
9150; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9151; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9152; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9153; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9154; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9155; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9156; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
9157; GFX940-TGSPLIT-NEXT:    s_endpgm
9158;
9159; GFX11-WGP-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
9160; GFX11-WGP:       ; %bb.0: ; %entry
9161; GFX11-WGP-NEXT:    s_clause 0x1
9162; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
9163; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
9164; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
9165; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9166; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9167; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9168; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
9169; GFX11-WGP-NEXT:    s_endpgm
9170;
9171; GFX11-CU-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
9172; GFX11-CU:       ; %bb.0: ; %entry
9173; GFX11-CU-NEXT:    s_clause 0x1
9174; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
9175; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
9176; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
9177; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9178; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9179; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9180; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
9181; GFX11-CU-NEXT:    s_endpgm
9182    i32 addrspace(1)* %out, i32 %in, i32 %old) {
9183entry:
9184  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
9185  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst monotonic
9186  ret void
9187}
9188
9189define amdgpu_kernel void @global_singlethread_one_as_monotonic_acquire_cmpxchg(
9190; GFX6-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
9191; GFX6:       ; %bb.0: ; %entry
9192; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
9193; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9194; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
9195; GFX6-NEXT:    s_mov_b32 s2, -1
9196; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9197; GFX6-NEXT:    v_mov_b32_e32 v0, s6
9198; GFX6-NEXT:    v_mov_b32_e32 v1, s7
9199; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
9200; GFX6-NEXT:    s_endpgm
9201;
9202; GFX7-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
9203; GFX7:       ; %bb.0: ; %entry
9204; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9205; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9206; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9207; GFX7-NEXT:    s_add_u32 s0, s0, 16
9208; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9209; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9210; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9211; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9212; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9213; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9214; GFX7-NEXT:    s_endpgm
9215;
9216; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
9217; GFX10-WGP:       ; %bb.0: ; %entry
9218; GFX10-WGP-NEXT:    s_clause 0x1
9219; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9220; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9221; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
9222; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9223; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9224; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9225; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9226; GFX10-WGP-NEXT:    s_endpgm
9227;
9228; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
9229; GFX10-CU:       ; %bb.0: ; %entry
9230; GFX10-CU-NEXT:    s_clause 0x1
9231; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9232; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9233; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
9234; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9235; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9236; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9237; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9238; GFX10-CU-NEXT:    s_endpgm
9239;
9240; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
9241; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9242; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
9243; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
9244; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
9245; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
9246; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9247; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
9248; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
9249; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
9250; SKIP-CACHE-INV-NEXT:    s_endpgm
9251;
9252; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
9253; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9254; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9255; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9256; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9257; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9258; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9259; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9260; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9261;
9262; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
9263; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9264; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9265; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9266; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9267; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9268; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9269; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9270; GFX90A-TGSPLIT-NEXT:    s_endpgm
9271;
9272; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
9273; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9274; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9275; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9276; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9277; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9278; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9279; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
9280; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9281;
9282; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
9283; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9284; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9285; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9286; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9287; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9288; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9289; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
9290; GFX940-TGSPLIT-NEXT:    s_endpgm
9291;
9292; GFX11-WGP-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
9293; GFX11-WGP:       ; %bb.0: ; %entry
9294; GFX11-WGP-NEXT:    s_clause 0x1
9295; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
9296; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
9297; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
9298; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9299; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9300; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9301; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
9302; GFX11-WGP-NEXT:    s_endpgm
9303;
9304; GFX11-CU-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
9305; GFX11-CU:       ; %bb.0: ; %entry
9306; GFX11-CU-NEXT:    s_clause 0x1
9307; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
9308; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
9309; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
9310; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9311; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9312; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9313; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
9314; GFX11-CU-NEXT:    s_endpgm
9315    i32 addrspace(1)* %out, i32 %in, i32 %old) {
9316entry:
9317  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
9318  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic acquire
9319  ret void
9320}
9321
9322define amdgpu_kernel void @global_singlethread_one_as_acquire_acquire_cmpxchg(
9323; GFX6-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
9324; GFX6:       ; %bb.0: ; %entry
9325; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
9326; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9327; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
9328; GFX6-NEXT:    s_mov_b32 s2, -1
9329; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9330; GFX6-NEXT:    v_mov_b32_e32 v0, s6
9331; GFX6-NEXT:    v_mov_b32_e32 v1, s7
9332; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
9333; GFX6-NEXT:    s_endpgm
9334;
9335; GFX7-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
9336; GFX7:       ; %bb.0: ; %entry
9337; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9338; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9339; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9340; GFX7-NEXT:    s_add_u32 s0, s0, 16
9341; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9342; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9343; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9344; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9345; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9346; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9347; GFX7-NEXT:    s_endpgm
9348;
9349; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
9350; GFX10-WGP:       ; %bb.0: ; %entry
9351; GFX10-WGP-NEXT:    s_clause 0x1
9352; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9353; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9354; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
9355; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9356; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9357; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9358; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9359; GFX10-WGP-NEXT:    s_endpgm
9360;
9361; GFX10-CU-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
9362; GFX10-CU:       ; %bb.0: ; %entry
9363; GFX10-CU-NEXT:    s_clause 0x1
9364; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9365; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9366; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
9367; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9368; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9369; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9370; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9371; GFX10-CU-NEXT:    s_endpgm
9372;
9373; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
9374; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9375; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
9376; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
9377; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
9378; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
9379; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9380; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
9381; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
9382; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
9383; SKIP-CACHE-INV-NEXT:    s_endpgm
9384;
9385; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
9386; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9387; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9388; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9389; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9390; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9391; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9392; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9393; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9394;
9395; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
9396; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9397; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9398; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9399; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9400; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9401; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9402; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9403; GFX90A-TGSPLIT-NEXT:    s_endpgm
9404;
9405; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
9406; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9407; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9408; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9409; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9410; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9411; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9412; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
9413; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9414;
9415; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
9416; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9417; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9418; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9419; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9420; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9421; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9422; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
9423; GFX940-TGSPLIT-NEXT:    s_endpgm
9424;
9425; GFX11-WGP-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
9426; GFX11-WGP:       ; %bb.0: ; %entry
9427; GFX11-WGP-NEXT:    s_clause 0x1
9428; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
9429; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
9430; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
9431; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9432; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9433; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9434; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
9435; GFX11-WGP-NEXT:    s_endpgm
9436;
9437; GFX11-CU-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
9438; GFX11-CU:       ; %bb.0: ; %entry
9439; GFX11-CU-NEXT:    s_clause 0x1
9440; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
9441; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
9442; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
9443; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9444; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9445; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9446; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
9447; GFX11-CU-NEXT:    s_endpgm
9448    i32 addrspace(1)* %out, i32 %in, i32 %old) {
9449entry:
9450  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
9451  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire acquire
9452  ret void
9453}
9454
9455define amdgpu_kernel void @global_singlethread_one_as_release_acquire_cmpxchg(
9456; GFX6-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
9457; GFX6:       ; %bb.0: ; %entry
9458; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
9459; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9460; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
9461; GFX6-NEXT:    s_mov_b32 s2, -1
9462; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9463; GFX6-NEXT:    v_mov_b32_e32 v0, s6
9464; GFX6-NEXT:    v_mov_b32_e32 v1, s7
9465; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
9466; GFX6-NEXT:    s_endpgm
9467;
9468; GFX7-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
9469; GFX7:       ; %bb.0: ; %entry
9470; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9471; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9472; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9473; GFX7-NEXT:    s_add_u32 s0, s0, 16
9474; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9475; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9476; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9477; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9478; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9479; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9480; GFX7-NEXT:    s_endpgm
9481;
9482; GFX10-WGP-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
9483; GFX10-WGP:       ; %bb.0: ; %entry
9484; GFX10-WGP-NEXT:    s_clause 0x1
9485; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9486; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9487; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
9488; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9489; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9490; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9491; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9492; GFX10-WGP-NEXT:    s_endpgm
9493;
9494; GFX10-CU-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
9495; GFX10-CU:       ; %bb.0: ; %entry
9496; GFX10-CU-NEXT:    s_clause 0x1
9497; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9498; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9499; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
9500; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9501; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9502; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9503; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9504; GFX10-CU-NEXT:    s_endpgm
9505;
9506; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
9507; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9508; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
9509; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
9510; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
9511; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
9512; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9513; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
9514; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
9515; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
9516; SKIP-CACHE-INV-NEXT:    s_endpgm
9517;
9518; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
9519; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9520; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9521; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9522; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9523; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9524; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9525; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9526; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9527;
9528; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
9529; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9530; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9531; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9532; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9533; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9534; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9535; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9536; GFX90A-TGSPLIT-NEXT:    s_endpgm
9537;
9538; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
9539; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9540; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9541; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9542; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9543; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9544; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9545; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
9546; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9547;
9548; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
9549; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9550; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9551; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9552; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9553; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9554; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9555; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
9556; GFX940-TGSPLIT-NEXT:    s_endpgm
9557;
9558; GFX11-WGP-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
9559; GFX11-WGP:       ; %bb.0: ; %entry
9560; GFX11-WGP-NEXT:    s_clause 0x1
9561; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
9562; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
9563; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
9564; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9565; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9566; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9567; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
9568; GFX11-WGP-NEXT:    s_endpgm
9569;
9570; GFX11-CU-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
9571; GFX11-CU:       ; %bb.0: ; %entry
9572; GFX11-CU-NEXT:    s_clause 0x1
9573; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
9574; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
9575; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
9576; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9577; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9578; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9579; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
9580; GFX11-CU-NEXT:    s_endpgm
9581    i32 addrspace(1)* %out, i32 %in, i32 %old) {
9582entry:
9583  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
9584  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release acquire
9585  ret void
9586}
9587
9588define amdgpu_kernel void @global_singlethread_one_as_acq_rel_acquire_cmpxchg(
9589; GFX6-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
9590; GFX6:       ; %bb.0: ; %entry
9591; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
9592; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9593; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
9594; GFX6-NEXT:    s_mov_b32 s2, -1
9595; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9596; GFX6-NEXT:    v_mov_b32_e32 v0, s6
9597; GFX6-NEXT:    v_mov_b32_e32 v1, s7
9598; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
9599; GFX6-NEXT:    s_endpgm
9600;
9601; GFX7-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
9602; GFX7:       ; %bb.0: ; %entry
9603; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9604; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9605; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9606; GFX7-NEXT:    s_add_u32 s0, s0, 16
9607; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9608; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9609; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9610; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9611; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9612; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9613; GFX7-NEXT:    s_endpgm
9614;
9615; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
9616; GFX10-WGP:       ; %bb.0: ; %entry
9617; GFX10-WGP-NEXT:    s_clause 0x1
9618; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9619; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9620; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
9621; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9622; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9623; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9624; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9625; GFX10-WGP-NEXT:    s_endpgm
9626;
9627; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
9628; GFX10-CU:       ; %bb.0: ; %entry
9629; GFX10-CU-NEXT:    s_clause 0x1
9630; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9631; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9632; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
9633; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9634; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9635; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9636; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9637; GFX10-CU-NEXT:    s_endpgm
9638;
9639; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
9640; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9641; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
9642; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
9643; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
9644; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
9645; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9646; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
9647; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
9648; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
9649; SKIP-CACHE-INV-NEXT:    s_endpgm
9650;
9651; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
9652; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9653; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9654; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9655; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9656; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9657; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9658; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9659; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9660;
9661; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
9662; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9663; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9664; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9665; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9666; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9667; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9668; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9669; GFX90A-TGSPLIT-NEXT:    s_endpgm
9670;
9671; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
9672; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9673; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9674; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9675; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9676; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9677; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9678; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
9679; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9680;
9681; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
9682; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9683; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9684; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9685; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9686; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9687; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9688; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
9689; GFX940-TGSPLIT-NEXT:    s_endpgm
9690;
9691; GFX11-WGP-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
9692; GFX11-WGP:       ; %bb.0: ; %entry
9693; GFX11-WGP-NEXT:    s_clause 0x1
9694; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
9695; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
9696; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
9697; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9698; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9699; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9700; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
9701; GFX11-WGP-NEXT:    s_endpgm
9702;
9703; GFX11-CU-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
9704; GFX11-CU:       ; %bb.0: ; %entry
9705; GFX11-CU-NEXT:    s_clause 0x1
9706; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
9707; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
9708; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
9709; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9710; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9711; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9712; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
9713; GFX11-CU-NEXT:    s_endpgm
9714    i32 addrspace(1)* %out, i32 %in, i32 %old) {
9715entry:
9716  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
9717  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel acquire
9718  ret void
9719}
9720
9721define amdgpu_kernel void @global_singlethread_one_as_seq_cst_acquire_cmpxchg(
9722; GFX6-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
9723; GFX6:       ; %bb.0: ; %entry
9724; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
9725; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9726; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
9727; GFX6-NEXT:    s_mov_b32 s2, -1
9728; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9729; GFX6-NEXT:    v_mov_b32_e32 v0, s6
9730; GFX6-NEXT:    v_mov_b32_e32 v1, s7
9731; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
9732; GFX6-NEXT:    s_endpgm
9733;
9734; GFX7-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
9735; GFX7:       ; %bb.0: ; %entry
9736; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9737; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9738; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9739; GFX7-NEXT:    s_add_u32 s0, s0, 16
9740; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9741; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9742; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9743; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9744; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9745; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9746; GFX7-NEXT:    s_endpgm
9747;
9748; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
9749; GFX10-WGP:       ; %bb.0: ; %entry
9750; GFX10-WGP-NEXT:    s_clause 0x1
9751; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9752; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9753; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
9754; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9755; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9756; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9757; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9758; GFX10-WGP-NEXT:    s_endpgm
9759;
9760; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
9761; GFX10-CU:       ; %bb.0: ; %entry
9762; GFX10-CU-NEXT:    s_clause 0x1
9763; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9764; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9765; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
9766; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9767; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9768; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9769; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9770; GFX10-CU-NEXT:    s_endpgm
9771;
9772; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
9773; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9774; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
9775; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
9776; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
9777; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
9778; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9779; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
9780; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
9781; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
9782; SKIP-CACHE-INV-NEXT:    s_endpgm
9783;
9784; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
9785; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9786; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9787; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9788; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9789; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9790; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9791; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9792; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9793;
9794; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
9795; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9796; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9797; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9798; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9799; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9800; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9801; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9802; GFX90A-TGSPLIT-NEXT:    s_endpgm
9803;
9804; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
9805; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9806; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9807; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9808; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9809; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9810; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9811; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
9812; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9813;
9814; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
9815; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9816; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9817; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9818; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9819; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9820; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9821; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
9822; GFX940-TGSPLIT-NEXT:    s_endpgm
9823;
9824; GFX11-WGP-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
9825; GFX11-WGP:       ; %bb.0: ; %entry
9826; GFX11-WGP-NEXT:    s_clause 0x1
9827; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
9828; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
9829; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
9830; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9831; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9832; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9833; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
9834; GFX11-WGP-NEXT:    s_endpgm
9835;
9836; GFX11-CU-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
9837; GFX11-CU:       ; %bb.0: ; %entry
9838; GFX11-CU-NEXT:    s_clause 0x1
9839; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
9840; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
9841; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
9842; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9843; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9844; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9845; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
9846; GFX11-CU-NEXT:    s_endpgm
9847    i32 addrspace(1)* %out, i32 %in, i32 %old) {
9848entry:
9849  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
9850  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst acquire
9851  ret void
9852}
9853
9854define amdgpu_kernel void @global_singlethread_one_as_monotonic_seq_cst_cmpxchg(
9855; GFX6-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9856; GFX6:       ; %bb.0: ; %entry
9857; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
9858; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9859; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
9860; GFX6-NEXT:    s_mov_b32 s2, -1
9861; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9862; GFX6-NEXT:    v_mov_b32_e32 v0, s6
9863; GFX6-NEXT:    v_mov_b32_e32 v1, s7
9864; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
9865; GFX6-NEXT:    s_endpgm
9866;
9867; GFX7-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9868; GFX7:       ; %bb.0: ; %entry
9869; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9870; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9871; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9872; GFX7-NEXT:    s_add_u32 s0, s0, 16
9873; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9874; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9875; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9876; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9877; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9878; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9879; GFX7-NEXT:    s_endpgm
9880;
9881; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9882; GFX10-WGP:       ; %bb.0: ; %entry
9883; GFX10-WGP-NEXT:    s_clause 0x1
9884; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9885; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9886; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
9887; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9888; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9889; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9890; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9891; GFX10-WGP-NEXT:    s_endpgm
9892;
9893; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9894; GFX10-CU:       ; %bb.0: ; %entry
9895; GFX10-CU-NEXT:    s_clause 0x1
9896; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9897; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9898; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
9899; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9900; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9901; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9902; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9903; GFX10-CU-NEXT:    s_endpgm
9904;
9905; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9906; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9907; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
9908; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
9909; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
9910; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
9911; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9912; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
9913; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
9914; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
9915; SKIP-CACHE-INV-NEXT:    s_endpgm
9916;
9917; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9918; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9919; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9920; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9921; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9922; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9923; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9924; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9925; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9926;
9927; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9928; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9929; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9930; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9931; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9932; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9933; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9934; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
9935; GFX90A-TGSPLIT-NEXT:    s_endpgm
9936;
9937; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9938; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9939; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9940; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9941; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9942; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9943; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9944; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
9945; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9946;
9947; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9948; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9949; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
9950; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
9951; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9952; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9953; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9954; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
9955; GFX940-TGSPLIT-NEXT:    s_endpgm
9956;
9957; GFX11-WGP-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9958; GFX11-WGP:       ; %bb.0: ; %entry
9959; GFX11-WGP-NEXT:    s_clause 0x1
9960; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
9961; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
9962; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
9963; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9964; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9965; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9966; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
9967; GFX11-WGP-NEXT:    s_endpgm
9968;
9969; GFX11-CU-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
9970; GFX11-CU:       ; %bb.0: ; %entry
9971; GFX11-CU-NEXT:    s_clause 0x1
9972; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
9973; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
9974; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
9975; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9976; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9977; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9978; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
9979; GFX11-CU-NEXT:    s_endpgm
9980    i32 addrspace(1)* %out, i32 %in, i32 %old) {
9981entry:
9982  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
9983  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic seq_cst
9984  ret void
9985}
9986
9987define amdgpu_kernel void @global_singlethread_one_as_acquire_seq_cst_cmpxchg(
9988; GFX6-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
9989; GFX6:       ; %bb.0: ; %entry
9990; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
9991; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9992; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
9993; GFX6-NEXT:    s_mov_b32 s2, -1
9994; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9995; GFX6-NEXT:    v_mov_b32_e32 v0, s6
9996; GFX6-NEXT:    v_mov_b32_e32 v1, s7
9997; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
9998; GFX6-NEXT:    s_endpgm
9999;
10000; GFX7-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
10001; GFX7:       ; %bb.0: ; %entry
10002; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10003; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10004; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10005; GFX7-NEXT:    s_add_u32 s0, s0, 16
10006; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10007; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10008; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10009; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10010; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10011; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10012; GFX7-NEXT:    s_endpgm
10013;
10014; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
10015; GFX10-WGP:       ; %bb.0: ; %entry
10016; GFX10-WGP-NEXT:    s_clause 0x1
10017; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10018; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10019; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
10020; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10021; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10022; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10023; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
10024; GFX10-WGP-NEXT:    s_endpgm
10025;
10026; GFX10-CU-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
10027; GFX10-CU:       ; %bb.0: ; %entry
10028; GFX10-CU-NEXT:    s_clause 0x1
10029; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10030; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10031; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
10032; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10033; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10034; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10035; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
10036; GFX10-CU-NEXT:    s_endpgm
10037;
10038; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
10039; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10040; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
10041; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
10042; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
10043; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
10044; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10045; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10046; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10047; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
10048; SKIP-CACHE-INV-NEXT:    s_endpgm
10049;
10050; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
10051; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10052; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10053; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10054; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10055; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10056; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10057; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
10058; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10059;
10060; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
10061; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10062; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10063; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10064; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10065; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10066; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10067; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
10068; GFX90A-TGSPLIT-NEXT:    s_endpgm
10069;
10070; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
10071; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10072; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10073; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10074; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10075; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10076; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10077; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
10078; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10079;
10080; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
10081; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10082; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10083; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10084; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10085; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10086; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10087; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
10088; GFX940-TGSPLIT-NEXT:    s_endpgm
10089;
10090; GFX11-WGP-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
10091; GFX11-WGP:       ; %bb.0: ; %entry
10092; GFX11-WGP-NEXT:    s_clause 0x1
10093; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
10094; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
10095; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
10096; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10097; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10098; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10099; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
10100; GFX11-WGP-NEXT:    s_endpgm
10101;
10102; GFX11-CU-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
10103; GFX11-CU:       ; %bb.0: ; %entry
10104; GFX11-CU-NEXT:    s_clause 0x1
10105; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
10106; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
10107; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
10108; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10109; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10110; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10111; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
10112; GFX11-CU-NEXT:    s_endpgm
10113    i32 addrspace(1)* %out, i32 %in, i32 %old) {
10114entry:
10115  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
10116  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire seq_cst
10117  ret void
10118}
10119
10120define amdgpu_kernel void @global_singlethread_one_as_release_seq_cst_cmpxchg(
10121; GFX6-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
10122; GFX6:       ; %bb.0: ; %entry
10123; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
10124; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10125; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
10126; GFX6-NEXT:    s_mov_b32 s2, -1
10127; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10128; GFX6-NEXT:    v_mov_b32_e32 v0, s6
10129; GFX6-NEXT:    v_mov_b32_e32 v1, s7
10130; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
10131; GFX6-NEXT:    s_endpgm
10132;
10133; GFX7-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
10134; GFX7:       ; %bb.0: ; %entry
10135; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10136; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10137; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10138; GFX7-NEXT:    s_add_u32 s0, s0, 16
10139; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10140; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10141; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10142; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10143; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10144; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10145; GFX7-NEXT:    s_endpgm
10146;
10147; GFX10-WGP-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
10148; GFX10-WGP:       ; %bb.0: ; %entry
10149; GFX10-WGP-NEXT:    s_clause 0x1
10150; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10151; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10152; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
10153; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10154; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10155; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10156; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
10157; GFX10-WGP-NEXT:    s_endpgm
10158;
10159; GFX10-CU-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
10160; GFX10-CU:       ; %bb.0: ; %entry
10161; GFX10-CU-NEXT:    s_clause 0x1
10162; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10163; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10164; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
10165; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10166; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10167; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10168; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
10169; GFX10-CU-NEXT:    s_endpgm
10170;
10171; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
10172; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10173; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
10174; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
10175; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
10176; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
10177; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10178; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10179; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10180; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
10181; SKIP-CACHE-INV-NEXT:    s_endpgm
10182;
10183; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
10184; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10185; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10186; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10187; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10188; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10189; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10190; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
10191; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10192;
10193; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
10194; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10195; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10196; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10197; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10198; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10199; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10200; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
10201; GFX90A-TGSPLIT-NEXT:    s_endpgm
10202;
10203; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
10204; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10205; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10206; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10207; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10208; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10209; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10210; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
10211; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10212;
10213; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
10214; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10215; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10216; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10217; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10218; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10219; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10220; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
10221; GFX940-TGSPLIT-NEXT:    s_endpgm
10222;
10223; GFX11-WGP-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
10224; GFX11-WGP:       ; %bb.0: ; %entry
10225; GFX11-WGP-NEXT:    s_clause 0x1
10226; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
10227; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
10228; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
10229; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10230; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10231; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10232; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
10233; GFX11-WGP-NEXT:    s_endpgm
10234;
10235; GFX11-CU-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
10236; GFX11-CU:       ; %bb.0: ; %entry
10237; GFX11-CU-NEXT:    s_clause 0x1
10238; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
10239; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
10240; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
10241; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10242; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10243; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10244; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
10245; GFX11-CU-NEXT:    s_endpgm
10246    i32 addrspace(1)* %out, i32 %in, i32 %old) {
10247entry:
10248  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
10249  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release seq_cst
10250  ret void
10251}
10252
10253define amdgpu_kernel void @global_singlethread_one_as_acq_rel_seq_cst_cmpxchg(
10254; GFX6-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10255; GFX6:       ; %bb.0: ; %entry
10256; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
10257; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10258; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
10259; GFX6-NEXT:    s_mov_b32 s2, -1
10260; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10261; GFX6-NEXT:    v_mov_b32_e32 v0, s6
10262; GFX6-NEXT:    v_mov_b32_e32 v1, s7
10263; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
10264; GFX6-NEXT:    s_endpgm
10265;
10266; GFX7-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10267; GFX7:       ; %bb.0: ; %entry
10268; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10269; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10270; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10271; GFX7-NEXT:    s_add_u32 s0, s0, 16
10272; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10273; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10274; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10275; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10276; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10277; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10278; GFX7-NEXT:    s_endpgm
10279;
10280; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10281; GFX10-WGP:       ; %bb.0: ; %entry
10282; GFX10-WGP-NEXT:    s_clause 0x1
10283; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10284; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10285; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
10286; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10287; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10288; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10289; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
10290; GFX10-WGP-NEXT:    s_endpgm
10291;
10292; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10293; GFX10-CU:       ; %bb.0: ; %entry
10294; GFX10-CU-NEXT:    s_clause 0x1
10295; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10296; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10297; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
10298; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10299; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10300; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10301; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
10302; GFX10-CU-NEXT:    s_endpgm
10303;
10304; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10305; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10306; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
10307; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
10308; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
10309; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
10310; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10311; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10312; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10313; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
10314; SKIP-CACHE-INV-NEXT:    s_endpgm
10315;
10316; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10317; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10318; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10319; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10320; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10321; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10322; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10323; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
10324; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10325;
10326; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10327; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10328; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10329; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10330; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10331; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10332; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10333; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
10334; GFX90A-TGSPLIT-NEXT:    s_endpgm
10335;
10336; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10337; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10338; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10339; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10340; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10341; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10342; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10343; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
10344; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10345;
10346; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10347; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10348; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10349; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10350; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10351; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10352; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10353; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
10354; GFX940-TGSPLIT-NEXT:    s_endpgm
10355;
10356; GFX11-WGP-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10357; GFX11-WGP:       ; %bb.0: ; %entry
10358; GFX11-WGP-NEXT:    s_clause 0x1
10359; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
10360; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
10361; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
10362; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10363; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10364; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10365; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
10366; GFX11-WGP-NEXT:    s_endpgm
10367;
10368; GFX11-CU-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
10369; GFX11-CU:       ; %bb.0: ; %entry
10370; GFX11-CU-NEXT:    s_clause 0x1
10371; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
10372; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
10373; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
10374; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10375; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10376; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10377; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
10378; GFX11-CU-NEXT:    s_endpgm
10379    i32 addrspace(1)* %out, i32 %in, i32 %old) {
10380entry:
10381  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
10382  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel seq_cst
10383  ret void
10384}
10385
10386define amdgpu_kernel void @global_singlethread_one_as_seq_cst_seq_cst_cmpxchg(
10387; GFX6-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10388; GFX6:       ; %bb.0: ; %entry
10389; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
10390; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10391; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
10392; GFX6-NEXT:    s_mov_b32 s2, -1
10393; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10394; GFX6-NEXT:    v_mov_b32_e32 v0, s6
10395; GFX6-NEXT:    v_mov_b32_e32 v1, s7
10396; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
10397; GFX6-NEXT:    s_endpgm
10398;
10399; GFX7-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10400; GFX7:       ; %bb.0: ; %entry
10401; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10402; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10403; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10404; GFX7-NEXT:    s_add_u32 s0, s0, 16
10405; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10406; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10407; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10408; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10409; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10410; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10411; GFX7-NEXT:    s_endpgm
10412;
10413; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10414; GFX10-WGP:       ; %bb.0: ; %entry
10415; GFX10-WGP-NEXT:    s_clause 0x1
10416; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10417; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10418; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
10419; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10420; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10421; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10422; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
10423; GFX10-WGP-NEXT:    s_endpgm
10424;
10425; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10426; GFX10-CU:       ; %bb.0: ; %entry
10427; GFX10-CU-NEXT:    s_clause 0x1
10428; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10429; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10430; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
10431; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10432; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10433; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10434; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
10435; GFX10-CU-NEXT:    s_endpgm
10436;
10437; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10438; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10439; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
10440; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
10441; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
10442; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
10443; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10444; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10445; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10446; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
10447; SKIP-CACHE-INV-NEXT:    s_endpgm
10448;
10449; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10450; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10451; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10452; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10453; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10454; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10455; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10456; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
10457; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10458;
10459; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10460; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10461; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10462; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10463; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10464; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10465; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10466; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
10467; GFX90A-TGSPLIT-NEXT:    s_endpgm
10468;
10469; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10470; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10471; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10472; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10473; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10474; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10475; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10476; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
10477; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10478;
10479; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10480; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10481; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10482; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10483; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10484; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10485; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10486; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
10487; GFX940-TGSPLIT-NEXT:    s_endpgm
10488;
10489; GFX11-WGP-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10490; GFX11-WGP:       ; %bb.0: ; %entry
10491; GFX11-WGP-NEXT:    s_clause 0x1
10492; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
10493; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
10494; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
10495; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10496; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10497; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10498; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
10499; GFX11-WGP-NEXT:    s_endpgm
10500;
10501; GFX11-CU-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
10502; GFX11-CU:       ; %bb.0: ; %entry
10503; GFX11-CU-NEXT:    s_clause 0x1
10504; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
10505; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
10506; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
10507; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10508; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10509; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10510; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
10511; GFX11-CU-NEXT:    s_endpgm
10512    i32 addrspace(1)* %out, i32 %in, i32 %old) {
10513entry:
10514  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
10515  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst seq_cst
10516  ret void
10517}
10518
10519define amdgpu_kernel void @global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg(
10520; GFX6-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10521; GFX6:       ; %bb.0: ; %entry
10522; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
10523; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10524; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
10525; GFX6-NEXT:    s_mov_b32 s2, -1
10526; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10527; GFX6-NEXT:    v_mov_b32_e32 v0, s6
10528; GFX6-NEXT:    v_mov_b32_e32 v1, s7
10529; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
10530; GFX6-NEXT:    s_waitcnt vmcnt(0)
10531; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
10532; GFX6-NEXT:    s_endpgm
10533;
10534; GFX7-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10535; GFX7:       ; %bb.0: ; %entry
10536; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10537; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10538; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10539; GFX7-NEXT:    s_add_u32 s4, s0, 16
10540; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10541; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10542; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10543; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10544; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10545; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10546; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10547; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10548; GFX7-NEXT:    s_waitcnt vmcnt(0)
10549; GFX7-NEXT:    flat_store_dword v[0:1], v2
10550; GFX7-NEXT:    s_endpgm
10551;
10552; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10553; GFX10-WGP:       ; %bb.0: ; %entry
10554; GFX10-WGP-NEXT:    s_clause 0x1
10555; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10556; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10557; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
10558; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10559; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10560; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10561; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10562; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10563; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
10564; GFX10-WGP-NEXT:    s_endpgm
10565;
10566; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10567; GFX10-CU:       ; %bb.0: ; %entry
10568; GFX10-CU-NEXT:    s_clause 0x1
10569; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10570; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10571; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
10572; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10573; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10574; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10575; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10576; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10577; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
10578; GFX10-CU-NEXT:    s_endpgm
10579;
10580; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10581; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10582; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
10583; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
10584; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
10585; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
10586; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10587; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10588; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10589; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
10590; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10591; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
10592; SKIP-CACHE-INV-NEXT:    s_endpgm
10593;
10594; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10595; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10596; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10597; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10598; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10599; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10600; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10601; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10602; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10603; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
10604; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10605;
10606; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10607; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10608; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10609; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10610; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10611; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10612; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10613; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10614; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10615; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
10616; GFX90A-TGSPLIT-NEXT:    s_endpgm
10617;
10618; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10619; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10620; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10621; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10622; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10623; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10624; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10625; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
10626; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10627; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
10628; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10629;
10630; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10631; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10632; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10633; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10634; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10635; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10636; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10637; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
10638; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10639; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
10640; GFX940-TGSPLIT-NEXT:    s_endpgm
10641;
10642; GFX11-WGP-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10643; GFX11-WGP:       ; %bb.0: ; %entry
10644; GFX11-WGP-NEXT:    s_clause 0x1
10645; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
10646; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
10647; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
10648; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10649; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10650; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10651; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
10652; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
10653; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
10654; GFX11-WGP-NEXT:    s_endpgm
10655;
10656; GFX11-CU-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
10657; GFX11-CU:       ; %bb.0: ; %entry
10658; GFX11-CU-NEXT:    s_clause 0x1
10659; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
10660; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
10661; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
10662; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10663; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10664; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10665; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
10666; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
10667; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
10668; GFX11-CU-NEXT:    s_endpgm
10669    i32 addrspace(1)* %out, i32 %in, i32 %old) {
10670entry:
10671  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
10672  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic monotonic
10673  %val0 = extractvalue { i32, i1 } %val, 0
10674  store i32 %val0, i32 addrspace(1)* %out, align 4
10675  ret void
10676}
10677
10678define amdgpu_kernel void @global_singlethread_one_as_acquire_monotonic_ret_cmpxchg(
10679; GFX6-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10680; GFX6:       ; %bb.0: ; %entry
10681; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
10682; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10683; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
10684; GFX6-NEXT:    s_mov_b32 s2, -1
10685; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10686; GFX6-NEXT:    v_mov_b32_e32 v0, s6
10687; GFX6-NEXT:    v_mov_b32_e32 v1, s7
10688; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
10689; GFX6-NEXT:    s_waitcnt vmcnt(0)
10690; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
10691; GFX6-NEXT:    s_endpgm
10692;
10693; GFX7-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10694; GFX7:       ; %bb.0: ; %entry
10695; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10696; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10697; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10698; GFX7-NEXT:    s_add_u32 s4, s0, 16
10699; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10700; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10701; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10702; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10703; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10704; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10705; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10706; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10707; GFX7-NEXT:    s_waitcnt vmcnt(0)
10708; GFX7-NEXT:    flat_store_dword v[0:1], v2
10709; GFX7-NEXT:    s_endpgm
10710;
10711; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10712; GFX10-WGP:       ; %bb.0: ; %entry
10713; GFX10-WGP-NEXT:    s_clause 0x1
10714; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10715; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10716; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
10717; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10718; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10719; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10720; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10721; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10722; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
10723; GFX10-WGP-NEXT:    s_endpgm
10724;
10725; GFX10-CU-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10726; GFX10-CU:       ; %bb.0: ; %entry
10727; GFX10-CU-NEXT:    s_clause 0x1
10728; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10729; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10730; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
10731; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10732; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10733; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10734; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10735; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10736; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
10737; GFX10-CU-NEXT:    s_endpgm
10738;
10739; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10740; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10741; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
10742; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
10743; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
10744; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
10745; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10746; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10747; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10748; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
10749; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10750; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
10751; SKIP-CACHE-INV-NEXT:    s_endpgm
10752;
10753; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10754; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10755; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10756; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10757; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10758; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10759; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10760; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10761; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10762; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
10763; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10764;
10765; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10766; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10767; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10768; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10769; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10770; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10771; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10772; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10773; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10774; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
10775; GFX90A-TGSPLIT-NEXT:    s_endpgm
10776;
10777; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10778; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10779; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10780; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10781; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10782; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10783; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10784; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
10785; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10786; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
10787; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10788;
10789; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10790; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10791; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10792; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10793; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10794; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10795; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10796; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
10797; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10798; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
10799; GFX940-TGSPLIT-NEXT:    s_endpgm
10800;
10801; GFX11-WGP-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10802; GFX11-WGP:       ; %bb.0: ; %entry
10803; GFX11-WGP-NEXT:    s_clause 0x1
10804; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
10805; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
10806; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
10807; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10808; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10809; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10810; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
10811; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
10812; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
10813; GFX11-WGP-NEXT:    s_endpgm
10814;
10815; GFX11-CU-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
10816; GFX11-CU:       ; %bb.0: ; %entry
10817; GFX11-CU-NEXT:    s_clause 0x1
10818; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
10819; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
10820; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
10821; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10822; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10823; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10824; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
10825; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
10826; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
10827; GFX11-CU-NEXT:    s_endpgm
10828    i32 addrspace(1)* %out, i32 %in, i32 %old) {
10829entry:
10830  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
10831  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire monotonic
10832  %val0 = extractvalue { i32, i1 } %val, 0
10833  store i32 %val0, i32 addrspace(1)* %out, align 4
10834  ret void
10835}
10836
10837define amdgpu_kernel void @global_singlethread_one_as_release_monotonic_ret_cmpxchg(
10838; GFX6-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
10839; GFX6:       ; %bb.0: ; %entry
10840; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
10841; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10842; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
10843; GFX6-NEXT:    s_mov_b32 s2, -1
10844; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
10845; GFX6-NEXT:    v_mov_b32_e32 v0, s6
10846; GFX6-NEXT:    v_mov_b32_e32 v1, s7
10847; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
10848; GFX6-NEXT:    s_waitcnt vmcnt(0)
10849; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
10850; GFX6-NEXT:    s_endpgm
10851;
10852; GFX7-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
10853; GFX7:       ; %bb.0: ; %entry
10854; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10855; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10856; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10857; GFX7-NEXT:    s_add_u32 s4, s0, 16
10858; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10859; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10860; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10861; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10862; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10863; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10864; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10865; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10866; GFX7-NEXT:    s_waitcnt vmcnt(0)
10867; GFX7-NEXT:    flat_store_dword v[0:1], v2
10868; GFX7-NEXT:    s_endpgm
10869;
10870; GFX10-WGP-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
10871; GFX10-WGP:       ; %bb.0: ; %entry
10872; GFX10-WGP-NEXT:    s_clause 0x1
10873; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10874; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10875; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
10876; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10877; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10878; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10879; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10880; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10881; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
10882; GFX10-WGP-NEXT:    s_endpgm
10883;
10884; GFX10-CU-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
10885; GFX10-CU:       ; %bb.0: ; %entry
10886; GFX10-CU-NEXT:    s_clause 0x1
10887; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10888; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10889; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
10890; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10891; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10892; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10893; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10894; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10895; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
10896; GFX10-CU-NEXT:    s_endpgm
10897;
10898; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
10899; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10900; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
10901; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
10902; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
10903; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
10904; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10905; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10906; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10907; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
10908; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10909; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
10910; SKIP-CACHE-INV-NEXT:    s_endpgm
10911;
10912; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
10913; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10914; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10915; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10916; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10917; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10918; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10919; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10920; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10921; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
10922; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10923;
10924; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
10925; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10926; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
10927; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
10928; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10929; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10930; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10931; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
10932; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10933; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
10934; GFX90A-TGSPLIT-NEXT:    s_endpgm
10935;
10936; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
10937; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10938; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10939; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10940; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10941; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10942; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10943; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
10944; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10945; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
10946; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10947;
10948; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
10949; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10950; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
10951; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
10952; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
10953; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10954; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10955; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
10956; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10957; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
10958; GFX940-TGSPLIT-NEXT:    s_endpgm
10959;
10960; GFX11-WGP-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
10961; GFX11-WGP:       ; %bb.0: ; %entry
10962; GFX11-WGP-NEXT:    s_clause 0x1
10963; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
10964; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
10965; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
10966; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10967; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10968; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10969; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
10970; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
10971; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
10972; GFX11-WGP-NEXT:    s_endpgm
10973;
10974; GFX11-CU-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
10975; GFX11-CU:       ; %bb.0: ; %entry
10976; GFX11-CU-NEXT:    s_clause 0x1
10977; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
10978; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
10979; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
10980; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10981; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10982; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10983; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
10984; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
10985; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
10986; GFX11-CU-NEXT:    s_endpgm
10987    i32 addrspace(1)* %out, i32 %in, i32 %old) {
10988entry:
10989  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
10990  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release monotonic
10991  %val0 = extractvalue { i32, i1 } %val, 0
10992  store i32 %val0, i32 addrspace(1)* %out, align 4
10993  ret void
10994}
10995
10996define amdgpu_kernel void @global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg(
10997; GFX6-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
10998; GFX6:       ; %bb.0: ; %entry
10999; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
11000; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11001; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
11002; GFX6-NEXT:    s_mov_b32 s2, -1
11003; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
11004; GFX6-NEXT:    v_mov_b32_e32 v0, s6
11005; GFX6-NEXT:    v_mov_b32_e32 v1, s7
11006; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
11007; GFX6-NEXT:    s_waitcnt vmcnt(0)
11008; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
11009; GFX6-NEXT:    s_endpgm
11010;
11011; GFX7-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
11012; GFX7:       ; %bb.0: ; %entry
11013; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11014; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11015; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11016; GFX7-NEXT:    s_add_u32 s4, s0, 16
11017; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11018; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11019; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11020; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11021; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11022; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11023; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11024; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11025; GFX7-NEXT:    s_waitcnt vmcnt(0)
11026; GFX7-NEXT:    flat_store_dword v[0:1], v2
11027; GFX7-NEXT:    s_endpgm
11028;
11029; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
11030; GFX10-WGP:       ; %bb.0: ; %entry
11031; GFX10-WGP-NEXT:    s_clause 0x1
11032; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11033; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11034; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
11035; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11036; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11037; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11038; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11039; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11040; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
11041; GFX10-WGP-NEXT:    s_endpgm
11042;
11043; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
11044; GFX10-CU:       ; %bb.0: ; %entry
11045; GFX10-CU-NEXT:    s_clause 0x1
11046; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11047; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11048; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
11049; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11050; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11051; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11052; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11053; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11054; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
11055; GFX10-CU-NEXT:    s_endpgm
11056;
11057; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
11058; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11059; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
11060; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
11061; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
11062; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
11063; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11064; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11065; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11066; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
11067; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11068; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
11069; SKIP-CACHE-INV-NEXT:    s_endpgm
11070;
11071; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
11072; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11073; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11074; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11075; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11076; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11077; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11078; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11079; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11080; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
11081; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11082;
11083; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
11084; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11085; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11086; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11087; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11088; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11089; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11090; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11091; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11092; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
11093; GFX90A-TGSPLIT-NEXT:    s_endpgm
11094;
11095; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
11096; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11097; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
11098; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
11099; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11100; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11101; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11102; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
11103; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11104; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
11105; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11106;
11107; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
11108; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11109; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
11110; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
11111; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11112; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11113; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11114; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
11115; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11116; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
11117; GFX940-TGSPLIT-NEXT:    s_endpgm
11118;
11119; GFX11-WGP-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
11120; GFX11-WGP:       ; %bb.0: ; %entry
11121; GFX11-WGP-NEXT:    s_clause 0x1
11122; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
11123; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
11124; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
11125; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11126; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
11127; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
11128; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
11129; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11130; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
11131; GFX11-WGP-NEXT:    s_endpgm
11132;
11133; GFX11-CU-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
11134; GFX11-CU:       ; %bb.0: ; %entry
11135; GFX11-CU-NEXT:    s_clause 0x1
11136; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
11137; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
11138; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
11139; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11140; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
11141; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
11142; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
11143; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11144; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
11145; GFX11-CU-NEXT:    s_endpgm
11146    i32 addrspace(1)* %out, i32 %in, i32 %old) {
11147entry:
11148  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
11149  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel monotonic
11150  %val0 = extractvalue { i32, i1 } %val, 0
11151  store i32 %val0, i32 addrspace(1)* %out, align 4
11152  ret void
11153}
11154
11155define amdgpu_kernel void @global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg(
11156; GFX6-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11157; GFX6:       ; %bb.0: ; %entry
11158; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
11159; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11160; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
11161; GFX6-NEXT:    s_mov_b32 s2, -1
11162; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
11163; GFX6-NEXT:    v_mov_b32_e32 v0, s6
11164; GFX6-NEXT:    v_mov_b32_e32 v1, s7
11165; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
11166; GFX6-NEXT:    s_waitcnt vmcnt(0)
11167; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
11168; GFX6-NEXT:    s_endpgm
11169;
11170; GFX7-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11171; GFX7:       ; %bb.0: ; %entry
11172; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11173; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11174; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11175; GFX7-NEXT:    s_add_u32 s4, s0, 16
11176; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11177; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11178; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11179; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11180; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11181; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11182; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11183; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11184; GFX7-NEXT:    s_waitcnt vmcnt(0)
11185; GFX7-NEXT:    flat_store_dword v[0:1], v2
11186; GFX7-NEXT:    s_endpgm
11187;
11188; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11189; GFX10-WGP:       ; %bb.0: ; %entry
11190; GFX10-WGP-NEXT:    s_clause 0x1
11191; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11192; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11193; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
11194; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11195; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11196; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11197; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11198; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11199; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
11200; GFX10-WGP-NEXT:    s_endpgm
11201;
11202; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11203; GFX10-CU:       ; %bb.0: ; %entry
11204; GFX10-CU-NEXT:    s_clause 0x1
11205; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11206; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11207; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
11208; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11209; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11210; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11211; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11212; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11213; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
11214; GFX10-CU-NEXT:    s_endpgm
11215;
11216; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11217; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11218; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
11219; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
11220; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
11221; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
11222; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11223; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11224; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11225; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
11226; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11227; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
11228; SKIP-CACHE-INV-NEXT:    s_endpgm
11229;
11230; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11231; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11232; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11233; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11234; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11235; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11236; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11237; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11238; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11239; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
11240; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11241;
11242; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11243; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11244; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11245; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11246; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11247; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11248; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11249; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11250; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11251; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
11252; GFX90A-TGSPLIT-NEXT:    s_endpgm
11253;
11254; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11255; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11256; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
11257; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
11258; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11259; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11260; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11261; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
11262; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11263; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
11264; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11265;
11266; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11267; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11268; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
11269; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
11270; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11271; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11272; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11273; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
11274; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11275; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
11276; GFX940-TGSPLIT-NEXT:    s_endpgm
11277;
11278; GFX11-WGP-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11279; GFX11-WGP:       ; %bb.0: ; %entry
11280; GFX11-WGP-NEXT:    s_clause 0x1
11281; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
11282; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
11283; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
11284; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11285; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
11286; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
11287; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
11288; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11289; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
11290; GFX11-WGP-NEXT:    s_endpgm
11291;
11292; GFX11-CU-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
11293; GFX11-CU:       ; %bb.0: ; %entry
11294; GFX11-CU-NEXT:    s_clause 0x1
11295; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
11296; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
11297; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
11298; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11299; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
11300; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
11301; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
11302; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11303; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
11304; GFX11-CU-NEXT:    s_endpgm
11305    i32 addrspace(1)* %out, i32 %in, i32 %old) {
11306entry:
11307  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
11308  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst monotonic
11309  %val0 = extractvalue { i32, i1 } %val, 0
11310  store i32 %val0, i32 addrspace(1)* %out, align 4
11311  ret void
11312}
11313
11314define amdgpu_kernel void @global_singlethread_one_as_monotonic_acquire_ret_cmpxchg(
11315; GFX6-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11316; GFX6:       ; %bb.0: ; %entry
11317; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
11318; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11319; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
11320; GFX6-NEXT:    s_mov_b32 s2, -1
11321; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
11322; GFX6-NEXT:    v_mov_b32_e32 v0, s6
11323; GFX6-NEXT:    v_mov_b32_e32 v1, s7
11324; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
11325; GFX6-NEXT:    s_waitcnt vmcnt(0)
11326; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
11327; GFX6-NEXT:    s_endpgm
11328;
11329; GFX7-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11330; GFX7:       ; %bb.0: ; %entry
11331; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11332; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11333; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11334; GFX7-NEXT:    s_add_u32 s4, s0, 16
11335; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11336; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11337; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11338; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11339; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11340; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11341; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11342; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11343; GFX7-NEXT:    s_waitcnt vmcnt(0)
11344; GFX7-NEXT:    flat_store_dword v[0:1], v2
11345; GFX7-NEXT:    s_endpgm
11346;
11347; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11348; GFX10-WGP:       ; %bb.0: ; %entry
11349; GFX10-WGP-NEXT:    s_clause 0x1
11350; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11351; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11352; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
11353; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11354; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11355; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11356; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11357; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11358; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
11359; GFX10-WGP-NEXT:    s_endpgm
11360;
11361; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11362; GFX10-CU:       ; %bb.0: ; %entry
11363; GFX10-CU-NEXT:    s_clause 0x1
11364; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11365; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11366; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
11367; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11368; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11369; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11370; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11371; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11372; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
11373; GFX10-CU-NEXT:    s_endpgm
11374;
11375; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11376; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11377; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
11378; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
11379; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
11380; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
11381; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11382; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11383; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11384; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
11385; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11386; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
11387; SKIP-CACHE-INV-NEXT:    s_endpgm
11388;
11389; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11390; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11391; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11392; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11393; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11394; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11395; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11396; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11397; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11398; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
11399; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11400;
11401; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11402; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11403; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11404; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11405; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11406; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11407; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11408; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11409; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11410; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
11411; GFX90A-TGSPLIT-NEXT:    s_endpgm
11412;
11413; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11414; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11415; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
11416; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
11417; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11418; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11419; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11420; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
11421; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11422; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
11423; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11424;
11425; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11426; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11427; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
11428; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
11429; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11430; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11431; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11432; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
11433; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11434; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
11435; GFX940-TGSPLIT-NEXT:    s_endpgm
11436;
11437; GFX11-WGP-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11438; GFX11-WGP:       ; %bb.0: ; %entry
11439; GFX11-WGP-NEXT:    s_clause 0x1
11440; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
11441; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
11442; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
11443; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11444; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
11445; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
11446; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
11447; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11448; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
11449; GFX11-WGP-NEXT:    s_endpgm
11450;
11451; GFX11-CU-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
11452; GFX11-CU:       ; %bb.0: ; %entry
11453; GFX11-CU-NEXT:    s_clause 0x1
11454; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
11455; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
11456; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
11457; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11458; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
11459; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
11460; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
11461; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11462; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
11463; GFX11-CU-NEXT:    s_endpgm
11464    i32 addrspace(1)* %out, i32 %in, i32 %old) {
11465entry:
11466  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
11467  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic acquire
11468  %val0 = extractvalue { i32, i1 } %val, 0
11469  store i32 %val0, i32 addrspace(1)* %out, align 4
11470  ret void
11471}
11472
11473define amdgpu_kernel void @global_singlethread_one_as_acquire_acquire_ret_cmpxchg(
11474; GFX6-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11475; GFX6:       ; %bb.0: ; %entry
11476; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
11477; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11478; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
11479; GFX6-NEXT:    s_mov_b32 s2, -1
11480; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
11481; GFX6-NEXT:    v_mov_b32_e32 v0, s6
11482; GFX6-NEXT:    v_mov_b32_e32 v1, s7
11483; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
11484; GFX6-NEXT:    s_waitcnt vmcnt(0)
11485; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
11486; GFX6-NEXT:    s_endpgm
11487;
11488; GFX7-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11489; GFX7:       ; %bb.0: ; %entry
11490; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11491; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11492; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11493; GFX7-NEXT:    s_add_u32 s4, s0, 16
11494; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11495; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11496; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11497; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11498; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11499; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11500; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11501; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11502; GFX7-NEXT:    s_waitcnt vmcnt(0)
11503; GFX7-NEXT:    flat_store_dword v[0:1], v2
11504; GFX7-NEXT:    s_endpgm
11505;
11506; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11507; GFX10-WGP:       ; %bb.0: ; %entry
11508; GFX10-WGP-NEXT:    s_clause 0x1
11509; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11510; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11511; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
11512; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11513; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11514; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11515; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11516; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11517; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
11518; GFX10-WGP-NEXT:    s_endpgm
11519;
11520; GFX10-CU-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11521; GFX10-CU:       ; %bb.0: ; %entry
11522; GFX10-CU-NEXT:    s_clause 0x1
11523; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11524; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11525; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
11526; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11527; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11528; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11529; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11530; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11531; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
11532; GFX10-CU-NEXT:    s_endpgm
11533;
11534; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11535; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11536; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
11537; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
11538; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
11539; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
11540; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11541; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11542; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11543; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
11544; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11545; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
11546; SKIP-CACHE-INV-NEXT:    s_endpgm
11547;
11548; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11549; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11550; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11551; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11552; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11553; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11554; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11555; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11556; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11557; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
11558; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11559;
11560; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11561; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11562; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11563; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11564; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11565; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11566; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11567; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11568; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11569; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
11570; GFX90A-TGSPLIT-NEXT:    s_endpgm
11571;
11572; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11573; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11574; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
11575; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
11576; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11577; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11578; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11579; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
11580; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11581; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
11582; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11583;
11584; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11585; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11586; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
11587; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
11588; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11589; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11590; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11591; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
11592; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11593; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
11594; GFX940-TGSPLIT-NEXT:    s_endpgm
11595;
11596; GFX11-WGP-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11597; GFX11-WGP:       ; %bb.0: ; %entry
11598; GFX11-WGP-NEXT:    s_clause 0x1
11599; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
11600; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
11601; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
11602; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11603; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
11604; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
11605; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
11606; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11607; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
11608; GFX11-WGP-NEXT:    s_endpgm
11609;
11610; GFX11-CU-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
11611; GFX11-CU:       ; %bb.0: ; %entry
11612; GFX11-CU-NEXT:    s_clause 0x1
11613; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
11614; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
11615; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
11616; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11617; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
11618; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
11619; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
11620; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11621; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
11622; GFX11-CU-NEXT:    s_endpgm
11623    i32 addrspace(1)* %out, i32 %in, i32 %old) {
11624entry:
11625  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
11626  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire acquire
11627  %val0 = extractvalue { i32, i1 } %val, 0
11628  store i32 %val0, i32 addrspace(1)* %out, align 4
11629  ret void
11630}
11631
11632define amdgpu_kernel void @global_singlethread_one_as_release_acquire_ret_cmpxchg(
11633; GFX6-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
11634; GFX6:       ; %bb.0: ; %entry
11635; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
11636; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11637; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
11638; GFX6-NEXT:    s_mov_b32 s2, -1
11639; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
11640; GFX6-NEXT:    v_mov_b32_e32 v0, s6
11641; GFX6-NEXT:    v_mov_b32_e32 v1, s7
11642; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
11643; GFX6-NEXT:    s_waitcnt vmcnt(0)
11644; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
11645; GFX6-NEXT:    s_endpgm
11646;
11647; GFX7-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
11648; GFX7:       ; %bb.0: ; %entry
11649; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11650; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11651; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11652; GFX7-NEXT:    s_add_u32 s4, s0, 16
11653; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11654; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11655; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11656; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11657; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11658; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11659; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11660; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11661; GFX7-NEXT:    s_waitcnt vmcnt(0)
11662; GFX7-NEXT:    flat_store_dword v[0:1], v2
11663; GFX7-NEXT:    s_endpgm
11664;
11665; GFX10-WGP-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
11666; GFX10-WGP:       ; %bb.0: ; %entry
11667; GFX10-WGP-NEXT:    s_clause 0x1
11668; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11669; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11670; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
11671; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11672; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11673; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11674; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11675; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11676; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
11677; GFX10-WGP-NEXT:    s_endpgm
11678;
11679; GFX10-CU-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
11680; GFX10-CU:       ; %bb.0: ; %entry
11681; GFX10-CU-NEXT:    s_clause 0x1
11682; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11683; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11684; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
11685; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11686; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11687; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11688; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11689; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11690; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
11691; GFX10-CU-NEXT:    s_endpgm
11692;
11693; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
11694; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11695; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
11696; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
11697; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
11698; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
11699; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11700; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11701; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11702; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
11703; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11704; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
11705; SKIP-CACHE-INV-NEXT:    s_endpgm
11706;
11707; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
11708; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11709; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11710; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11711; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11712; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11713; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11714; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11715; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11716; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
11717; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11718;
11719; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
11720; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11721; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11722; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11723; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11724; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11725; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11726; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11727; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11728; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
11729; GFX90A-TGSPLIT-NEXT:    s_endpgm
11730;
11731; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
11732; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11733; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
11734; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
11735; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11736; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11737; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11738; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
11739; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11740; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
11741; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11742;
11743; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
11744; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11745; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
11746; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
11747; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11748; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11749; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11750; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
11751; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11752; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
11753; GFX940-TGSPLIT-NEXT:    s_endpgm
11754;
11755; GFX11-WGP-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
11756; GFX11-WGP:       ; %bb.0: ; %entry
11757; GFX11-WGP-NEXT:    s_clause 0x1
11758; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
11759; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
11760; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
11761; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11762; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
11763; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
11764; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
11765; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11766; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
11767; GFX11-WGP-NEXT:    s_endpgm
11768;
11769; GFX11-CU-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
11770; GFX11-CU:       ; %bb.0: ; %entry
11771; GFX11-CU-NEXT:    s_clause 0x1
11772; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
11773; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
11774; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
11775; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11776; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
11777; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
11778; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
11779; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11780; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
11781; GFX11-CU-NEXT:    s_endpgm
11782    i32 addrspace(1)* %out, i32 %in, i32 %old) {
11783entry:
11784  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
11785  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release acquire
11786  %val0 = extractvalue { i32, i1 } %val, 0
11787  store i32 %val0, i32 addrspace(1)* %out, align 4
11788  ret void
11789}
11790
11791define amdgpu_kernel void @global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg(
11792; GFX6-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11793; GFX6:       ; %bb.0: ; %entry
11794; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
11795; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11796; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
11797; GFX6-NEXT:    s_mov_b32 s2, -1
11798; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
11799; GFX6-NEXT:    v_mov_b32_e32 v0, s6
11800; GFX6-NEXT:    v_mov_b32_e32 v1, s7
11801; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
11802; GFX6-NEXT:    s_waitcnt vmcnt(0)
11803; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
11804; GFX6-NEXT:    s_endpgm
11805;
11806; GFX7-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11807; GFX7:       ; %bb.0: ; %entry
11808; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11809; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11810; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11811; GFX7-NEXT:    s_add_u32 s4, s0, 16
11812; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11813; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11814; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11815; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11816; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11817; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11818; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11819; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11820; GFX7-NEXT:    s_waitcnt vmcnt(0)
11821; GFX7-NEXT:    flat_store_dword v[0:1], v2
11822; GFX7-NEXT:    s_endpgm
11823;
11824; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11825; GFX10-WGP:       ; %bb.0: ; %entry
11826; GFX10-WGP-NEXT:    s_clause 0x1
11827; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11828; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11829; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
11830; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11831; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11832; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11833; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11834; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11835; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
11836; GFX10-WGP-NEXT:    s_endpgm
11837;
11838; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11839; GFX10-CU:       ; %bb.0: ; %entry
11840; GFX10-CU-NEXT:    s_clause 0x1
11841; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11842; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11843; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
11844; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11845; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11846; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11847; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11848; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11849; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
11850; GFX10-CU-NEXT:    s_endpgm
11851;
11852; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11853; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11854; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
11855; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
11856; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
11857; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
11858; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11859; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11860; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11861; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
11862; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11863; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
11864; SKIP-CACHE-INV-NEXT:    s_endpgm
11865;
11866; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11867; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11868; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11869; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11870; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11871; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11872; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11873; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11874; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11875; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
11876; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11877;
11878; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11879; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11880; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11881; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11882; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11883; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11884; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11885; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11886; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11887; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
11888; GFX90A-TGSPLIT-NEXT:    s_endpgm
11889;
11890; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11891; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11892; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
11893; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
11894; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11895; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11896; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11897; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
11898; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11899; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
11900; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11901;
11902; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11903; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11904; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
11905; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
11906; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
11907; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11908; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11909; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
11910; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11911; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
11912; GFX940-TGSPLIT-NEXT:    s_endpgm
11913;
11914; GFX11-WGP-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11915; GFX11-WGP:       ; %bb.0: ; %entry
11916; GFX11-WGP-NEXT:    s_clause 0x1
11917; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
11918; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
11919; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
11920; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11921; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
11922; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
11923; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
11924; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11925; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
11926; GFX11-WGP-NEXT:    s_endpgm
11927;
11928; GFX11-CU-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
11929; GFX11-CU:       ; %bb.0: ; %entry
11930; GFX11-CU-NEXT:    s_clause 0x1
11931; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
11932; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
11933; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
11934; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11935; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
11936; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
11937; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
11938; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11939; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
11940; GFX11-CU-NEXT:    s_endpgm
11941    i32 addrspace(1)* %out, i32 %in, i32 %old) {
11942entry:
11943  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
11944  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel acquire
11945  %val0 = extractvalue { i32, i1 } %val, 0
11946  store i32 %val0, i32 addrspace(1)* %out, align 4
11947  ret void
11948}
11949
11950define amdgpu_kernel void @global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg(
11951; GFX6-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
11952; GFX6:       ; %bb.0: ; %entry
11953; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
11954; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11955; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
11956; GFX6-NEXT:    s_mov_b32 s2, -1
11957; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
11958; GFX6-NEXT:    v_mov_b32_e32 v0, s6
11959; GFX6-NEXT:    v_mov_b32_e32 v1, s7
11960; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
11961; GFX6-NEXT:    s_waitcnt vmcnt(0)
11962; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
11963; GFX6-NEXT:    s_endpgm
11964;
11965; GFX7-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
11966; GFX7:       ; %bb.0: ; %entry
11967; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11968; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11969; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11970; GFX7-NEXT:    s_add_u32 s4, s0, 16
11971; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11972; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11973; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11974; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11975; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11976; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11977; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11978; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11979; GFX7-NEXT:    s_waitcnt vmcnt(0)
11980; GFX7-NEXT:    flat_store_dword v[0:1], v2
11981; GFX7-NEXT:    s_endpgm
11982;
11983; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
11984; GFX10-WGP:       ; %bb.0: ; %entry
11985; GFX10-WGP-NEXT:    s_clause 0x1
11986; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
11987; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
11988; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
11989; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11990; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11991; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11992; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
11993; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11994; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
11995; GFX10-WGP-NEXT:    s_endpgm
11996;
11997; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
11998; GFX10-CU:       ; %bb.0: ; %entry
11999; GFX10-CU-NEXT:    s_clause 0x1
12000; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12001; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12002; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
12003; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12004; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12005; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12006; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12007; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12008; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
12009; GFX10-CU-NEXT:    s_endpgm
12010;
12011; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
12012; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12013; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
12014; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
12015; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
12016; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
12017; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12018; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12019; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12020; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
12021; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12022; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
12023; SKIP-CACHE-INV-NEXT:    s_endpgm
12024;
12025; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
12026; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12027; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12028; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12029; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12030; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12031; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12032; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12033; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12034; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
12035; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12036;
12037; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
12038; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12039; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12040; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12041; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12042; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12043; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12044; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12045; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12046; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
12047; GFX90A-TGSPLIT-NEXT:    s_endpgm
12048;
12049; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
12050; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12051; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
12052; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
12053; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12054; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12055; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12056; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
12057; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12058; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
12059; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12060;
12061; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
12062; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12063; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
12064; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
12065; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12066; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12067; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12068; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
12069; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12070; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
12071; GFX940-TGSPLIT-NEXT:    s_endpgm
12072;
12073; GFX11-WGP-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
12074; GFX11-WGP:       ; %bb.0: ; %entry
12075; GFX11-WGP-NEXT:    s_clause 0x1
12076; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
12077; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
12078; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
12079; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12080; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
12081; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
12082; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
12083; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12084; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
12085; GFX11-WGP-NEXT:    s_endpgm
12086;
12087; GFX11-CU-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
12088; GFX11-CU:       ; %bb.0: ; %entry
12089; GFX11-CU-NEXT:    s_clause 0x1
12090; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
12091; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
12092; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
12093; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12094; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
12095; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
12096; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
12097; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
12098; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
12099; GFX11-CU-NEXT:    s_endpgm
12100    i32 addrspace(1)* %out, i32 %in, i32 %old) {
12101entry:
12102  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
12103  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst acquire
12104  %val0 = extractvalue { i32, i1 } %val, 0
12105  store i32 %val0, i32 addrspace(1)* %out, align 4
12106  ret void
12107}
12108
12109define amdgpu_kernel void @global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg(
12110; GFX6-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12111; GFX6:       ; %bb.0: ; %entry
12112; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
12113; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12114; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
12115; GFX6-NEXT:    s_mov_b32 s2, -1
12116; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
12117; GFX6-NEXT:    v_mov_b32_e32 v0, s6
12118; GFX6-NEXT:    v_mov_b32_e32 v1, s7
12119; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
12120; GFX6-NEXT:    s_waitcnt vmcnt(0)
12121; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
12122; GFX6-NEXT:    s_endpgm
12123;
12124; GFX7-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12125; GFX7:       ; %bb.0: ; %entry
12126; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12127; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12128; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12129; GFX7-NEXT:    s_add_u32 s4, s0, 16
12130; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12131; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12132; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12133; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12134; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12135; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12136; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12137; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12138; GFX7-NEXT:    s_waitcnt vmcnt(0)
12139; GFX7-NEXT:    flat_store_dword v[0:1], v2
12140; GFX7-NEXT:    s_endpgm
12141;
12142; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12143; GFX10-WGP:       ; %bb.0: ; %entry
12144; GFX10-WGP-NEXT:    s_clause 0x1
12145; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12146; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12147; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
12148; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12149; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12150; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12151; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12152; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12153; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
12154; GFX10-WGP-NEXT:    s_endpgm
12155;
12156; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12157; GFX10-CU:       ; %bb.0: ; %entry
12158; GFX10-CU-NEXT:    s_clause 0x1
12159; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12160; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12161; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
12162; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12163; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12164; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12165; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12166; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12167; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
12168; GFX10-CU-NEXT:    s_endpgm
12169;
12170; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12171; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12172; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
12173; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
12174; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
12175; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
12176; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12177; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12178; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12179; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
12180; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12181; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
12182; SKIP-CACHE-INV-NEXT:    s_endpgm
12183;
12184; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12185; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12186; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12187; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12188; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12189; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12190; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12191; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12192; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12193; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
12194; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12195;
12196; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12197; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12198; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12199; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12200; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12201; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12202; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12203; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12204; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12205; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
12206; GFX90A-TGSPLIT-NEXT:    s_endpgm
12207;
12208; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12209; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12210; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
12211; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
12212; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12213; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12214; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12215; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
12216; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12217; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
12218; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12219;
12220; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12221; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12222; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
12223; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
12224; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12225; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12226; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12227; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
12228; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12229; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
12230; GFX940-TGSPLIT-NEXT:    s_endpgm
12231;
12232; GFX11-WGP-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12233; GFX11-WGP:       ; %bb.0: ; %entry
12234; GFX11-WGP-NEXT:    s_clause 0x1
12235; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
12236; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
12237; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
12238; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12239; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
12240; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
12241; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
12242; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12243; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
12244; GFX11-WGP-NEXT:    s_endpgm
12245;
12246; GFX11-CU-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
12247; GFX11-CU:       ; %bb.0: ; %entry
12248; GFX11-CU-NEXT:    s_clause 0x1
12249; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
12250; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
12251; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
12252; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12253; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
12254; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
12255; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
12256; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
12257; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
12258; GFX11-CU-NEXT:    s_endpgm
12259    i32 addrspace(1)* %out, i32 %in, i32 %old) {
12260entry:
12261  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
12262  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic seq_cst
12263  %val0 = extractvalue { i32, i1 } %val, 0
12264  store i32 %val0, i32 addrspace(1)* %out, align 4
12265  ret void
12266}
12267
12268define amdgpu_kernel void @global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg(
12269; GFX6-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12270; GFX6:       ; %bb.0: ; %entry
12271; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
12272; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12273; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
12274; GFX6-NEXT:    s_mov_b32 s2, -1
12275; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
12276; GFX6-NEXT:    v_mov_b32_e32 v0, s6
12277; GFX6-NEXT:    v_mov_b32_e32 v1, s7
12278; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
12279; GFX6-NEXT:    s_waitcnt vmcnt(0)
12280; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
12281; GFX6-NEXT:    s_endpgm
12282;
12283; GFX7-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12284; GFX7:       ; %bb.0: ; %entry
12285; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12286; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12287; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12288; GFX7-NEXT:    s_add_u32 s4, s0, 16
12289; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12290; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12291; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12292; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12293; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12294; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12295; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12296; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12297; GFX7-NEXT:    s_waitcnt vmcnt(0)
12298; GFX7-NEXT:    flat_store_dword v[0:1], v2
12299; GFX7-NEXT:    s_endpgm
12300;
12301; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12302; GFX10-WGP:       ; %bb.0: ; %entry
12303; GFX10-WGP-NEXT:    s_clause 0x1
12304; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12305; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12306; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
12307; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12308; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12309; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12310; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12311; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12312; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
12313; GFX10-WGP-NEXT:    s_endpgm
12314;
12315; GFX10-CU-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12316; GFX10-CU:       ; %bb.0: ; %entry
12317; GFX10-CU-NEXT:    s_clause 0x1
12318; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12319; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12320; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
12321; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12322; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12323; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12324; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12325; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12326; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
12327; GFX10-CU-NEXT:    s_endpgm
12328;
12329; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12330; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12331; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
12332; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
12333; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
12334; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
12335; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12336; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12337; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12338; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
12339; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12340; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
12341; SKIP-CACHE-INV-NEXT:    s_endpgm
12342;
12343; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12344; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12345; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12346; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12347; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12348; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12349; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12350; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12351; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12352; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
12353; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12354;
12355; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12356; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12357; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12358; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12359; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12360; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12361; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12362; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12363; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12364; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
12365; GFX90A-TGSPLIT-NEXT:    s_endpgm
12366;
12367; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12368; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12369; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
12370; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
12371; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12372; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12373; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12374; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
12375; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12376; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
12377; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12378;
12379; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12380; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12381; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
12382; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
12383; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12384; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12385; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12386; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
12387; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12388; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
12389; GFX940-TGSPLIT-NEXT:    s_endpgm
12390;
12391; GFX11-WGP-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12392; GFX11-WGP:       ; %bb.0: ; %entry
12393; GFX11-WGP-NEXT:    s_clause 0x1
12394; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
12395; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
12396; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
12397; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12398; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
12399; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
12400; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
12401; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12402; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
12403; GFX11-WGP-NEXT:    s_endpgm
12404;
12405; GFX11-CU-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
12406; GFX11-CU:       ; %bb.0: ; %entry
12407; GFX11-CU-NEXT:    s_clause 0x1
12408; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
12409; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
12410; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
12411; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12412; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
12413; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
12414; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
12415; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
12416; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
12417; GFX11-CU-NEXT:    s_endpgm
12418    i32 addrspace(1)* %out, i32 %in, i32 %old) {
12419entry:
12420  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
12421  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire seq_cst
12422  %val0 = extractvalue { i32, i1 } %val, 0
12423  store i32 %val0, i32 addrspace(1)* %out, align 4
12424  ret void
12425}
12426
12427define amdgpu_kernel void @global_singlethread_one_as_release_seq_cst_ret_cmpxchg(
12428; GFX6-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12429; GFX6:       ; %bb.0: ; %entry
12430; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
12431; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12432; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
12433; GFX6-NEXT:    s_mov_b32 s2, -1
12434; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
12435; GFX6-NEXT:    v_mov_b32_e32 v0, s6
12436; GFX6-NEXT:    v_mov_b32_e32 v1, s7
12437; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
12438; GFX6-NEXT:    s_waitcnt vmcnt(0)
12439; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
12440; GFX6-NEXT:    s_endpgm
12441;
12442; GFX7-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12443; GFX7:       ; %bb.0: ; %entry
12444; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12445; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12446; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12447; GFX7-NEXT:    s_add_u32 s4, s0, 16
12448; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12449; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12450; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12451; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12452; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12453; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12454; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12455; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12456; GFX7-NEXT:    s_waitcnt vmcnt(0)
12457; GFX7-NEXT:    flat_store_dword v[0:1], v2
12458; GFX7-NEXT:    s_endpgm
12459;
12460; GFX10-WGP-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12461; GFX10-WGP:       ; %bb.0: ; %entry
12462; GFX10-WGP-NEXT:    s_clause 0x1
12463; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12464; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12465; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
12466; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12467; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12468; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12469; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12470; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12471; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
12472; GFX10-WGP-NEXT:    s_endpgm
12473;
12474; GFX10-CU-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12475; GFX10-CU:       ; %bb.0: ; %entry
12476; GFX10-CU-NEXT:    s_clause 0x1
12477; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12478; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12479; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
12480; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12481; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12482; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12483; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12484; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12485; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
12486; GFX10-CU-NEXT:    s_endpgm
12487;
12488; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12489; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12490; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
12491; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
12492; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
12493; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
12494; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12495; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12496; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12497; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
12498; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12499; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
12500; SKIP-CACHE-INV-NEXT:    s_endpgm
12501;
12502; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12503; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12504; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12505; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12506; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12507; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12508; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12509; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12510; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12511; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
12512; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12513;
12514; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12515; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12516; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12517; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12518; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12519; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12520; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12521; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12522; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12523; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
12524; GFX90A-TGSPLIT-NEXT:    s_endpgm
12525;
12526; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12527; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12528; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
12529; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
12530; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12531; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12532; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12533; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
12534; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12535; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
12536; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12537;
12538; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12539; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12540; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
12541; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
12542; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12543; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12544; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12545; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
12546; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12547; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
12548; GFX940-TGSPLIT-NEXT:    s_endpgm
12549;
12550; GFX11-WGP-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12551; GFX11-WGP:       ; %bb.0: ; %entry
12552; GFX11-WGP-NEXT:    s_clause 0x1
12553; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
12554; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
12555; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
12556; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12557; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
12558; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
12559; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
12560; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12561; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
12562; GFX11-WGP-NEXT:    s_endpgm
12563;
12564; GFX11-CU-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
12565; GFX11-CU:       ; %bb.0: ; %entry
12566; GFX11-CU-NEXT:    s_clause 0x1
12567; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
12568; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
12569; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
12570; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12571; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
12572; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
12573; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
12574; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
12575; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
12576; GFX11-CU-NEXT:    s_endpgm
12577    i32 addrspace(1)* %out, i32 %in, i32 %old) {
12578entry:
12579  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
12580  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release seq_cst
12581  %val0 = extractvalue { i32, i1 } %val, 0
12582  store i32 %val0, i32 addrspace(1)* %out, align 4
12583  ret void
12584}
12585
12586define amdgpu_kernel void @global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg(
12587; GFX6-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12588; GFX6:       ; %bb.0: ; %entry
12589; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
12590; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12591; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
12592; GFX6-NEXT:    s_mov_b32 s2, -1
12593; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
12594; GFX6-NEXT:    v_mov_b32_e32 v0, s6
12595; GFX6-NEXT:    v_mov_b32_e32 v1, s7
12596; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
12597; GFX6-NEXT:    s_waitcnt vmcnt(0)
12598; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
12599; GFX6-NEXT:    s_endpgm
12600;
12601; GFX7-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12602; GFX7:       ; %bb.0: ; %entry
12603; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12604; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12605; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12606; GFX7-NEXT:    s_add_u32 s4, s0, 16
12607; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12608; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12609; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12610; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12611; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12612; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12613; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12614; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12615; GFX7-NEXT:    s_waitcnt vmcnt(0)
12616; GFX7-NEXT:    flat_store_dword v[0:1], v2
12617; GFX7-NEXT:    s_endpgm
12618;
12619; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12620; GFX10-WGP:       ; %bb.0: ; %entry
12621; GFX10-WGP-NEXT:    s_clause 0x1
12622; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12623; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12624; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
12625; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12626; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12627; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12628; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12629; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12630; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
12631; GFX10-WGP-NEXT:    s_endpgm
12632;
12633; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12634; GFX10-CU:       ; %bb.0: ; %entry
12635; GFX10-CU-NEXT:    s_clause 0x1
12636; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12637; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12638; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
12639; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12640; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12641; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12642; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12643; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12644; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
12645; GFX10-CU-NEXT:    s_endpgm
12646;
12647; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12648; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12649; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
12650; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
12651; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
12652; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
12653; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12654; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12655; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12656; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
12657; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12658; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
12659; SKIP-CACHE-INV-NEXT:    s_endpgm
12660;
12661; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12662; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12663; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12664; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12665; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12666; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12667; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12668; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12669; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12670; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
12671; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12672;
12673; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12674; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12675; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12676; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12677; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12678; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12679; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12680; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12681; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12682; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
12683; GFX90A-TGSPLIT-NEXT:    s_endpgm
12684;
12685; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12686; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12687; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
12688; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
12689; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12690; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12691; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12692; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
12693; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12694; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
12695; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12696;
12697; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12698; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12699; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
12700; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
12701; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12702; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12703; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12704; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
12705; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12706; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
12707; GFX940-TGSPLIT-NEXT:    s_endpgm
12708;
12709; GFX11-WGP-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12710; GFX11-WGP:       ; %bb.0: ; %entry
12711; GFX11-WGP-NEXT:    s_clause 0x1
12712; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
12713; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
12714; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
12715; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12716; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
12717; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
12718; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
12719; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12720; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
12721; GFX11-WGP-NEXT:    s_endpgm
12722;
12723; GFX11-CU-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
12724; GFX11-CU:       ; %bb.0: ; %entry
12725; GFX11-CU-NEXT:    s_clause 0x1
12726; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
12727; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
12728; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
12729; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12730; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
12731; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
12732; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
12733; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
12734; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
12735; GFX11-CU-NEXT:    s_endpgm
12736    i32 addrspace(1)* %out, i32 %in, i32 %old) {
12737entry:
12738  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
12739  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel seq_cst
12740  %val0 = extractvalue { i32, i1 } %val, 0
12741  store i32 %val0, i32 addrspace(1)* %out, align 4
12742  ret void
12743}
12744
12745define amdgpu_kernel void @global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg(
12746; GFX6-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12747; GFX6:       ; %bb.0: ; %entry
12748; GFX6-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x2
12749; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12750; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
12751; GFX6-NEXT:    s_mov_b32 s2, -1
12752; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
12753; GFX6-NEXT:    v_mov_b32_e32 v0, s6
12754; GFX6-NEXT:    v_mov_b32_e32 v1, s7
12755; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
12756; GFX6-NEXT:    s_waitcnt vmcnt(0)
12757; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
12758; GFX6-NEXT:    s_endpgm
12759;
12760; GFX7-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12761; GFX7:       ; %bb.0: ; %entry
12762; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12763; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12764; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12765; GFX7-NEXT:    s_add_u32 s4, s0, 16
12766; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12767; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12768; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12769; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12770; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12771; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12772; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12773; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12774; GFX7-NEXT:    s_waitcnt vmcnt(0)
12775; GFX7-NEXT:    flat_store_dword v[0:1], v2
12776; GFX7-NEXT:    s_endpgm
12777;
12778; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12779; GFX10-WGP:       ; %bb.0: ; %entry
12780; GFX10-WGP-NEXT:    s_clause 0x1
12781; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12782; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12783; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
12784; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12785; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12786; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12787; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12788; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12789; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
12790; GFX10-WGP-NEXT:    s_endpgm
12791;
12792; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12793; GFX10-CU:       ; %bb.0: ; %entry
12794; GFX10-CU-NEXT:    s_clause 0x1
12795; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12796; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12797; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
12798; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12799; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12800; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12801; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12802; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12803; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
12804; GFX10-CU-NEXT:    s_endpgm
12805;
12806; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12807; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12808; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2
12809; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
12810; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
12811; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
12812; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12813; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12814; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12815; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
12816; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12817; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
12818; SKIP-CACHE-INV-NEXT:    s_endpgm
12819;
12820; GFX90A-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12821; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12822; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12823; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12824; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12825; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12826; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12827; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12828; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12829; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
12830; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12831;
12832; GFX90A-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12833; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12834; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
12835; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
12836; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12837; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12838; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12839; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
12840; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12841; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[2:3]
12842; GFX90A-TGSPLIT-NEXT:    s_endpgm
12843;
12844; GFX940-NOTTGSPLIT-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12845; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12846; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
12847; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
12848; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12849; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12850; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12851; GFX940-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
12852; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12853; GFX940-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
12854; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12855;
12856; GFX940-TGSPLIT-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12857; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12858; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
12859; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
12860; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
12861; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12862; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12863; GFX940-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 sc0
12864; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12865; GFX940-TGSPLIT-NEXT:    global_store_dword v2, v0, s[4:5]
12866; GFX940-TGSPLIT-NEXT:    s_endpgm
12867;
12868; GFX11-WGP-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12869; GFX11-WGP:       ; %bb.0: ; %entry
12870; GFX11-WGP-NEXT:    s_clause 0x1
12871; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
12872; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
12873; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, 0
12874; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12875; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
12876; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
12877; GFX11-WGP-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
12878; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12879; GFX11-WGP-NEXT:    global_store_b32 v2, v0, s[0:1]
12880; GFX11-WGP-NEXT:    s_endpgm
12881;
12882; GFX11-CU-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
12883; GFX11-CU:       ; %bb.0: ; %entry
12884; GFX11-CU-NEXT:    s_clause 0x1
12885; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
12886; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
12887; GFX11-CU-NEXT:    v_mov_b32_e32 v2, 0
12888; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12889; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
12890; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
12891; GFX11-CU-NEXT:    global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
12892; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
12893; GFX11-CU-NEXT:    global_store_b32 v2, v0, s[0:1]
12894; GFX11-CU-NEXT:    s_endpgm
12895    i32 addrspace(1)* %out, i32 %in, i32 %old) {
12896entry:
12897  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
12898  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst seq_cst
12899  %val0 = extractvalue { i32, i1 } %val, 0
12900  store i32 %val0, i32 addrspace(1)* %out, align 4
12901  ret void
12902}
12903
12904