1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
3; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
4; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
5; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
6; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-NOTTGSPLIT %s
7; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-TGSPLIT %s
8; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-NOTTGSPLIT %s
9; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
10
11define amdgpu_kernel void @flat_singlethread_unordered_load(
12; GFX7-LABEL: flat_singlethread_unordered_load:
13; GFX7:       ; %bb.0: ; %entry
14; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
15; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
16; GFX7-NEXT:    v_mov_b32_e32 v0, s0
17; GFX7-NEXT:    v_mov_b32_e32 v1, s1
18; GFX7-NEXT:    flat_load_dword v2, v[0:1]
19; GFX7-NEXT:    v_mov_b32_e32 v0, s2
20; GFX7-NEXT:    v_mov_b32_e32 v1, s3
21; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
22; GFX7-NEXT:    flat_store_dword v[0:1], v2
23; GFX7-NEXT:    s_endpgm
24;
25; GFX10-WGP-LABEL: flat_singlethread_unordered_load:
26; GFX10-WGP:       ; %bb.0: ; %entry
27; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
28; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
29; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
30; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
31; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
32; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
33; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
34; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
35; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
36; GFX10-WGP-NEXT:    s_endpgm
37;
38; GFX10-CU-LABEL: flat_singlethread_unordered_load:
39; GFX10-CU:       ; %bb.0: ; %entry
40; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
41; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
42; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
43; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
44; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
45; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
46; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
47; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
48; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
49; GFX10-CU-NEXT:    s_endpgm
50;
51; SKIP-CACHE-INV-LABEL: flat_singlethread_unordered_load:
52; SKIP-CACHE-INV:       ; %bb.0: ; %entry
53; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
54; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
55; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
56; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
57; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
58; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
59; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
60; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
61; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
62; SKIP-CACHE-INV-NEXT:    s_endpgm
63;
64; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_unordered_load:
65; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
66; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
67; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
68; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
69; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
70; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
71; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
72; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
73; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
74; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
75; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
76;
77; GFX90A-TGSPLIT-LABEL: flat_singlethread_unordered_load:
78; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
79; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
80; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
81; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
82; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
83; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
84; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
85; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
86; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
87; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
88; GFX90A-TGSPLIT-NEXT:    s_endpgm
89;
90; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_unordered_load:
91; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
92; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
93; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
94; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
95; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
96; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
97; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
98; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
99; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
100; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
101; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
102;
103; GFX940-TGSPLIT-LABEL: flat_singlethread_unordered_load:
104; GFX940-TGSPLIT:       ; %bb.0: ; %entry
105; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
106; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
107; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
108; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
109; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
110; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
111; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
112; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
113; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
114; GFX940-TGSPLIT-NEXT:    s_endpgm
115    i32* %in, i32* %out) {
116entry:
117  %val = load atomic i32, i32* %in syncscope("singlethread") unordered, align 4
118  store i32 %val, i32* %out
119  ret void
120}
121
122define amdgpu_kernel void @flat_singlethread_monotonic_load(
123; GFX7-LABEL: flat_singlethread_monotonic_load:
124; GFX7:       ; %bb.0: ; %entry
125; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
126; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
127; GFX7-NEXT:    v_mov_b32_e32 v0, s0
128; GFX7-NEXT:    v_mov_b32_e32 v1, s1
129; GFX7-NEXT:    flat_load_dword v2, v[0:1]
130; GFX7-NEXT:    v_mov_b32_e32 v0, s2
131; GFX7-NEXT:    v_mov_b32_e32 v1, s3
132; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
133; GFX7-NEXT:    flat_store_dword v[0:1], v2
134; GFX7-NEXT:    s_endpgm
135;
136; GFX10-WGP-LABEL: flat_singlethread_monotonic_load:
137; GFX10-WGP:       ; %bb.0: ; %entry
138; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
139; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
140; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
141; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
142; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
143; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
144; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
145; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
146; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
147; GFX10-WGP-NEXT:    s_endpgm
148;
149; GFX10-CU-LABEL: flat_singlethread_monotonic_load:
150; GFX10-CU:       ; %bb.0: ; %entry
151; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
152; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
153; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
154; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
155; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
156; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
157; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
158; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
159; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
160; GFX10-CU-NEXT:    s_endpgm
161;
162; SKIP-CACHE-INV-LABEL: flat_singlethread_monotonic_load:
163; SKIP-CACHE-INV:       ; %bb.0: ; %entry
164; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
165; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
166; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
167; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
168; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
169; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
170; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
171; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
172; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
173; SKIP-CACHE-INV-NEXT:    s_endpgm
174;
175; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_load:
176; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
177; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
178; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
179; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
180; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
181; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
182; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
183; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
184; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
185; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
186; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
187;
188; GFX90A-TGSPLIT-LABEL: flat_singlethread_monotonic_load:
189; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
190; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
191; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
192; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
193; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
194; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
195; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
196; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
197; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
198; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
199; GFX90A-TGSPLIT-NEXT:    s_endpgm
200;
201; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_load:
202; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
203; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
204; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
205; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
206; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
207; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
208; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
209; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
210; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
211; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
212; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
213;
214; GFX940-TGSPLIT-LABEL: flat_singlethread_monotonic_load:
215; GFX940-TGSPLIT:       ; %bb.0: ; %entry
216; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
217; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
218; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
219; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
220; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
221; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
222; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
223; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
224; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
225; GFX940-TGSPLIT-NEXT:    s_endpgm
226    i32* %in, i32* %out) {
227entry:
228  %val = load atomic i32, i32* %in syncscope("singlethread") monotonic, align 4
229  store i32 %val, i32* %out
230  ret void
231}
232
233define amdgpu_kernel void @flat_singlethread_acquire_load(
234; GFX7-LABEL: flat_singlethread_acquire_load:
235; GFX7:       ; %bb.0: ; %entry
236; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
237; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
238; GFX7-NEXT:    v_mov_b32_e32 v0, s0
239; GFX7-NEXT:    v_mov_b32_e32 v1, s1
240; GFX7-NEXT:    flat_load_dword v2, v[0:1]
241; GFX7-NEXT:    v_mov_b32_e32 v0, s2
242; GFX7-NEXT:    v_mov_b32_e32 v1, s3
243; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
244; GFX7-NEXT:    flat_store_dword v[0:1], v2
245; GFX7-NEXT:    s_endpgm
246;
247; GFX10-WGP-LABEL: flat_singlethread_acquire_load:
248; GFX10-WGP:       ; %bb.0: ; %entry
249; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
250; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
251; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
252; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
253; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
254; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
255; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
256; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
257; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
258; GFX10-WGP-NEXT:    s_endpgm
259;
260; GFX10-CU-LABEL: flat_singlethread_acquire_load:
261; GFX10-CU:       ; %bb.0: ; %entry
262; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
263; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
264; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
265; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
266; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
267; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
268; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
269; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
270; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
271; GFX10-CU-NEXT:    s_endpgm
272;
273; SKIP-CACHE-INV-LABEL: flat_singlethread_acquire_load:
274; SKIP-CACHE-INV:       ; %bb.0: ; %entry
275; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
276; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
277; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
278; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
279; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
280; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
281; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
282; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
283; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
284; SKIP-CACHE-INV-NEXT:    s_endpgm
285;
286; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acquire_load:
287; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
288; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
289; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
290; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
291; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
292; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
293; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
294; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
295; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
296; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
297; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
298;
299; GFX90A-TGSPLIT-LABEL: flat_singlethread_acquire_load:
300; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
301; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
302; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
303; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
304; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
305; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
306; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
307; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
308; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
309; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
310; GFX90A-TGSPLIT-NEXT:    s_endpgm
311;
312; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acquire_load:
313; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
314; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
315; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
316; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
317; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
318; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
319; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
320; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
321; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
322; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
323; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
324;
325; GFX940-TGSPLIT-LABEL: flat_singlethread_acquire_load:
326; GFX940-TGSPLIT:       ; %bb.0: ; %entry
327; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
328; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
329; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
330; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
331; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
332; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
333; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
334; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
335; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
336; GFX940-TGSPLIT-NEXT:    s_endpgm
337    i32* %in, i32* %out) {
338entry:
339  %val = load atomic i32, i32* %in syncscope("singlethread") acquire, align 4
340  store i32 %val, i32* %out
341  ret void
342}
343
344define amdgpu_kernel void @flat_singlethread_seq_cst_load(
345; GFX7-LABEL: flat_singlethread_seq_cst_load:
346; GFX7:       ; %bb.0: ; %entry
347; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
348; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
349; GFX7-NEXT:    v_mov_b32_e32 v0, s0
350; GFX7-NEXT:    v_mov_b32_e32 v1, s1
351; GFX7-NEXT:    flat_load_dword v2, v[0:1]
352; GFX7-NEXT:    v_mov_b32_e32 v0, s2
353; GFX7-NEXT:    v_mov_b32_e32 v1, s3
354; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
355; GFX7-NEXT:    flat_store_dword v[0:1], v2
356; GFX7-NEXT:    s_endpgm
357;
358; GFX10-WGP-LABEL: flat_singlethread_seq_cst_load:
359; GFX10-WGP:       ; %bb.0: ; %entry
360; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
361; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
362; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
363; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
364; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
365; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
366; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
367; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
368; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
369; GFX10-WGP-NEXT:    s_endpgm
370;
371; GFX10-CU-LABEL: flat_singlethread_seq_cst_load:
372; GFX10-CU:       ; %bb.0: ; %entry
373; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
374; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
375; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
376; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
377; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
378; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
379; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
380; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
381; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
382; GFX10-CU-NEXT:    s_endpgm
383;
384; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_load:
385; SKIP-CACHE-INV:       ; %bb.0: ; %entry
386; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
387; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
388; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
389; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
390; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
391; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
392; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
393; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
394; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
395; SKIP-CACHE-INV-NEXT:    s_endpgm
396;
397; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_load:
398; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
399; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
400; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
401; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
402; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
403; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
404; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
405; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
406; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
407; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
408; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
409;
410; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_load:
411; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
412; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
413; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
414; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
415; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
416; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
417; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
418; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
419; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
420; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
421; GFX90A-TGSPLIT-NEXT:    s_endpgm
422;
423; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_load:
424; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
425; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
426; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
427; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
428; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
429; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
430; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
431; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
432; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
433; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
434; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
435;
436; GFX940-TGSPLIT-LABEL: flat_singlethread_seq_cst_load:
437; GFX940-TGSPLIT:       ; %bb.0: ; %entry
438; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
439; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
440; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
441; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
442; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
443; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
444; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
445; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
446; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
447; GFX940-TGSPLIT-NEXT:    s_endpgm
448    i32* %in, i32* %out) {
449entry:
450  %val = load atomic i32, i32* %in syncscope("singlethread") seq_cst, align 4
451  store i32 %val, i32* %out
452  ret void
453}
454
455define amdgpu_kernel void @flat_singlethread_unordered_store(
456; GFX7-LABEL: flat_singlethread_unordered_store:
457; GFX7:       ; %bb.0: ; %entry
458; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
459; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
460; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
461; GFX7-NEXT:    v_mov_b32_e32 v0, s0
462; GFX7-NEXT:    v_mov_b32_e32 v1, s1
463; GFX7-NEXT:    v_mov_b32_e32 v2, s2
464; GFX7-NEXT:    flat_store_dword v[0:1], v2
465; GFX7-NEXT:    s_endpgm
466;
467; GFX10-WGP-LABEL: flat_singlethread_unordered_store:
468; GFX10-WGP:       ; %bb.0: ; %entry
469; GFX10-WGP-NEXT:    s_clause 0x1
470; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
471; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
472; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
473; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
474; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
475; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
476; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
477; GFX10-WGP-NEXT:    s_endpgm
478;
479; GFX10-CU-LABEL: flat_singlethread_unordered_store:
480; GFX10-CU:       ; %bb.0: ; %entry
481; GFX10-CU-NEXT:    s_clause 0x1
482; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
483; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
484; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
485; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
486; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
487; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
488; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
489; GFX10-CU-NEXT:    s_endpgm
490;
491; SKIP-CACHE-INV-LABEL: flat_singlethread_unordered_store:
492; SKIP-CACHE-INV:       ; %bb.0: ; %entry
493; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
494; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
495; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
496; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
497; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
498; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
499; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
500; SKIP-CACHE-INV-NEXT:    s_endpgm
501;
502; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_unordered_store:
503; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
504; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
505; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
506; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
507; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
508; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
509; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
510; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
511;
512; GFX90A-TGSPLIT-LABEL: flat_singlethread_unordered_store:
513; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
514; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
515; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
516; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
517; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
518; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
519; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
520; GFX90A-TGSPLIT-NEXT:    s_endpgm
521;
522; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_unordered_store:
523; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
524; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
525; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
526; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
527; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
528; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
529; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
530; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
531;
532; GFX940-TGSPLIT-LABEL: flat_singlethread_unordered_store:
533; GFX940-TGSPLIT:       ; %bb.0: ; %entry
534; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
535; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
536; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
537; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
538; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
539; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
540; GFX940-TGSPLIT-NEXT:    s_endpgm
541    i32 %in, i32* %out) {
542entry:
543  store atomic i32 %in, i32* %out syncscope("singlethread") unordered, align 4
544  ret void
545}
546
547define amdgpu_kernel void @flat_singlethread_monotonic_store(
548; GFX7-LABEL: flat_singlethread_monotonic_store:
549; GFX7:       ; %bb.0: ; %entry
550; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
551; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
552; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
553; GFX7-NEXT:    v_mov_b32_e32 v0, s0
554; GFX7-NEXT:    v_mov_b32_e32 v1, s1
555; GFX7-NEXT:    v_mov_b32_e32 v2, s2
556; GFX7-NEXT:    flat_store_dword v[0:1], v2
557; GFX7-NEXT:    s_endpgm
558;
559; GFX10-WGP-LABEL: flat_singlethread_monotonic_store:
560; GFX10-WGP:       ; %bb.0: ; %entry
561; GFX10-WGP-NEXT:    s_clause 0x1
562; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
563; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
564; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
565; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
566; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
567; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
568; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
569; GFX10-WGP-NEXT:    s_endpgm
570;
571; GFX10-CU-LABEL: flat_singlethread_monotonic_store:
572; GFX10-CU:       ; %bb.0: ; %entry
573; GFX10-CU-NEXT:    s_clause 0x1
574; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
575; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
576; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
577; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
578; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
579; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
580; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
581; GFX10-CU-NEXT:    s_endpgm
582;
583; SKIP-CACHE-INV-LABEL: flat_singlethread_monotonic_store:
584; SKIP-CACHE-INV:       ; %bb.0: ; %entry
585; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
586; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
587; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
588; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
589; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
590; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
591; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
592; SKIP-CACHE-INV-NEXT:    s_endpgm
593;
594; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_store:
595; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
596; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
597; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
598; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
599; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
600; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
601; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
602; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
603;
604; GFX90A-TGSPLIT-LABEL: flat_singlethread_monotonic_store:
605; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
606; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
607; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
608; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
609; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
610; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
611; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
612; GFX90A-TGSPLIT-NEXT:    s_endpgm
613;
614; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_store:
615; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
616; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
617; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
618; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
619; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
620; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
621; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
622; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
623;
624; GFX940-TGSPLIT-LABEL: flat_singlethread_monotonic_store:
625; GFX940-TGSPLIT:       ; %bb.0: ; %entry
626; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
627; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
628; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
629; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
630; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
631; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
632; GFX940-TGSPLIT-NEXT:    s_endpgm
633    i32 %in, i32* %out) {
634entry:
635  store atomic i32 %in, i32* %out syncscope("singlethread") monotonic, align 4
636  ret void
637}
638
639define amdgpu_kernel void @flat_singlethread_release_store(
640; GFX7-LABEL: flat_singlethread_release_store:
641; GFX7:       ; %bb.0: ; %entry
642; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
643; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
644; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
645; GFX7-NEXT:    v_mov_b32_e32 v0, s0
646; GFX7-NEXT:    v_mov_b32_e32 v1, s1
647; GFX7-NEXT:    v_mov_b32_e32 v2, s2
648; GFX7-NEXT:    flat_store_dword v[0:1], v2
649; GFX7-NEXT:    s_endpgm
650;
651; GFX10-WGP-LABEL: flat_singlethread_release_store:
652; GFX10-WGP:       ; %bb.0: ; %entry
653; GFX10-WGP-NEXT:    s_clause 0x1
654; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
655; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
656; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
657; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
658; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
659; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
660; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
661; GFX10-WGP-NEXT:    s_endpgm
662;
663; GFX10-CU-LABEL: flat_singlethread_release_store:
664; GFX10-CU:       ; %bb.0: ; %entry
665; GFX10-CU-NEXT:    s_clause 0x1
666; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
667; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
668; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
669; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
670; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
671; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
672; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
673; GFX10-CU-NEXT:    s_endpgm
674;
675; SKIP-CACHE-INV-LABEL: flat_singlethread_release_store:
676; SKIP-CACHE-INV:       ; %bb.0: ; %entry
677; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
678; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
679; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
680; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
681; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
682; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
683; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
684; SKIP-CACHE-INV-NEXT:    s_endpgm
685;
686; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_release_store:
687; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
688; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
689; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
690; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
691; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
692; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
693; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
694; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
695;
696; GFX90A-TGSPLIT-LABEL: flat_singlethread_release_store:
697; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
698; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
699; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
700; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
701; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
702; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
703; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
704; GFX90A-TGSPLIT-NEXT:    s_endpgm
705;
706; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_release_store:
707; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
708; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
709; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
710; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
711; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
712; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
713; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
714; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
715;
716; GFX940-TGSPLIT-LABEL: flat_singlethread_release_store:
717; GFX940-TGSPLIT:       ; %bb.0: ; %entry
718; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
719; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
720; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
721; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
722; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
723; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
724; GFX940-TGSPLIT-NEXT:    s_endpgm
725    i32 %in, i32* %out) {
726entry:
727  store atomic i32 %in, i32* %out syncscope("singlethread") release, align 4
728  ret void
729}
730
731define amdgpu_kernel void @flat_singlethread_seq_cst_store(
732; GFX7-LABEL: flat_singlethread_seq_cst_store:
733; GFX7:       ; %bb.0: ; %entry
734; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
735; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
736; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
737; GFX7-NEXT:    v_mov_b32_e32 v0, s0
738; GFX7-NEXT:    v_mov_b32_e32 v1, s1
739; GFX7-NEXT:    v_mov_b32_e32 v2, s2
740; GFX7-NEXT:    flat_store_dword v[0:1], v2
741; GFX7-NEXT:    s_endpgm
742;
743; GFX10-WGP-LABEL: flat_singlethread_seq_cst_store:
744; GFX10-WGP:       ; %bb.0: ; %entry
745; GFX10-WGP-NEXT:    s_clause 0x1
746; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
747; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
748; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
749; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
750; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
751; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
752; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
753; GFX10-WGP-NEXT:    s_endpgm
754;
755; GFX10-CU-LABEL: flat_singlethread_seq_cst_store:
756; GFX10-CU:       ; %bb.0: ; %entry
757; GFX10-CU-NEXT:    s_clause 0x1
758; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
759; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
760; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
761; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
762; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
763; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
764; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
765; GFX10-CU-NEXT:    s_endpgm
766;
767; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_store:
768; SKIP-CACHE-INV:       ; %bb.0: ; %entry
769; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
770; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
771; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
772; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
773; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
774; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
775; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
776; SKIP-CACHE-INV-NEXT:    s_endpgm
777;
778; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_store:
779; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
780; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
781; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
782; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
783; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
784; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
785; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
786; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
787;
788; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_store:
789; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
790; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
791; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
792; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
793; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
794; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
795; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
796; GFX90A-TGSPLIT-NEXT:    s_endpgm
797;
798; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_store:
799; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
800; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
801; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
802; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
803; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
804; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
805; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
806; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
807;
808; GFX940-TGSPLIT-LABEL: flat_singlethread_seq_cst_store:
809; GFX940-TGSPLIT:       ; %bb.0: ; %entry
810; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
811; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
812; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
813; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
814; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
815; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
816; GFX940-TGSPLIT-NEXT:    s_endpgm
817    i32 %in, i32* %out) {
818entry:
819  store atomic i32 %in, i32* %out syncscope("singlethread") seq_cst, align 4
820  ret void
821}
822
823define amdgpu_kernel void @flat_singlethread_monotonic_atomicrmw(
824; GFX7-LABEL: flat_singlethread_monotonic_atomicrmw:
825; GFX7:       ; %bb.0: ; %entry
826; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
827; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
828; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
829; GFX7-NEXT:    v_mov_b32_e32 v0, s0
830; GFX7-NEXT:    v_mov_b32_e32 v1, s1
831; GFX7-NEXT:    v_mov_b32_e32 v2, s2
832; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
833; GFX7-NEXT:    s_endpgm
834;
835; GFX10-WGP-LABEL: flat_singlethread_monotonic_atomicrmw:
836; GFX10-WGP:       ; %bb.0: ; %entry
837; GFX10-WGP-NEXT:    s_clause 0x1
838; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
839; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
840; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
841; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
842; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
843; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
844; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
845; GFX10-WGP-NEXT:    s_endpgm
846;
847; GFX10-CU-LABEL: flat_singlethread_monotonic_atomicrmw:
848; GFX10-CU:       ; %bb.0: ; %entry
849; GFX10-CU-NEXT:    s_clause 0x1
850; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
851; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
852; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
853; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
854; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
855; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
856; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
857; GFX10-CU-NEXT:    s_endpgm
858;
859; SKIP-CACHE-INV-LABEL: flat_singlethread_monotonic_atomicrmw:
860; SKIP-CACHE-INV:       ; %bb.0: ; %entry
861; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
862; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
863; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
864; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
865; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
866; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
867; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
868; SKIP-CACHE-INV-NEXT:    s_endpgm
869;
870; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_atomicrmw:
871; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
872; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
873; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
874; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
875; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
876; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
877; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
878; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
879;
880; GFX90A-TGSPLIT-LABEL: flat_singlethread_monotonic_atomicrmw:
881; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
882; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
883; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
884; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
885; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
886; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
887; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
888; GFX90A-TGSPLIT-NEXT:    s_endpgm
889;
890; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_atomicrmw:
891; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
892; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
893; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
894; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
895; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
896; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
897; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
898; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
899;
900; GFX940-TGSPLIT-LABEL: flat_singlethread_monotonic_atomicrmw:
901; GFX940-TGSPLIT:       ; %bb.0: ; %entry
902; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
903; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
904; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
905; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
906; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
907; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
908; GFX940-TGSPLIT-NEXT:    s_endpgm
909    i32* %out, i32 %in) {
910entry:
911  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") monotonic
912  ret void
913}
914
915define amdgpu_kernel void @flat_singlethread_acquire_atomicrmw(
916; GFX7-LABEL: flat_singlethread_acquire_atomicrmw:
917; GFX7:       ; %bb.0: ; %entry
918; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
919; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
920; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
921; GFX7-NEXT:    v_mov_b32_e32 v0, s0
922; GFX7-NEXT:    v_mov_b32_e32 v1, s1
923; GFX7-NEXT:    v_mov_b32_e32 v2, s2
924; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
925; GFX7-NEXT:    s_endpgm
926;
927; GFX10-WGP-LABEL: flat_singlethread_acquire_atomicrmw:
928; GFX10-WGP:       ; %bb.0: ; %entry
929; GFX10-WGP-NEXT:    s_clause 0x1
930; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
931; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
932; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
933; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
934; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
935; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
936; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
937; GFX10-WGP-NEXT:    s_endpgm
938;
939; GFX10-CU-LABEL: flat_singlethread_acquire_atomicrmw:
940; GFX10-CU:       ; %bb.0: ; %entry
941; GFX10-CU-NEXT:    s_clause 0x1
942; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
943; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
944; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
945; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
946; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
947; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
948; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
949; GFX10-CU-NEXT:    s_endpgm
950;
951; SKIP-CACHE-INV-LABEL: flat_singlethread_acquire_atomicrmw:
952; SKIP-CACHE-INV:       ; %bb.0: ; %entry
953; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
954; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
955; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
956; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
957; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
958; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
959; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
960; SKIP-CACHE-INV-NEXT:    s_endpgm
961;
962; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acquire_atomicrmw:
963; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
964; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
965; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
966; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
967; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
968; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
969; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
970; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
971;
972; GFX90A-TGSPLIT-LABEL: flat_singlethread_acquire_atomicrmw:
973; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
974; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
975; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
976; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
977; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
978; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
979; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
980; GFX90A-TGSPLIT-NEXT:    s_endpgm
981;
982; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acquire_atomicrmw:
983; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
984; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
985; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
986; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
987; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
988; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
989; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
990; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
991;
992; GFX940-TGSPLIT-LABEL: flat_singlethread_acquire_atomicrmw:
993; GFX940-TGSPLIT:       ; %bb.0: ; %entry
994; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
995; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
996; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
997; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
998; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
999; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1000; GFX940-TGSPLIT-NEXT:    s_endpgm
1001    i32* %out, i32 %in) {
1002entry:
1003  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") acquire
1004  ret void
1005}
1006
1007define amdgpu_kernel void @flat_singlethread_release_atomicrmw(
1008; GFX7-LABEL: flat_singlethread_release_atomicrmw:
1009; GFX7:       ; %bb.0: ; %entry
1010; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1011; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1012; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1013; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1014; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1015; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1016; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1017; GFX7-NEXT:    s_endpgm
1018;
1019; GFX10-WGP-LABEL: flat_singlethread_release_atomicrmw:
1020; GFX10-WGP:       ; %bb.0: ; %entry
1021; GFX10-WGP-NEXT:    s_clause 0x1
1022; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1023; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1024; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1025; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1026; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1027; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1028; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1029; GFX10-WGP-NEXT:    s_endpgm
1030;
1031; GFX10-CU-LABEL: flat_singlethread_release_atomicrmw:
1032; GFX10-CU:       ; %bb.0: ; %entry
1033; GFX10-CU-NEXT:    s_clause 0x1
1034; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1035; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1036; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1037; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1038; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1039; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1040; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1041; GFX10-CU-NEXT:    s_endpgm
1042;
1043; SKIP-CACHE-INV-LABEL: flat_singlethread_release_atomicrmw:
1044; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1045; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1046; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1047; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1048; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1049; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1050; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1051; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1052; SKIP-CACHE-INV-NEXT:    s_endpgm
1053;
1054; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_release_atomicrmw:
1055; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1056; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1057; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1058; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1059; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1060; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1061; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1062; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1063;
1064; GFX90A-TGSPLIT-LABEL: flat_singlethread_release_atomicrmw:
1065; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1066; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1067; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1068; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1069; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1070; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1071; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1072; GFX90A-TGSPLIT-NEXT:    s_endpgm
1073;
1074; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_release_atomicrmw:
1075; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1076; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1077; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1078; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1079; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1080; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1081; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1082; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1083;
1084; GFX940-TGSPLIT-LABEL: flat_singlethread_release_atomicrmw:
1085; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1086; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1087; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1088; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1089; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1090; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1091; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1092; GFX940-TGSPLIT-NEXT:    s_endpgm
1093    i32* %out, i32 %in) {
1094entry:
1095  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") release
1096  ret void
1097}
1098
1099define amdgpu_kernel void @flat_singlethread_acq_rel_atomicrmw(
1100; GFX7-LABEL: flat_singlethread_acq_rel_atomicrmw:
1101; GFX7:       ; %bb.0: ; %entry
1102; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1103; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1104; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1105; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1106; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1107; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1108; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1109; GFX7-NEXT:    s_endpgm
1110;
1111; GFX10-WGP-LABEL: flat_singlethread_acq_rel_atomicrmw:
1112; GFX10-WGP:       ; %bb.0: ; %entry
1113; GFX10-WGP-NEXT:    s_clause 0x1
1114; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1115; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1116; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1117; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1118; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1119; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1120; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1121; GFX10-WGP-NEXT:    s_endpgm
1122;
1123; GFX10-CU-LABEL: flat_singlethread_acq_rel_atomicrmw:
1124; GFX10-CU:       ; %bb.0: ; %entry
1125; GFX10-CU-NEXT:    s_clause 0x1
1126; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1127; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1128; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1129; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1130; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1131; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1132; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1133; GFX10-CU-NEXT:    s_endpgm
1134;
1135; SKIP-CACHE-INV-LABEL: flat_singlethread_acq_rel_atomicrmw:
1136; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1137; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1138; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1139; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1140; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1141; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1142; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1143; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1144; SKIP-CACHE-INV-NEXT:    s_endpgm
1145;
1146; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_atomicrmw:
1147; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1148; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1149; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1150; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1151; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1152; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1153; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1154; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1155;
1156; GFX90A-TGSPLIT-LABEL: flat_singlethread_acq_rel_atomicrmw:
1157; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1158; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1159; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1160; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1161; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1162; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1163; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1164; GFX90A-TGSPLIT-NEXT:    s_endpgm
1165;
1166; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_atomicrmw:
1167; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1168; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1169; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1170; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1171; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1172; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1173; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1174; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1175;
1176; GFX940-TGSPLIT-LABEL: flat_singlethread_acq_rel_atomicrmw:
1177; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1178; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1179; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1180; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1181; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1182; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1183; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1184; GFX940-TGSPLIT-NEXT:    s_endpgm
1185    i32* %out, i32 %in) {
1186entry:
1187  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") acq_rel
1188  ret void
1189}
1190
1191define amdgpu_kernel void @flat_singlethread_seq_cst_atomicrmw(
1192; GFX7-LABEL: flat_singlethread_seq_cst_atomicrmw:
1193; GFX7:       ; %bb.0: ; %entry
1194; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1195; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1196; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1197; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1198; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1199; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1200; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1201; GFX7-NEXT:    s_endpgm
1202;
1203; GFX10-WGP-LABEL: flat_singlethread_seq_cst_atomicrmw:
1204; GFX10-WGP:       ; %bb.0: ; %entry
1205; GFX10-WGP-NEXT:    s_clause 0x1
1206; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1207; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1208; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1209; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1210; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1211; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1212; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1213; GFX10-WGP-NEXT:    s_endpgm
1214;
1215; GFX10-CU-LABEL: flat_singlethread_seq_cst_atomicrmw:
1216; GFX10-CU:       ; %bb.0: ; %entry
1217; GFX10-CU-NEXT:    s_clause 0x1
1218; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1219; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1220; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1221; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1222; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1223; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1224; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1225; GFX10-CU-NEXT:    s_endpgm
1226;
1227; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_atomicrmw:
1228; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1229; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1230; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1231; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1232; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1233; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1234; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1235; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1236; SKIP-CACHE-INV-NEXT:    s_endpgm
1237;
1238; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_atomicrmw:
1239; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1240; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1241; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1242; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1243; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1244; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1245; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1246; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1247;
1248; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_atomicrmw:
1249; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1250; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1251; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1252; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1253; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1254; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1255; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1256; GFX90A-TGSPLIT-NEXT:    s_endpgm
1257;
1258; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_atomicrmw:
1259; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1260; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1261; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1262; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1263; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1264; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1265; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1266; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1267;
1268; GFX940-TGSPLIT-LABEL: flat_singlethread_seq_cst_atomicrmw:
1269; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1270; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1271; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1272; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1273; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1274; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1275; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1276; GFX940-TGSPLIT-NEXT:    s_endpgm
1277    i32* %out, i32 %in) {
1278entry:
1279  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") seq_cst
1280  ret void
1281}
1282
1283define amdgpu_kernel void @flat_singlethread_acquire_ret_atomicrmw(
1284; GFX7-LABEL: flat_singlethread_acquire_ret_atomicrmw:
1285; GFX7:       ; %bb.0: ; %entry
1286; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1287; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1288; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1289; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1290; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1291; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1292; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1293; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1294; GFX7-NEXT:    flat_store_dword v[0:1], v2
1295; GFX7-NEXT:    s_endpgm
1296;
1297; GFX10-WGP-LABEL: flat_singlethread_acquire_ret_atomicrmw:
1298; GFX10-WGP:       ; %bb.0: ; %entry
1299; GFX10-WGP-NEXT:    s_clause 0x1
1300; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1301; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1302; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1303; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1304; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1305; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1306; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1307; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1308; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1309; GFX10-WGP-NEXT:    s_endpgm
1310;
1311; GFX10-CU-LABEL: flat_singlethread_acquire_ret_atomicrmw:
1312; GFX10-CU:       ; %bb.0: ; %entry
1313; GFX10-CU-NEXT:    s_clause 0x1
1314; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1315; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1316; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1317; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1318; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1319; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1320; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1321; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1322; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1323; GFX10-CU-NEXT:    s_endpgm
1324;
1325; SKIP-CACHE-INV-LABEL: flat_singlethread_acquire_ret_atomicrmw:
1326; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1327; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1328; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1329; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1330; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1331; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1332; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1333; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1334; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1335; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1336; SKIP-CACHE-INV-NEXT:    s_endpgm
1337;
1338; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acquire_ret_atomicrmw:
1339; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1340; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1341; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1342; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1343; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1344; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1345; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1346; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1347; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1348; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1349;
1350; GFX90A-TGSPLIT-LABEL: flat_singlethread_acquire_ret_atomicrmw:
1351; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1352; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1353; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1354; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1355; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1356; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1357; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1358; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1359; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1360; GFX90A-TGSPLIT-NEXT:    s_endpgm
1361;
1362; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acquire_ret_atomicrmw:
1363; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1364; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1365; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1366; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1367; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1368; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1369; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
1370; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1371; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1372; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1373;
1374; GFX940-TGSPLIT-LABEL: flat_singlethread_acquire_ret_atomicrmw:
1375; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1376; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1377; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1378; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1379; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1380; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1381; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
1382; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1383; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1384; GFX940-TGSPLIT-NEXT:    s_endpgm
1385    i32* %out, i32 %in) {
1386entry:
1387  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") acquire
1388  store i32 %val, i32* %out, align 4
1389  ret void
1390}
1391
1392define amdgpu_kernel void @flat_singlethread_acq_rel_ret_atomicrmw(
1393; GFX7-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
1394; GFX7:       ; %bb.0: ; %entry
1395; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1396; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1397; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1398; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1399; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1400; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1401; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1402; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1403; GFX7-NEXT:    flat_store_dword v[0:1], v2
1404; GFX7-NEXT:    s_endpgm
1405;
1406; GFX10-WGP-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
1407; GFX10-WGP:       ; %bb.0: ; %entry
1408; GFX10-WGP-NEXT:    s_clause 0x1
1409; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1410; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1411; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1412; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1413; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1414; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1415; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1416; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1417; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1418; GFX10-WGP-NEXT:    s_endpgm
1419;
1420; GFX10-CU-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
1421; GFX10-CU:       ; %bb.0: ; %entry
1422; GFX10-CU-NEXT:    s_clause 0x1
1423; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1424; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1425; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1426; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1427; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1428; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1429; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1430; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1431; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1432; GFX10-CU-NEXT:    s_endpgm
1433;
1434; SKIP-CACHE-INV-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
1435; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1436; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1437; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1438; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1439; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1440; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1441; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1442; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1443; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1444; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1445; SKIP-CACHE-INV-NEXT:    s_endpgm
1446;
1447; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
1448; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1449; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1450; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1451; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1452; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1453; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1454; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1455; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1456; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1457; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1458;
1459; GFX90A-TGSPLIT-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
1460; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1461; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1462; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1463; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1464; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1465; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1466; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1467; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1468; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1469; GFX90A-TGSPLIT-NEXT:    s_endpgm
1470;
1471; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
1472; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1473; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1474; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1475; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1476; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1477; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1478; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
1479; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1480; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1481; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1482;
1483; GFX940-TGSPLIT-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
1484; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1485; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1486; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1487; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1488; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1489; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1490; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
1491; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1492; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1493; GFX940-TGSPLIT-NEXT:    s_endpgm
1494    i32* %out, i32 %in) {
1495entry:
1496  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") acq_rel
1497  store i32 %val, i32* %out, align 4
1498  ret void
1499}
1500
1501define amdgpu_kernel void @flat_singlethread_seq_cst_ret_atomicrmw(
1502; GFX7-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
1503; GFX7:       ; %bb.0: ; %entry
1504; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1505; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1506; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1507; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1508; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1509; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1510; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1511; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1512; GFX7-NEXT:    flat_store_dword v[0:1], v2
1513; GFX7-NEXT:    s_endpgm
1514;
1515; GFX10-WGP-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
1516; GFX10-WGP:       ; %bb.0: ; %entry
1517; GFX10-WGP-NEXT:    s_clause 0x1
1518; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1519; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1520; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1521; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1522; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1523; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1524; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1525; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1526; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1527; GFX10-WGP-NEXT:    s_endpgm
1528;
1529; GFX10-CU-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
1530; GFX10-CU:       ; %bb.0: ; %entry
1531; GFX10-CU-NEXT:    s_clause 0x1
1532; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1533; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1534; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1535; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1536; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1537; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1538; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1539; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1540; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1541; GFX10-CU-NEXT:    s_endpgm
1542;
1543; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
1544; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1545; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1546; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1547; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1548; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1549; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1550; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1551; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1552; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1553; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1554; SKIP-CACHE-INV-NEXT:    s_endpgm
1555;
1556; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
1557; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1558; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1559; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1560; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1561; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1562; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1563; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1564; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1565; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1566; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1567;
1568; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
1569; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1570; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1571; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1572; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1573; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1574; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1575; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1576; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1577; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1578; GFX90A-TGSPLIT-NEXT:    s_endpgm
1579;
1580; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
1581; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1582; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1583; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1584; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1585; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1586; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1587; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
1588; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1589; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1590; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1591;
1592; GFX940-TGSPLIT-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
1593; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1594; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1595; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1596; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1597; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1598; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1599; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
1600; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1601; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1602; GFX940-TGSPLIT-NEXT:    s_endpgm
1603    i32* %out, i32 %in) {
1604entry:
1605  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") seq_cst
1606  store i32 %val, i32* %out, align 4
1607  ret void
1608}
1609
1610define amdgpu_kernel void @flat_singlethread_monotonic_monotonic_cmpxchg(
1611; GFX7-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
1612; GFX7:       ; %bb.0: ; %entry
1613; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1614; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1615; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1616; GFX7-NEXT:    s_add_u32 s0, s0, 16
1617; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1618; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1619; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1620; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1621; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1622; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1623; GFX7-NEXT:    s_endpgm
1624;
1625; GFX10-WGP-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
1626; GFX10-WGP:       ; %bb.0: ; %entry
1627; GFX10-WGP-NEXT:    s_clause 0x1
1628; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1629; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1630; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1631; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1632; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1633; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1634; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1635; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1636; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1637; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1638; GFX10-WGP-NEXT:    s_endpgm
1639;
1640; GFX10-CU-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
1641; GFX10-CU:       ; %bb.0: ; %entry
1642; GFX10-CU-NEXT:    s_clause 0x1
1643; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1644; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1645; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1646; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1647; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1648; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1649; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1650; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1651; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1652; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1653; GFX10-CU-NEXT:    s_endpgm
1654;
1655; SKIP-CACHE-INV-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
1656; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1657; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1658; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
1659; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1660; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1661; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1662; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1663; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1664; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1665; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1666; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1667; SKIP-CACHE-INV-NEXT:    s_endpgm
1668;
1669; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
1670; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1671; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1672; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1673; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1674; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1675; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1676; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1677; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1678;
1679; GFX90A-TGSPLIT-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
1680; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1681; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1682; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1683; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1684; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1685; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1686; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1687; GFX90A-TGSPLIT-NEXT:    s_endpgm
1688;
1689; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
1690; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1691; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1692; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
1693; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1694; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1695; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
1696; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1697; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1698;
1699; GFX940-TGSPLIT-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
1700; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1701; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1702; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
1703; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1704; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1705; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
1706; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1707; GFX940-TGSPLIT-NEXT:    s_endpgm
1708    i32* %out, i32 %in, i32 %old) {
1709entry:
1710  %gep = getelementptr i32, i32* %out, i32 4
1711  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic monotonic
1712  ret void
1713}
1714
1715define amdgpu_kernel void @flat_singlethread_acquire_monotonic_cmpxchg(
1716; GFX7-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
1717; GFX7:       ; %bb.0: ; %entry
1718; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1719; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1720; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1721; GFX7-NEXT:    s_add_u32 s0, s0, 16
1722; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1723; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1724; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1725; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1726; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1727; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1728; GFX7-NEXT:    s_endpgm
1729;
1730; GFX10-WGP-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
1731; GFX10-WGP:       ; %bb.0: ; %entry
1732; GFX10-WGP-NEXT:    s_clause 0x1
1733; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1734; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1735; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1736; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1737; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1738; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1739; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1740; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1741; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1742; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1743; GFX10-WGP-NEXT:    s_endpgm
1744;
1745; GFX10-CU-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
1746; GFX10-CU:       ; %bb.0: ; %entry
1747; GFX10-CU-NEXT:    s_clause 0x1
1748; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1749; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1750; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1751; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1752; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1753; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1754; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1755; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1756; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1757; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1758; GFX10-CU-NEXT:    s_endpgm
1759;
1760; SKIP-CACHE-INV-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
1761; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1762; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1763; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
1764; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1765; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1766; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1767; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1768; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1769; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1770; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1771; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1772; SKIP-CACHE-INV-NEXT:    s_endpgm
1773;
1774; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
1775; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1776; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1777; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1778; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1779; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1780; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1781; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1782; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1783;
1784; GFX90A-TGSPLIT-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
1785; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1786; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1787; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1788; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1789; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1790; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1791; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1792; GFX90A-TGSPLIT-NEXT:    s_endpgm
1793;
1794; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
1795; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1796; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1797; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
1798; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1799; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1800; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
1801; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1802; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1803;
1804; GFX940-TGSPLIT-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
1805; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1806; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1807; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
1808; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1809; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1810; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
1811; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1812; GFX940-TGSPLIT-NEXT:    s_endpgm
1813    i32* %out, i32 %in, i32 %old) {
1814entry:
1815  %gep = getelementptr i32, i32* %out, i32 4
1816  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acquire monotonic
1817  ret void
1818}
1819
1820define amdgpu_kernel void @flat_singlethread_release_monotonic_cmpxchg(
1821; GFX7-LABEL: flat_singlethread_release_monotonic_cmpxchg:
1822; GFX7:       ; %bb.0: ; %entry
1823; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1824; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1825; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1826; GFX7-NEXT:    s_add_u32 s0, s0, 16
1827; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1828; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1829; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1830; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1831; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1832; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1833; GFX7-NEXT:    s_endpgm
1834;
1835; GFX10-WGP-LABEL: flat_singlethread_release_monotonic_cmpxchg:
1836; GFX10-WGP:       ; %bb.0: ; %entry
1837; GFX10-WGP-NEXT:    s_clause 0x1
1838; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1839; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1840; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1841; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1842; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1843; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1844; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1845; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1846; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1847; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1848; GFX10-WGP-NEXT:    s_endpgm
1849;
1850; GFX10-CU-LABEL: flat_singlethread_release_monotonic_cmpxchg:
1851; GFX10-CU:       ; %bb.0: ; %entry
1852; GFX10-CU-NEXT:    s_clause 0x1
1853; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1854; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1855; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1856; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1857; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1858; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1859; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1860; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1861; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1862; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1863; GFX10-CU-NEXT:    s_endpgm
1864;
1865; SKIP-CACHE-INV-LABEL: flat_singlethread_release_monotonic_cmpxchg:
1866; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1867; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1868; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
1869; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1870; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1871; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1872; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1873; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1874; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1875; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1876; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1877; SKIP-CACHE-INV-NEXT:    s_endpgm
1878;
1879; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_release_monotonic_cmpxchg:
1880; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1881; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1882; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1883; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1884; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1885; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1886; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1887; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1888;
1889; GFX90A-TGSPLIT-LABEL: flat_singlethread_release_monotonic_cmpxchg:
1890; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1891; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1892; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1893; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1894; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1895; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1896; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1897; GFX90A-TGSPLIT-NEXT:    s_endpgm
1898;
1899; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_release_monotonic_cmpxchg:
1900; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1901; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1902; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
1903; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1904; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1905; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
1906; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1907; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1908;
1909; GFX940-TGSPLIT-LABEL: flat_singlethread_release_monotonic_cmpxchg:
1910; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1911; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1912; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
1913; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1914; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1915; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
1916; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1917; GFX940-TGSPLIT-NEXT:    s_endpgm
1918    i32* %out, i32 %in, i32 %old) {
1919entry:
1920  %gep = getelementptr i32, i32* %out, i32 4
1921  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") release monotonic
1922  ret void
1923}
1924
1925define amdgpu_kernel void @flat_singlethread_acq_rel_monotonic_cmpxchg(
1926; GFX7-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
1927; GFX7:       ; %bb.0: ; %entry
1928; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1929; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1930; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1931; GFX7-NEXT:    s_add_u32 s0, s0, 16
1932; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1933; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1934; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1935; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1936; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1937; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1938; GFX7-NEXT:    s_endpgm
1939;
1940; GFX10-WGP-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
1941; GFX10-WGP:       ; %bb.0: ; %entry
1942; GFX10-WGP-NEXT:    s_clause 0x1
1943; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1944; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1945; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1946; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1947; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1948; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1949; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1950; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1951; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1952; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1953; GFX10-WGP-NEXT:    s_endpgm
1954;
1955; GFX10-CU-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
1956; GFX10-CU:       ; %bb.0: ; %entry
1957; GFX10-CU-NEXT:    s_clause 0x1
1958; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1959; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1960; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1961; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1962; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1963; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1964; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1965; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1966; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1967; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1968; GFX10-CU-NEXT:    s_endpgm
1969;
1970; SKIP-CACHE-INV-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
1971; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1972; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1973; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
1974; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1975; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1976; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1977; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1978; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1979; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1980; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1981; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1982; SKIP-CACHE-INV-NEXT:    s_endpgm
1983;
1984; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
1985; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1986; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1987; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1988; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1989; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1990; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1991; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1992; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1993;
1994; GFX90A-TGSPLIT-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
1995; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1996; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1997; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1998; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1999; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2000; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2001; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2002; GFX90A-TGSPLIT-NEXT:    s_endpgm
2003;
2004; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
2005; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2006; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2007; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2008; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2009; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2010; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2011; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2012; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2013;
2014; GFX940-TGSPLIT-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
2015; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2016; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2017; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2018; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2019; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2020; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2021; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2022; GFX940-TGSPLIT-NEXT:    s_endpgm
2023    i32* %out, i32 %in, i32 %old) {
2024entry:
2025  %gep = getelementptr i32, i32* %out, i32 4
2026  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel monotonic
2027  ret void
2028}
2029
2030define amdgpu_kernel void @flat_singlethread_seq_cst_monotonic_cmpxchg(
2031; GFX7-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
2032; GFX7:       ; %bb.0: ; %entry
2033; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2034; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2035; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2036; GFX7-NEXT:    s_add_u32 s0, s0, 16
2037; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2038; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2039; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2040; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2041; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2042; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2043; GFX7-NEXT:    s_endpgm
2044;
2045; GFX10-WGP-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
2046; GFX10-WGP:       ; %bb.0: ; %entry
2047; GFX10-WGP-NEXT:    s_clause 0x1
2048; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2049; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2050; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2051; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2052; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2053; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2054; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2055; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2056; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2057; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2058; GFX10-WGP-NEXT:    s_endpgm
2059;
2060; GFX10-CU-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
2061; GFX10-CU:       ; %bb.0: ; %entry
2062; GFX10-CU-NEXT:    s_clause 0x1
2063; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2064; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2065; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2066; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2067; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2068; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2069; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2070; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2071; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2072; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2073; GFX10-CU-NEXT:    s_endpgm
2074;
2075; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
2076; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2077; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2078; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2079; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2080; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2081; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2082; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2083; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2084; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2085; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2086; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2087; SKIP-CACHE-INV-NEXT:    s_endpgm
2088;
2089; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
2090; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2091; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2092; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2093; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2094; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2095; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2096; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2097; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2098;
2099; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
2100; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2101; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2102; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2103; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2104; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2105; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2106; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2107; GFX90A-TGSPLIT-NEXT:    s_endpgm
2108;
2109; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
2110; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2111; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2112; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2113; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2114; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2115; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2116; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2117; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2118;
2119; GFX940-TGSPLIT-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
2120; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2121; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2122; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2123; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2124; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2125; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2126; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2127; GFX940-TGSPLIT-NEXT:    s_endpgm
2128    i32* %out, i32 %in, i32 %old) {
2129entry:
2130  %gep = getelementptr i32, i32* %out, i32 4
2131  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst monotonic
2132  ret void
2133}
2134
2135define amdgpu_kernel void @flat_singlethread_monotonic_acquire_cmpxchg(
2136; GFX7-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
2137; GFX7:       ; %bb.0: ; %entry
2138; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2139; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2140; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2141; GFX7-NEXT:    s_add_u32 s0, s0, 16
2142; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2143; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2144; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2145; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2146; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2147; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2148; GFX7-NEXT:    s_endpgm
2149;
2150; GFX10-WGP-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
2151; GFX10-WGP:       ; %bb.0: ; %entry
2152; GFX10-WGP-NEXT:    s_clause 0x1
2153; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2154; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2155; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2156; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2157; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2158; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2159; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2160; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2161; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2162; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2163; GFX10-WGP-NEXT:    s_endpgm
2164;
2165; GFX10-CU-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
2166; GFX10-CU:       ; %bb.0: ; %entry
2167; GFX10-CU-NEXT:    s_clause 0x1
2168; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2169; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2170; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2171; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2172; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2173; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2174; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2175; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2176; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2177; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2178; GFX10-CU-NEXT:    s_endpgm
2179;
2180; SKIP-CACHE-INV-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
2181; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2182; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2183; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2184; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2185; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2186; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2187; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2188; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2189; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2190; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2191; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2192; SKIP-CACHE-INV-NEXT:    s_endpgm
2193;
2194; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
2195; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2196; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2197; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2198; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2199; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2200; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2201; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2202; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2203;
2204; GFX90A-TGSPLIT-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
2205; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2206; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2207; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2208; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2209; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2210; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2211; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2212; GFX90A-TGSPLIT-NEXT:    s_endpgm
2213;
2214; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
2215; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2216; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2217; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2218; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2219; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2220; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2221; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2222; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2223;
2224; GFX940-TGSPLIT-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
2225; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2226; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2227; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2228; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2229; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2230; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2231; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2232; GFX940-TGSPLIT-NEXT:    s_endpgm
2233    i32* %out, i32 %in, i32 %old) {
2234entry:
2235  %gep = getelementptr i32, i32* %out, i32 4
2236  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic acquire
2237  ret void
2238}
2239
2240define amdgpu_kernel void @flat_singlethread_acquire_acquire_cmpxchg(
2241; GFX7-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
2242; GFX7:       ; %bb.0: ; %entry
2243; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2244; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2245; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2246; GFX7-NEXT:    s_add_u32 s0, s0, 16
2247; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2248; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2249; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2250; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2251; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2252; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2253; GFX7-NEXT:    s_endpgm
2254;
2255; GFX10-WGP-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
2256; GFX10-WGP:       ; %bb.0: ; %entry
2257; GFX10-WGP-NEXT:    s_clause 0x1
2258; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2259; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2260; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2261; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2262; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2263; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2264; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2265; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2266; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2267; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2268; GFX10-WGP-NEXT:    s_endpgm
2269;
2270; GFX10-CU-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
2271; GFX10-CU:       ; %bb.0: ; %entry
2272; GFX10-CU-NEXT:    s_clause 0x1
2273; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2274; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2275; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2276; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2277; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2278; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2279; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2280; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2281; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2282; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2283; GFX10-CU-NEXT:    s_endpgm
2284;
2285; SKIP-CACHE-INV-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
2286; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2287; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2288; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2289; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2290; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2291; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2292; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2293; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2294; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2295; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2296; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2297; SKIP-CACHE-INV-NEXT:    s_endpgm
2298;
2299; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
2300; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2301; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2302; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2303; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2304; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2305; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2306; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2307; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2308;
2309; GFX90A-TGSPLIT-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
2310; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2311; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2312; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2313; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2314; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2315; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2316; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2317; GFX90A-TGSPLIT-NEXT:    s_endpgm
2318;
2319; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
2320; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2321; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2322; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2323; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2324; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2325; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2326; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2327; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2328;
2329; GFX940-TGSPLIT-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
2330; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2331; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2332; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2333; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2334; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2335; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2336; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2337; GFX940-TGSPLIT-NEXT:    s_endpgm
2338    i32* %out, i32 %in, i32 %old) {
2339entry:
2340  %gep = getelementptr i32, i32* %out, i32 4
2341  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acquire acquire
2342  ret void
2343}
2344
2345define amdgpu_kernel void @flat_singlethread_release_acquire_cmpxchg(
2346; GFX7-LABEL: flat_singlethread_release_acquire_cmpxchg:
2347; GFX7:       ; %bb.0: ; %entry
2348; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2349; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2350; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2351; GFX7-NEXT:    s_add_u32 s0, s0, 16
2352; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2353; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2354; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2355; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2356; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2357; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2358; GFX7-NEXT:    s_endpgm
2359;
2360; GFX10-WGP-LABEL: flat_singlethread_release_acquire_cmpxchg:
2361; GFX10-WGP:       ; %bb.0: ; %entry
2362; GFX10-WGP-NEXT:    s_clause 0x1
2363; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2364; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2365; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2366; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2367; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2368; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2369; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2370; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2371; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2372; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2373; GFX10-WGP-NEXT:    s_endpgm
2374;
2375; GFX10-CU-LABEL: flat_singlethread_release_acquire_cmpxchg:
2376; GFX10-CU:       ; %bb.0: ; %entry
2377; GFX10-CU-NEXT:    s_clause 0x1
2378; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2379; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2380; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2381; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2382; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2383; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2384; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2385; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2386; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2387; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2388; GFX10-CU-NEXT:    s_endpgm
2389;
2390; SKIP-CACHE-INV-LABEL: flat_singlethread_release_acquire_cmpxchg:
2391; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2392; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2393; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2394; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2395; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2396; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2397; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2398; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2399; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2400; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2401; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2402; SKIP-CACHE-INV-NEXT:    s_endpgm
2403;
2404; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_release_acquire_cmpxchg:
2405; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2406; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2407; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2408; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2409; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2410; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2411; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2412; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2413;
2414; GFX90A-TGSPLIT-LABEL: flat_singlethread_release_acquire_cmpxchg:
2415; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2416; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2417; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2418; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2419; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2420; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2421; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2422; GFX90A-TGSPLIT-NEXT:    s_endpgm
2423;
2424; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_release_acquire_cmpxchg:
2425; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2426; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2427; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2428; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2429; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2430; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2431; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2432; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2433;
2434; GFX940-TGSPLIT-LABEL: flat_singlethread_release_acquire_cmpxchg:
2435; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2436; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2437; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2438; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2439; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2440; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2441; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2442; GFX940-TGSPLIT-NEXT:    s_endpgm
2443    i32* %out, i32 %in, i32 %old) {
2444entry:
2445  %gep = getelementptr i32, i32* %out, i32 4
2446  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") release acquire
2447  ret void
2448}
2449
2450define amdgpu_kernel void @flat_singlethread_acq_rel_acquire_cmpxchg(
2451; GFX7-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
2452; GFX7:       ; %bb.0: ; %entry
2453; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2454; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2455; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2456; GFX7-NEXT:    s_add_u32 s0, s0, 16
2457; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2458; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2459; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2460; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2461; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2462; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2463; GFX7-NEXT:    s_endpgm
2464;
2465; GFX10-WGP-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
2466; GFX10-WGP:       ; %bb.0: ; %entry
2467; GFX10-WGP-NEXT:    s_clause 0x1
2468; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2469; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2470; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2471; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2472; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2473; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2474; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2475; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2476; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2477; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2478; GFX10-WGP-NEXT:    s_endpgm
2479;
2480; GFX10-CU-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
2481; GFX10-CU:       ; %bb.0: ; %entry
2482; GFX10-CU-NEXT:    s_clause 0x1
2483; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2484; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2485; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2486; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2487; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2488; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2489; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2490; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2491; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2492; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2493; GFX10-CU-NEXT:    s_endpgm
2494;
2495; SKIP-CACHE-INV-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
2496; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2497; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2498; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2499; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2500; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2501; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2502; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2503; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2504; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2505; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2506; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2507; SKIP-CACHE-INV-NEXT:    s_endpgm
2508;
2509; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
2510; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2511; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2512; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2513; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2514; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2515; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2516; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2517; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2518;
2519; GFX90A-TGSPLIT-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
2520; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2521; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2522; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2523; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2524; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2525; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2526; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2527; GFX90A-TGSPLIT-NEXT:    s_endpgm
2528;
2529; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
2530; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2531; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2532; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2533; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2534; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2535; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2536; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2537; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2538;
2539; GFX940-TGSPLIT-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
2540; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2541; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2542; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2543; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2544; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2545; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2546; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2547; GFX940-TGSPLIT-NEXT:    s_endpgm
2548    i32* %out, i32 %in, i32 %old) {
2549entry:
2550  %gep = getelementptr i32, i32* %out, i32 4
2551  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel acquire
2552  ret void
2553}
2554
2555define amdgpu_kernel void @flat_singlethread_seq_cst_acquire_cmpxchg(
2556; GFX7-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
2557; GFX7:       ; %bb.0: ; %entry
2558; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2559; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2560; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2561; GFX7-NEXT:    s_add_u32 s0, s0, 16
2562; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2563; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2564; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2565; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2566; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2567; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2568; GFX7-NEXT:    s_endpgm
2569;
2570; GFX10-WGP-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
2571; GFX10-WGP:       ; %bb.0: ; %entry
2572; GFX10-WGP-NEXT:    s_clause 0x1
2573; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2574; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2575; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2576; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2577; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2578; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2579; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2580; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2581; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2582; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2583; GFX10-WGP-NEXT:    s_endpgm
2584;
2585; GFX10-CU-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
2586; GFX10-CU:       ; %bb.0: ; %entry
2587; GFX10-CU-NEXT:    s_clause 0x1
2588; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2589; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2590; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2591; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2592; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2593; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2594; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2595; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2596; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2597; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2598; GFX10-CU-NEXT:    s_endpgm
2599;
2600; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
2601; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2602; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2603; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2604; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2605; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2606; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2607; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2608; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2609; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2610; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2611; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2612; SKIP-CACHE-INV-NEXT:    s_endpgm
2613;
2614; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
2615; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2616; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2617; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2618; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2619; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2620; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2621; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2622; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2623;
2624; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
2625; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2626; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2627; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2628; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2629; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2630; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2631; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2632; GFX90A-TGSPLIT-NEXT:    s_endpgm
2633;
2634; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
2635; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2636; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2637; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2638; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2639; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2640; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2641; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2642; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2643;
2644; GFX940-TGSPLIT-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
2645; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2646; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2647; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2648; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2649; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2650; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2651; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2652; GFX940-TGSPLIT-NEXT:    s_endpgm
2653    i32* %out, i32 %in, i32 %old) {
2654entry:
2655  %gep = getelementptr i32, i32* %out, i32 4
2656  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst acquire
2657  ret void
2658}
2659
2660define amdgpu_kernel void @flat_singlethread_monotonic_seq_cst_cmpxchg(
2661; GFX7-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
2662; GFX7:       ; %bb.0: ; %entry
2663; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2664; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2665; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2666; GFX7-NEXT:    s_add_u32 s0, s0, 16
2667; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2668; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2669; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2670; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2671; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2672; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2673; GFX7-NEXT:    s_endpgm
2674;
2675; GFX10-WGP-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
2676; GFX10-WGP:       ; %bb.0: ; %entry
2677; GFX10-WGP-NEXT:    s_clause 0x1
2678; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2679; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2680; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2681; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2682; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2683; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2684; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2685; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2686; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2687; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2688; GFX10-WGP-NEXT:    s_endpgm
2689;
2690; GFX10-CU-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
2691; GFX10-CU:       ; %bb.0: ; %entry
2692; GFX10-CU-NEXT:    s_clause 0x1
2693; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2694; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2695; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2696; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2697; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2698; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2699; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2700; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2701; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2702; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2703; GFX10-CU-NEXT:    s_endpgm
2704;
2705; SKIP-CACHE-INV-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
2706; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2707; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2708; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2709; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2710; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2711; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2712; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2713; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2714; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2715; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2716; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2717; SKIP-CACHE-INV-NEXT:    s_endpgm
2718;
2719; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
2720; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2721; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2722; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2723; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2724; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2725; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2726; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2727; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2728;
2729; GFX90A-TGSPLIT-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
2730; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2731; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2732; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2733; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2734; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2735; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2736; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2737; GFX90A-TGSPLIT-NEXT:    s_endpgm
2738;
2739; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
2740; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2741; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2742; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2743; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2744; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2745; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2746; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2747; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2748;
2749; GFX940-TGSPLIT-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
2750; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2751; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2752; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2753; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2754; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2755; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2756; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2757; GFX940-TGSPLIT-NEXT:    s_endpgm
2758    i32* %out, i32 %in, i32 %old) {
2759entry:
2760  %gep = getelementptr i32, i32* %out, i32 4
2761  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic seq_cst
2762  ret void
2763}
2764
2765define amdgpu_kernel void @flat_singlethread_acquire_seq_cst_cmpxchg(
2766; GFX7-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
2767; GFX7:       ; %bb.0: ; %entry
2768; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2769; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2770; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2771; GFX7-NEXT:    s_add_u32 s0, s0, 16
2772; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2773; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2774; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2775; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2776; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2777; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2778; GFX7-NEXT:    s_endpgm
2779;
2780; GFX10-WGP-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
2781; GFX10-WGP:       ; %bb.0: ; %entry
2782; GFX10-WGP-NEXT:    s_clause 0x1
2783; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2784; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2785; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2786; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2787; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2788; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2789; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2790; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2791; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2792; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2793; GFX10-WGP-NEXT:    s_endpgm
2794;
2795; GFX10-CU-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
2796; GFX10-CU:       ; %bb.0: ; %entry
2797; GFX10-CU-NEXT:    s_clause 0x1
2798; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2799; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2800; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2801; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2802; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2803; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2804; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2805; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2806; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2807; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2808; GFX10-CU-NEXT:    s_endpgm
2809;
2810; SKIP-CACHE-INV-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
2811; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2812; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2813; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2814; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2815; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2816; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2817; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2818; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2819; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2820; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2821; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2822; SKIP-CACHE-INV-NEXT:    s_endpgm
2823;
2824; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
2825; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2826; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2827; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2828; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2829; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2830; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2831; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2832; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2833;
2834; GFX90A-TGSPLIT-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
2835; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2836; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2837; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2838; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2839; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2840; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2841; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2842; GFX90A-TGSPLIT-NEXT:    s_endpgm
2843;
2844; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
2845; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2846; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2847; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2848; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2849; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2850; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2851; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2852; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2853;
2854; GFX940-TGSPLIT-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
2855; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2856; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2857; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2858; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2859; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2860; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2861; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2862; GFX940-TGSPLIT-NEXT:    s_endpgm
2863    i32* %out, i32 %in, i32 %old) {
2864entry:
2865  %gep = getelementptr i32, i32* %out, i32 4
2866  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acquire seq_cst
2867  ret void
2868}
2869
2870define amdgpu_kernel void @flat_singlethread_release_seq_cst_cmpxchg(
2871; GFX7-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
2872; GFX7:       ; %bb.0: ; %entry
2873; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2874; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2875; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2876; GFX7-NEXT:    s_add_u32 s0, s0, 16
2877; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2878; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2879; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2880; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2881; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2882; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2883; GFX7-NEXT:    s_endpgm
2884;
2885; GFX10-WGP-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
2886; GFX10-WGP:       ; %bb.0: ; %entry
2887; GFX10-WGP-NEXT:    s_clause 0x1
2888; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2889; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2890; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2891; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2892; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2893; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2894; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2895; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2896; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2897; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2898; GFX10-WGP-NEXT:    s_endpgm
2899;
2900; GFX10-CU-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
2901; GFX10-CU:       ; %bb.0: ; %entry
2902; GFX10-CU-NEXT:    s_clause 0x1
2903; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2904; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2905; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2906; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2907; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2908; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2909; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2910; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2911; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2912; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2913; GFX10-CU-NEXT:    s_endpgm
2914;
2915; SKIP-CACHE-INV-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
2916; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2917; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2918; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2919; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2920; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2921; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2922; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2923; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2924; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2925; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2926; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2927; SKIP-CACHE-INV-NEXT:    s_endpgm
2928;
2929; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
2930; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2931; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2932; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2933; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2934; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2935; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2936; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2937; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2938;
2939; GFX90A-TGSPLIT-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
2940; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2941; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2942; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2943; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2944; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2945; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2946; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2947; GFX90A-TGSPLIT-NEXT:    s_endpgm
2948;
2949; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
2950; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2951; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2952; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2953; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2954; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2955; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2956; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2957; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2958;
2959; GFX940-TGSPLIT-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
2960; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2961; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2962; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2963; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2964; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2965; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2966; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2967; GFX940-TGSPLIT-NEXT:    s_endpgm
2968    i32* %out, i32 %in, i32 %old) {
2969entry:
2970  %gep = getelementptr i32, i32* %out, i32 4
2971  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") release seq_cst
2972  ret void
2973}
2974
2975define amdgpu_kernel void @flat_singlethread_acq_rel_seq_cst_cmpxchg(
2976; GFX7-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
2977; GFX7:       ; %bb.0: ; %entry
2978; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2979; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2980; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2981; GFX7-NEXT:    s_add_u32 s0, s0, 16
2982; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2983; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2984; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2985; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2986; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2987; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2988; GFX7-NEXT:    s_endpgm
2989;
2990; GFX10-WGP-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
2991; GFX10-WGP:       ; %bb.0: ; %entry
2992; GFX10-WGP-NEXT:    s_clause 0x1
2993; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2994; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2995; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2996; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2997; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2998; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2999; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3000; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3001; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3002; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3003; GFX10-WGP-NEXT:    s_endpgm
3004;
3005; GFX10-CU-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
3006; GFX10-CU:       ; %bb.0: ; %entry
3007; GFX10-CU-NEXT:    s_clause 0x1
3008; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3009; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3010; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3011; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3012; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3013; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3014; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3015; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3016; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3017; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3018; GFX10-CU-NEXT:    s_endpgm
3019;
3020; SKIP-CACHE-INV-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
3021; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3022; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3023; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3024; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3025; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3026; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3027; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3028; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3029; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3030; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3031; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3032; SKIP-CACHE-INV-NEXT:    s_endpgm
3033;
3034; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
3035; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3036; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3037; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3038; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3039; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3040; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3041; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3042; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3043;
3044; GFX90A-TGSPLIT-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
3045; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3046; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3047; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3048; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3049; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3050; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3051; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3052; GFX90A-TGSPLIT-NEXT:    s_endpgm
3053;
3054; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
3055; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3056; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3057; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3058; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3059; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3060; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3061; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3062; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3063;
3064; GFX940-TGSPLIT-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
3065; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3066; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3067; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3068; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3069; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3070; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3071; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3072; GFX940-TGSPLIT-NEXT:    s_endpgm
3073    i32* %out, i32 %in, i32 %old) {
3074entry:
3075  %gep = getelementptr i32, i32* %out, i32 4
3076  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel seq_cst
3077  ret void
3078}
3079
3080define amdgpu_kernel void @flat_singlethread_seq_cst_seq_cst_cmpxchg(
3081; GFX7-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
3082; GFX7:       ; %bb.0: ; %entry
3083; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3084; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3085; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3086; GFX7-NEXT:    s_add_u32 s0, s0, 16
3087; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3088; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3089; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3090; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3091; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3092; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3093; GFX7-NEXT:    s_endpgm
3094;
3095; GFX10-WGP-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
3096; GFX10-WGP:       ; %bb.0: ; %entry
3097; GFX10-WGP-NEXT:    s_clause 0x1
3098; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3099; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3100; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3101; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3102; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3103; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3104; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3105; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3106; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3107; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3108; GFX10-WGP-NEXT:    s_endpgm
3109;
3110; GFX10-CU-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
3111; GFX10-CU:       ; %bb.0: ; %entry
3112; GFX10-CU-NEXT:    s_clause 0x1
3113; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3114; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3115; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3116; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3117; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3118; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3119; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3120; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3121; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3122; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3123; GFX10-CU-NEXT:    s_endpgm
3124;
3125; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
3126; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3127; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3128; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3129; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3130; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3131; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3132; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3133; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3134; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3135; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3136; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3137; SKIP-CACHE-INV-NEXT:    s_endpgm
3138;
3139; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
3140; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3141; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3142; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3143; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3144; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3145; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3146; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3147; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3148;
3149; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
3150; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3151; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3152; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3153; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3154; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3155; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3156; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3157; GFX90A-TGSPLIT-NEXT:    s_endpgm
3158;
3159; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
3160; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3161; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3162; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3163; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3164; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3165; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3166; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3167; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3168;
3169; GFX940-TGSPLIT-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
3170; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3171; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3172; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3173; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3174; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3175; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3176; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3177; GFX940-TGSPLIT-NEXT:    s_endpgm
3178    i32* %out, i32 %in, i32 %old) {
3179entry:
3180  %gep = getelementptr i32, i32* %out, i32 4
3181  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst seq_cst
3182  ret void
3183}
3184
3185define amdgpu_kernel void @flat_singlethread_monotonic_monotonic_ret_cmpxchg(
3186; GFX7-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
3187; GFX7:       ; %bb.0: ; %entry
3188; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3189; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3190; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3191; GFX7-NEXT:    s_add_u32 s4, s0, 16
3192; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3193; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3194; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3195; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3196; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3197; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3198; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3199; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3200; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3201; GFX7-NEXT:    flat_store_dword v[0:1], v2
3202; GFX7-NEXT:    s_endpgm
3203;
3204; GFX10-WGP-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
3205; GFX10-WGP:       ; %bb.0: ; %entry
3206; GFX10-WGP-NEXT:    s_clause 0x1
3207; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3208; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3209; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3210; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3211; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3212; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3213; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3214; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
3215; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3216; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3217; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3218; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3219; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3220; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3221; GFX10-WGP-NEXT:    s_endpgm
3222;
3223; GFX10-CU-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
3224; GFX10-CU:       ; %bb.0: ; %entry
3225; GFX10-CU-NEXT:    s_clause 0x1
3226; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3227; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3228; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3229; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3230; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3231; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3232; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3233; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3234; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3235; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3236; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3237; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3238; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3239; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3240; GFX10-CU-NEXT:    s_endpgm
3241;
3242; SKIP-CACHE-INV-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
3243; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3244; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3245; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3246; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3247; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3248; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3249; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3250; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3251; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3252; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3253; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3254; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3255; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3256; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3257; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3258; SKIP-CACHE-INV-NEXT:    s_endpgm
3259;
3260; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
3261; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3262; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3263; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3264; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3265; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3266; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3267; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3268; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3269; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3270; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3271;
3272; GFX90A-TGSPLIT-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
3273; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3274; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3275; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3276; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3277; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3278; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3279; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3280; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3281; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3282; GFX90A-TGSPLIT-NEXT:    s_endpgm
3283;
3284; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
3285; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3286; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3287; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3288; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3289; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3290; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3291; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
3292; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3293; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3294; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3295;
3296; GFX940-TGSPLIT-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
3297; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3298; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3299; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3300; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3301; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3302; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3303; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
3304; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3305; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3306; GFX940-TGSPLIT-NEXT:    s_endpgm
3307    i32* %out, i32 %in, i32 %old) {
3308entry:
3309  %gep = getelementptr i32, i32* %out, i32 4
3310  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic monotonic
3311  %val0 = extractvalue { i32, i1 } %val, 0
3312  store i32 %val0, i32* %out, align 4
3313  ret void
3314}
3315
3316define amdgpu_kernel void @flat_singlethread_acquire_monotonic_ret_cmpxchg(
3317; GFX7-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
3318; GFX7:       ; %bb.0: ; %entry
3319; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3320; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3321; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3322; GFX7-NEXT:    s_add_u32 s4, s0, 16
3323; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3324; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3325; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3326; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3327; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3328; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3329; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3330; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3331; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3332; GFX7-NEXT:    flat_store_dword v[0:1], v2
3333; GFX7-NEXT:    s_endpgm
3334;
3335; GFX10-WGP-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
3336; GFX10-WGP:       ; %bb.0: ; %entry
3337; GFX10-WGP-NEXT:    s_clause 0x1
3338; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3339; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3340; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3341; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3342; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3343; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3344; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3345; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
3346; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3347; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3348; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3349; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3350; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3351; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3352; GFX10-WGP-NEXT:    s_endpgm
3353;
3354; GFX10-CU-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
3355; GFX10-CU:       ; %bb.0: ; %entry
3356; GFX10-CU-NEXT:    s_clause 0x1
3357; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3358; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3359; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3360; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3361; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3362; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3363; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3364; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3365; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3366; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3367; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3368; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3369; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3370; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3371; GFX10-CU-NEXT:    s_endpgm
3372;
3373; SKIP-CACHE-INV-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
3374; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3375; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3376; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3377; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3378; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3379; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3380; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3381; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3382; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3383; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3384; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3385; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3386; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3387; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3388; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3389; SKIP-CACHE-INV-NEXT:    s_endpgm
3390;
3391; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
3392; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3393; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3394; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3395; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3396; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3397; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3398; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3399; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3400; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3401; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3402;
3403; GFX90A-TGSPLIT-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
3404; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3405; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3406; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3407; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3408; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3409; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3410; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3411; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3412; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3413; GFX90A-TGSPLIT-NEXT:    s_endpgm
3414;
3415; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
3416; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3417; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3418; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3419; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3420; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3421; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3422; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
3423; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3424; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3425; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3426;
3427; GFX940-TGSPLIT-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
3428; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3429; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3430; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3431; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3432; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3433; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3434; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
3435; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3436; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3437; GFX940-TGSPLIT-NEXT:    s_endpgm
3438    i32* %out, i32 %in, i32 %old) {
3439entry:
3440  %gep = getelementptr i32, i32* %out, i32 4
3441  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acquire monotonic
3442  %val0 = extractvalue { i32, i1 } %val, 0
3443  store i32 %val0, i32* %out, align 4
3444  ret void
3445}
3446
3447define amdgpu_kernel void @flat_singlethread_release_monotonic_ret_cmpxchg(
3448; GFX7-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
3449; GFX7:       ; %bb.0: ; %entry
3450; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3451; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3452; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3453; GFX7-NEXT:    s_add_u32 s4, s0, 16
3454; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3455; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3456; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3457; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3458; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3459; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3460; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3461; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3462; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3463; GFX7-NEXT:    flat_store_dword v[0:1], v2
3464; GFX7-NEXT:    s_endpgm
3465;
3466; GFX10-WGP-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
3467; GFX10-WGP:       ; %bb.0: ; %entry
3468; GFX10-WGP-NEXT:    s_clause 0x1
3469; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3470; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3471; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3472; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3473; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3474; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3475; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3476; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
3477; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3478; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3479; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3480; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3481; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3482; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3483; GFX10-WGP-NEXT:    s_endpgm
3484;
3485; GFX10-CU-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
3486; GFX10-CU:       ; %bb.0: ; %entry
3487; GFX10-CU-NEXT:    s_clause 0x1
3488; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3489; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3490; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3491; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3492; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3493; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3494; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3495; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3496; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3497; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3498; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3499; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3500; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3501; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3502; GFX10-CU-NEXT:    s_endpgm
3503;
3504; SKIP-CACHE-INV-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
3505; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3506; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3507; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3508; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3509; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3510; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3511; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3512; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3513; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3514; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3515; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3516; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3517; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3518; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3519; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3520; SKIP-CACHE-INV-NEXT:    s_endpgm
3521;
3522; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
3523; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3524; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3525; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3526; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3527; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3528; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3529; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3530; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3531; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3532; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3533;
3534; GFX90A-TGSPLIT-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
3535; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3536; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3537; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3538; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3539; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3540; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3541; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3542; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3543; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3544; GFX90A-TGSPLIT-NEXT:    s_endpgm
3545;
3546; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
3547; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3548; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3549; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3550; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3551; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3552; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3553; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
3554; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3555; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3556; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3557;
3558; GFX940-TGSPLIT-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
3559; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3560; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3561; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3562; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3563; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3564; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3565; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
3566; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3567; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3568; GFX940-TGSPLIT-NEXT:    s_endpgm
3569    i32* %out, i32 %in, i32 %old) {
3570entry:
3571  %gep = getelementptr i32, i32* %out, i32 4
3572  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") release monotonic
3573  %val0 = extractvalue { i32, i1 } %val, 0
3574  store i32 %val0, i32* %out, align 4
3575  ret void
3576}
3577
3578define amdgpu_kernel void @flat_singlethread_acq_rel_monotonic_ret_cmpxchg(
3579; GFX7-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
3580; GFX7:       ; %bb.0: ; %entry
3581; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3582; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3583; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3584; GFX7-NEXT:    s_add_u32 s4, s0, 16
3585; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3586; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3587; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3588; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3589; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3590; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3591; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3592; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3593; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3594; GFX7-NEXT:    flat_store_dword v[0:1], v2
3595; GFX7-NEXT:    s_endpgm
3596;
3597; GFX10-WGP-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
3598; GFX10-WGP:       ; %bb.0: ; %entry
3599; GFX10-WGP-NEXT:    s_clause 0x1
3600; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3601; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3602; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3603; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3604; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3605; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3606; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3607; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
3608; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3609; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3610; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3611; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3612; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3613; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3614; GFX10-WGP-NEXT:    s_endpgm
3615;
3616; GFX10-CU-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
3617; GFX10-CU:       ; %bb.0: ; %entry
3618; GFX10-CU-NEXT:    s_clause 0x1
3619; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3620; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3621; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3622; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3623; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3624; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3625; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3626; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3627; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3628; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3629; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3630; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3631; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3632; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3633; GFX10-CU-NEXT:    s_endpgm
3634;
3635; SKIP-CACHE-INV-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
3636; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3637; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3638; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3639; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3640; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3641; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3642; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3643; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3644; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3645; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3646; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3647; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3648; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3649; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3650; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3651; SKIP-CACHE-INV-NEXT:    s_endpgm
3652;
3653; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
3654; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3655; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3656; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3657; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3658; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3659; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3660; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3661; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3662; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3663; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3664;
3665; GFX90A-TGSPLIT-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
3666; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3667; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3668; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3669; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3670; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3671; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3672; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3673; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3674; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3675; GFX90A-TGSPLIT-NEXT:    s_endpgm
3676;
3677; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
3678; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3679; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3680; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3681; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3682; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3683; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3684; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
3685; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3686; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3687; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3688;
3689; GFX940-TGSPLIT-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
3690; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3691; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3692; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3693; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3694; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3695; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3696; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
3697; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3698; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3699; GFX940-TGSPLIT-NEXT:    s_endpgm
3700    i32* %out, i32 %in, i32 %old) {
3701entry:
3702  %gep = getelementptr i32, i32* %out, i32 4
3703  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel monotonic
3704  %val0 = extractvalue { i32, i1 } %val, 0
3705  store i32 %val0, i32* %out, align 4
3706  ret void
3707}
3708
3709define amdgpu_kernel void @flat_singlethread_seq_cst_monotonic_ret_cmpxchg(
3710; GFX7-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
3711; GFX7:       ; %bb.0: ; %entry
3712; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3713; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3714; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3715; GFX7-NEXT:    s_add_u32 s4, s0, 16
3716; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3717; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3718; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3719; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3720; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3721; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3722; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3723; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3724; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3725; GFX7-NEXT:    flat_store_dword v[0:1], v2
3726; GFX7-NEXT:    s_endpgm
3727;
3728; GFX10-WGP-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
3729; GFX10-WGP:       ; %bb.0: ; %entry
3730; GFX10-WGP-NEXT:    s_clause 0x1
3731; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3732; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3733; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3734; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3735; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3736; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3737; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3738; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
3739; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3740; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3741; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3742; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3743; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3744; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3745; GFX10-WGP-NEXT:    s_endpgm
3746;
3747; GFX10-CU-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
3748; GFX10-CU:       ; %bb.0: ; %entry
3749; GFX10-CU-NEXT:    s_clause 0x1
3750; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3751; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3752; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3753; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3754; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3755; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3756; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3757; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3758; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3759; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3760; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3761; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3762; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3763; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3764; GFX10-CU-NEXT:    s_endpgm
3765;
3766; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
3767; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3768; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3769; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3770; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3771; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3772; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3773; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3774; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3775; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3776; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3777; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3778; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3779; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3780; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3781; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3782; SKIP-CACHE-INV-NEXT:    s_endpgm
3783;
3784; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
3785; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3786; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3787; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3788; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3789; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3790; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3791; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3792; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3793; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3794; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3795;
3796; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
3797; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3798; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3799; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3800; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3801; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3802; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3803; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3804; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3805; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3806; GFX90A-TGSPLIT-NEXT:    s_endpgm
3807;
3808; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
3809; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3810; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3811; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3812; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3813; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3814; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3815; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
3816; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3817; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3818; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3819;
3820; GFX940-TGSPLIT-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
3821; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3822; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3823; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3824; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3825; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3826; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3827; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
3828; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3829; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3830; GFX940-TGSPLIT-NEXT:    s_endpgm
3831    i32* %out, i32 %in, i32 %old) {
3832entry:
3833  %gep = getelementptr i32, i32* %out, i32 4
3834  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst monotonic
3835  %val0 = extractvalue { i32, i1 } %val, 0
3836  store i32 %val0, i32* %out, align 4
3837  ret void
3838}
3839
3840define amdgpu_kernel void @flat_singlethread_monotonic_acquire_ret_cmpxchg(
3841; GFX7-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
3842; GFX7:       ; %bb.0: ; %entry
3843; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3844; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3845; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3846; GFX7-NEXT:    s_add_u32 s4, s0, 16
3847; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3848; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3849; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3850; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3851; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3852; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3853; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3854; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3855; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3856; GFX7-NEXT:    flat_store_dword v[0:1], v2
3857; GFX7-NEXT:    s_endpgm
3858;
3859; GFX10-WGP-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
3860; GFX10-WGP:       ; %bb.0: ; %entry
3861; GFX10-WGP-NEXT:    s_clause 0x1
3862; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3863; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3864; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3865; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3866; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3867; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3868; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3869; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
3870; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3871; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3872; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3873; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3874; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3875; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3876; GFX10-WGP-NEXT:    s_endpgm
3877;
3878; GFX10-CU-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
3879; GFX10-CU:       ; %bb.0: ; %entry
3880; GFX10-CU-NEXT:    s_clause 0x1
3881; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3882; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3883; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3884; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3885; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3886; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3887; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3888; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3889; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3890; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3891; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3892; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3893; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3894; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3895; GFX10-CU-NEXT:    s_endpgm
3896;
3897; SKIP-CACHE-INV-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
3898; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3899; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3900; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3901; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3902; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3903; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3904; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3905; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3906; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3907; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3908; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3909; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3910; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3911; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3912; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3913; SKIP-CACHE-INV-NEXT:    s_endpgm
3914;
3915; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
3916; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3917; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3918; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3919; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3920; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3921; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3922; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3923; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3924; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3925; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3926;
3927; GFX90A-TGSPLIT-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
3928; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3929; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3930; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3931; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3932; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3933; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3934; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3935; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3936; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3937; GFX90A-TGSPLIT-NEXT:    s_endpgm
3938;
3939; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
3940; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3941; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3942; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3943; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3944; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3945; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3946; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
3947; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3948; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3949; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3950;
3951; GFX940-TGSPLIT-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
3952; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3953; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3954; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3955; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3956; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3957; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3958; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
3959; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3960; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3961; GFX940-TGSPLIT-NEXT:    s_endpgm
3962    i32* %out, i32 %in, i32 %old) {
3963entry:
3964  %gep = getelementptr i32, i32* %out, i32 4
3965  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic acquire
3966  %val0 = extractvalue { i32, i1 } %val, 0
3967  store i32 %val0, i32* %out, align 4
3968  ret void
3969}
3970
3971define amdgpu_kernel void @flat_singlethread_acquire_acquire_ret_cmpxchg(
3972; GFX7-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
3973; GFX7:       ; %bb.0: ; %entry
3974; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3975; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3976; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3977; GFX7-NEXT:    s_add_u32 s4, s0, 16
3978; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3979; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3980; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3981; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3982; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3983; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3984; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3985; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3986; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3987; GFX7-NEXT:    flat_store_dword v[0:1], v2
3988; GFX7-NEXT:    s_endpgm
3989;
3990; GFX10-WGP-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
3991; GFX10-WGP:       ; %bb.0: ; %entry
3992; GFX10-WGP-NEXT:    s_clause 0x1
3993; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3994; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3995; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3996; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3997; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3998; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3999; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4000; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4001; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4002; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4003; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4004; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4005; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4006; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4007; GFX10-WGP-NEXT:    s_endpgm
4008;
4009; GFX10-CU-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
4010; GFX10-CU:       ; %bb.0: ; %entry
4011; GFX10-CU-NEXT:    s_clause 0x1
4012; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4013; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4014; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4015; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4016; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4017; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4018; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4019; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4020; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4021; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4022; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4023; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4024; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4025; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4026; GFX10-CU-NEXT:    s_endpgm
4027;
4028; SKIP-CACHE-INV-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
4029; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4030; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4031; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4032; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4033; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4034; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4035; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4036; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4037; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4038; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4039; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4040; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4041; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4042; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4043; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4044; SKIP-CACHE-INV-NEXT:    s_endpgm
4045;
4046; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
4047; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4048; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4049; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4050; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4051; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4052; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4053; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4054; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4055; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4056; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4057;
4058; GFX90A-TGSPLIT-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
4059; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4060; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4061; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4062; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4063; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4064; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4065; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4066; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4067; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4068; GFX90A-TGSPLIT-NEXT:    s_endpgm
4069;
4070; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
4071; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4072; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4073; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4074; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4075; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4076; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4077; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4078; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4079; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4080; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4081;
4082; GFX940-TGSPLIT-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
4083; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4084; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4085; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4086; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4087; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4088; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4089; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4090; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4091; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4092; GFX940-TGSPLIT-NEXT:    s_endpgm
4093    i32* %out, i32 %in, i32 %old) {
4094entry:
4095  %gep = getelementptr i32, i32* %out, i32 4
4096  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acquire acquire
4097  %val0 = extractvalue { i32, i1 } %val, 0
4098  store i32 %val0, i32* %out, align 4
4099  ret void
4100}
4101
4102define amdgpu_kernel void @flat_singlethread_release_acquire_ret_cmpxchg(
4103; GFX7-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
4104; GFX7:       ; %bb.0: ; %entry
4105; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4106; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4107; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4108; GFX7-NEXT:    s_add_u32 s4, s0, 16
4109; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4110; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4111; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4112; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4113; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4114; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4115; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4116; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4117; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4118; GFX7-NEXT:    flat_store_dword v[0:1], v2
4119; GFX7-NEXT:    s_endpgm
4120;
4121; GFX10-WGP-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
4122; GFX10-WGP:       ; %bb.0: ; %entry
4123; GFX10-WGP-NEXT:    s_clause 0x1
4124; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4125; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4126; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4127; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4128; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4129; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4130; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4131; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4132; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4133; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4134; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4135; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4136; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4137; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4138; GFX10-WGP-NEXT:    s_endpgm
4139;
4140; GFX10-CU-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
4141; GFX10-CU:       ; %bb.0: ; %entry
4142; GFX10-CU-NEXT:    s_clause 0x1
4143; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4144; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4145; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4146; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4147; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4148; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4149; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4150; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4151; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4152; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4153; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4154; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4155; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4156; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4157; GFX10-CU-NEXT:    s_endpgm
4158;
4159; SKIP-CACHE-INV-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
4160; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4161; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4162; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4163; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4164; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4165; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4166; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4167; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4168; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4169; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4170; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4171; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4172; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4173; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4174; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4175; SKIP-CACHE-INV-NEXT:    s_endpgm
4176;
4177; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
4178; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4179; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4180; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4181; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4182; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4183; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4184; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4185; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4186; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4187; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4188;
4189; GFX90A-TGSPLIT-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
4190; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4191; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4192; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4193; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4194; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4195; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4196; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4197; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4198; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4199; GFX90A-TGSPLIT-NEXT:    s_endpgm
4200;
4201; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
4202; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4203; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4204; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4205; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4206; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4207; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4208; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4209; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4210; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4211; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4212;
4213; GFX940-TGSPLIT-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
4214; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4215; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4216; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4217; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4218; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4219; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4220; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4221; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4222; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4223; GFX940-TGSPLIT-NEXT:    s_endpgm
4224    i32* %out, i32 %in, i32 %old) {
4225entry:
4226  %gep = getelementptr i32, i32* %out, i32 4
4227  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") release acquire
4228  %val0 = extractvalue { i32, i1 } %val, 0
4229  store i32 %val0, i32* %out, align 4
4230  ret void
4231}
4232
4233define amdgpu_kernel void @flat_singlethread_acq_rel_acquire_ret_cmpxchg(
4234; GFX7-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
4235; GFX7:       ; %bb.0: ; %entry
4236; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4237; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4238; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4239; GFX7-NEXT:    s_add_u32 s4, s0, 16
4240; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4241; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4242; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4243; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4244; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4245; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4246; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4247; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4248; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4249; GFX7-NEXT:    flat_store_dword v[0:1], v2
4250; GFX7-NEXT:    s_endpgm
4251;
4252; GFX10-WGP-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
4253; GFX10-WGP:       ; %bb.0: ; %entry
4254; GFX10-WGP-NEXT:    s_clause 0x1
4255; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4256; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4257; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4258; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4259; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4260; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4261; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4262; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4263; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4264; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4265; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4266; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4267; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4268; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4269; GFX10-WGP-NEXT:    s_endpgm
4270;
4271; GFX10-CU-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
4272; GFX10-CU:       ; %bb.0: ; %entry
4273; GFX10-CU-NEXT:    s_clause 0x1
4274; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4275; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4276; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4277; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4278; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4279; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4280; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4281; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4282; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4283; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4284; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4285; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4286; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4287; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4288; GFX10-CU-NEXT:    s_endpgm
4289;
4290; SKIP-CACHE-INV-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
4291; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4292; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4293; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4294; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4295; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4296; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4297; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4298; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4299; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4300; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4301; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4302; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4303; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4304; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4305; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4306; SKIP-CACHE-INV-NEXT:    s_endpgm
4307;
4308; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
4309; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4310; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4311; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4312; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4313; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4314; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4315; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4316; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4317; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4318; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4319;
4320; GFX90A-TGSPLIT-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
4321; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4322; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4323; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4324; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4325; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4326; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4327; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4328; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4329; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4330; GFX90A-TGSPLIT-NEXT:    s_endpgm
4331;
4332; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
4333; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4334; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4335; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4336; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4337; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4338; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4339; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4340; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4341; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4342; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4343;
4344; GFX940-TGSPLIT-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
4345; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4346; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4347; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4348; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4349; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4350; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4351; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4352; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4353; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4354; GFX940-TGSPLIT-NEXT:    s_endpgm
4355    i32* %out, i32 %in, i32 %old) {
4356entry:
4357  %gep = getelementptr i32, i32* %out, i32 4
4358  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel acquire
4359  %val0 = extractvalue { i32, i1 } %val, 0
4360  store i32 %val0, i32* %out, align 4
4361  ret void
4362}
4363
4364define amdgpu_kernel void @flat_singlethread_seq_cst_acquire_ret_cmpxchg(
4365; GFX7-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
4366; GFX7:       ; %bb.0: ; %entry
4367; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4368; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4369; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4370; GFX7-NEXT:    s_add_u32 s4, s0, 16
4371; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4372; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4373; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4374; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4375; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4376; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4377; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4378; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4379; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4380; GFX7-NEXT:    flat_store_dword v[0:1], v2
4381; GFX7-NEXT:    s_endpgm
4382;
4383; GFX10-WGP-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
4384; GFX10-WGP:       ; %bb.0: ; %entry
4385; GFX10-WGP-NEXT:    s_clause 0x1
4386; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4387; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4388; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4389; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4390; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4391; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4392; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4393; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4394; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4395; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4396; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4397; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4398; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4399; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4400; GFX10-WGP-NEXT:    s_endpgm
4401;
4402; GFX10-CU-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
4403; GFX10-CU:       ; %bb.0: ; %entry
4404; GFX10-CU-NEXT:    s_clause 0x1
4405; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4406; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4407; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4408; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4409; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4410; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4411; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4412; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4413; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4414; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4415; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4416; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4417; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4418; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4419; GFX10-CU-NEXT:    s_endpgm
4420;
4421; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
4422; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4423; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4424; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4425; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4426; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4427; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4428; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4429; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4430; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4431; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4432; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4433; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4434; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4435; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4436; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4437; SKIP-CACHE-INV-NEXT:    s_endpgm
4438;
4439; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
4440; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4441; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4442; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4443; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4444; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4445; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4446; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4447; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4448; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4449; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4450;
4451; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
4452; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4453; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4454; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4455; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4456; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4457; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4458; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4459; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4460; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4461; GFX90A-TGSPLIT-NEXT:    s_endpgm
4462;
4463; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
4464; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4465; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4466; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4467; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4468; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4469; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4470; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4471; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4472; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4473; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4474;
4475; GFX940-TGSPLIT-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
4476; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4477; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4478; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4479; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4480; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4481; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4482; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4483; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4484; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4485; GFX940-TGSPLIT-NEXT:    s_endpgm
4486    i32* %out, i32 %in, i32 %old) {
4487entry:
4488  %gep = getelementptr i32, i32* %out, i32 4
4489  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst acquire
4490  %val0 = extractvalue { i32, i1 } %val, 0
4491  store i32 %val0, i32* %out, align 4
4492  ret void
4493}
4494
4495define amdgpu_kernel void @flat_singlethread_monotonic_seq_cst_ret_cmpxchg(
4496; GFX7-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
4497; GFX7:       ; %bb.0: ; %entry
4498; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4499; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4500; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4501; GFX7-NEXT:    s_add_u32 s4, s0, 16
4502; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4503; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4504; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4505; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4506; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4507; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4508; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4509; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4510; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4511; GFX7-NEXT:    flat_store_dword v[0:1], v2
4512; GFX7-NEXT:    s_endpgm
4513;
4514; GFX10-WGP-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
4515; GFX10-WGP:       ; %bb.0: ; %entry
4516; GFX10-WGP-NEXT:    s_clause 0x1
4517; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4518; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4519; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4520; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4521; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4522; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4523; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4524; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4525; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4526; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4527; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4528; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4529; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4530; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4531; GFX10-WGP-NEXT:    s_endpgm
4532;
4533; GFX10-CU-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
4534; GFX10-CU:       ; %bb.0: ; %entry
4535; GFX10-CU-NEXT:    s_clause 0x1
4536; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4537; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4538; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4539; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4540; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4541; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4542; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4543; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4544; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4545; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4546; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4547; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4548; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4549; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4550; GFX10-CU-NEXT:    s_endpgm
4551;
4552; SKIP-CACHE-INV-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
4553; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4554; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4555; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4556; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4557; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4558; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4559; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4560; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4561; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4562; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4563; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4564; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4565; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4566; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4567; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4568; SKIP-CACHE-INV-NEXT:    s_endpgm
4569;
4570; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
4571; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4572; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4573; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4574; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4575; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4576; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4577; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4578; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4579; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4580; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4581;
4582; GFX90A-TGSPLIT-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
4583; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4584; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4585; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4586; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4587; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4588; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4589; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4590; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4591; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4592; GFX90A-TGSPLIT-NEXT:    s_endpgm
4593;
4594; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
4595; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4596; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4597; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4598; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4599; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4600; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4601; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4602; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4603; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4604; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4605;
4606; GFX940-TGSPLIT-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
4607; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4608; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4609; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4610; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4611; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4612; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4613; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4614; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4615; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4616; GFX940-TGSPLIT-NEXT:    s_endpgm
4617    i32* %out, i32 %in, i32 %old) {
4618entry:
4619  %gep = getelementptr i32, i32* %out, i32 4
4620  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic seq_cst
4621  %val0 = extractvalue { i32, i1 } %val, 0
4622  store i32 %val0, i32* %out, align 4
4623  ret void
4624}
4625
4626define amdgpu_kernel void @flat_singlethread_acquire_seq_cst_ret_cmpxchg(
4627; GFX7-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
4628; GFX7:       ; %bb.0: ; %entry
4629; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4630; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4631; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4632; GFX7-NEXT:    s_add_u32 s4, s0, 16
4633; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4634; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4635; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4636; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4637; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4638; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4639; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4640; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4641; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4642; GFX7-NEXT:    flat_store_dword v[0:1], v2
4643; GFX7-NEXT:    s_endpgm
4644;
4645; GFX10-WGP-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
4646; GFX10-WGP:       ; %bb.0: ; %entry
4647; GFX10-WGP-NEXT:    s_clause 0x1
4648; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4649; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4650; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4651; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4652; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4653; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4654; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4655; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4656; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4657; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4658; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4659; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4660; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4661; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4662; GFX10-WGP-NEXT:    s_endpgm
4663;
4664; GFX10-CU-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
4665; GFX10-CU:       ; %bb.0: ; %entry
4666; GFX10-CU-NEXT:    s_clause 0x1
4667; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4668; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4669; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4670; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4671; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4672; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4673; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4674; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4675; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4676; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4677; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4678; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4679; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4680; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4681; GFX10-CU-NEXT:    s_endpgm
4682;
4683; SKIP-CACHE-INV-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
4684; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4685; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4686; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4687; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4688; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4689; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4690; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4691; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4692; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4693; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4694; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4695; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4696; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4697; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4698; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4699; SKIP-CACHE-INV-NEXT:    s_endpgm
4700;
4701; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
4702; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4703; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4704; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4705; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4706; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4707; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4708; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4709; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4710; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4711; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4712;
4713; GFX90A-TGSPLIT-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
4714; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4715; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4716; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4717; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4718; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4719; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4720; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4721; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4722; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4723; GFX90A-TGSPLIT-NEXT:    s_endpgm
4724;
4725; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
4726; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4727; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4728; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4729; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4730; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4731; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4732; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4733; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4734; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4735; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4736;
4737; GFX940-TGSPLIT-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
4738; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4739; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4740; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4741; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4742; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4743; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4744; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4745; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4746; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4747; GFX940-TGSPLIT-NEXT:    s_endpgm
4748    i32* %out, i32 %in, i32 %old) {
4749entry:
4750  %gep = getelementptr i32, i32* %out, i32 4
4751  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acquire seq_cst
4752  %val0 = extractvalue { i32, i1 } %val, 0
4753  store i32 %val0, i32* %out, align 4
4754  ret void
4755}
4756
4757define amdgpu_kernel void @flat_singlethread_release_seq_cst_ret_cmpxchg(
4758; GFX7-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
4759; GFX7:       ; %bb.0: ; %entry
4760; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4761; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4762; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4763; GFX7-NEXT:    s_add_u32 s4, s0, 16
4764; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4765; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4766; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4767; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4768; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4769; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4770; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4771; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4772; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4773; GFX7-NEXT:    flat_store_dword v[0:1], v2
4774; GFX7-NEXT:    s_endpgm
4775;
4776; GFX10-WGP-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
4777; GFX10-WGP:       ; %bb.0: ; %entry
4778; GFX10-WGP-NEXT:    s_clause 0x1
4779; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4780; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4781; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4782; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4783; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4784; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4785; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4786; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4787; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4788; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4789; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4790; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4791; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4792; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4793; GFX10-WGP-NEXT:    s_endpgm
4794;
4795; GFX10-CU-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
4796; GFX10-CU:       ; %bb.0: ; %entry
4797; GFX10-CU-NEXT:    s_clause 0x1
4798; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4799; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4800; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4801; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4802; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4803; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4804; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4805; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4806; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4807; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4808; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4809; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4810; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4811; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4812; GFX10-CU-NEXT:    s_endpgm
4813;
4814; SKIP-CACHE-INV-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
4815; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4816; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4817; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4818; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4819; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4820; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4821; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4822; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4823; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4824; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4825; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4826; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4827; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4828; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4829; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4830; SKIP-CACHE-INV-NEXT:    s_endpgm
4831;
4832; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
4833; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4834; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4835; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4836; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4837; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4838; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4839; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4840; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4841; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4842; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4843;
4844; GFX90A-TGSPLIT-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
4845; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4846; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4847; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4848; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4849; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4850; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4851; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4852; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4853; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4854; GFX90A-TGSPLIT-NEXT:    s_endpgm
4855;
4856; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
4857; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4858; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4859; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4860; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4861; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4862; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4863; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4864; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4865; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4866; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4867;
4868; GFX940-TGSPLIT-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
4869; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4870; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4871; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4872; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4873; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4874; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4875; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4876; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4877; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4878; GFX940-TGSPLIT-NEXT:    s_endpgm
4879    i32* %out, i32 %in, i32 %old) {
4880entry:
4881  %gep = getelementptr i32, i32* %out, i32 4
4882  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") release seq_cst
4883  %val0 = extractvalue { i32, i1 } %val, 0
4884  store i32 %val0, i32* %out, align 4
4885  ret void
4886}
4887
4888define amdgpu_kernel void @flat_singlethread_acq_rel_seq_cst_ret_cmpxchg(
4889; GFX7-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
4890; GFX7:       ; %bb.0: ; %entry
4891; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4892; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4893; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4894; GFX7-NEXT:    s_add_u32 s4, s0, 16
4895; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4896; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4897; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4898; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4899; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4900; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4901; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4902; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4903; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4904; GFX7-NEXT:    flat_store_dword v[0:1], v2
4905; GFX7-NEXT:    s_endpgm
4906;
4907; GFX10-WGP-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
4908; GFX10-WGP:       ; %bb.0: ; %entry
4909; GFX10-WGP-NEXT:    s_clause 0x1
4910; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4911; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4912; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4913; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4914; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4915; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4916; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4917; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4918; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4919; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4920; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4921; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4922; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4923; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4924; GFX10-WGP-NEXT:    s_endpgm
4925;
4926; GFX10-CU-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
4927; GFX10-CU:       ; %bb.0: ; %entry
4928; GFX10-CU-NEXT:    s_clause 0x1
4929; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4930; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4931; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4932; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4933; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4934; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4935; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4936; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4937; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4938; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4939; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4940; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4941; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4942; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4943; GFX10-CU-NEXT:    s_endpgm
4944;
4945; SKIP-CACHE-INV-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
4946; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4947; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4948; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4949; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4950; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4951; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4952; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4953; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4954; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4955; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4956; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4957; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4958; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4959; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4960; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4961; SKIP-CACHE-INV-NEXT:    s_endpgm
4962;
4963; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
4964; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4965; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4966; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4967; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4968; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4969; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4970; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4971; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4972; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4973; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4974;
4975; GFX90A-TGSPLIT-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
4976; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4977; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4978; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4979; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4980; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4981; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4982; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4983; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4984; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4985; GFX90A-TGSPLIT-NEXT:    s_endpgm
4986;
4987; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
4988; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4989; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4990; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4991; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4992; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4993; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4994; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4995; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4996; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4997; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4998;
4999; GFX940-TGSPLIT-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
5000; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5001; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5002; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5003; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5004; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5005; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5006; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5007; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5008; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5009; GFX940-TGSPLIT-NEXT:    s_endpgm
5010    i32* %out, i32 %in, i32 %old) {
5011entry:
5012  %gep = getelementptr i32, i32* %out, i32 4
5013  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel seq_cst
5014  %val0 = extractvalue { i32, i1 } %val, 0
5015  store i32 %val0, i32* %out, align 4
5016  ret void
5017}
5018
5019define amdgpu_kernel void @flat_singlethread_seq_cst_seq_cst_ret_cmpxchg(
5020; GFX7-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5021; GFX7:       ; %bb.0: ; %entry
5022; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5023; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5024; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5025; GFX7-NEXT:    s_add_u32 s4, s0, 16
5026; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5027; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5028; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5029; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5030; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5031; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5032; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5033; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5034; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5035; GFX7-NEXT:    flat_store_dword v[0:1], v2
5036; GFX7-NEXT:    s_endpgm
5037;
5038; GFX10-WGP-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5039; GFX10-WGP:       ; %bb.0: ; %entry
5040; GFX10-WGP-NEXT:    s_clause 0x1
5041; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5042; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5043; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5044; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5045; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5046; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5047; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5048; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5049; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5050; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5051; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5052; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5053; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5054; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5055; GFX10-WGP-NEXT:    s_endpgm
5056;
5057; GFX10-CU-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5058; GFX10-CU:       ; %bb.0: ; %entry
5059; GFX10-CU-NEXT:    s_clause 0x1
5060; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5061; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5062; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5063; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5064; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5065; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5066; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5067; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5068; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5069; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5070; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5071; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5072; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5073; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5074; GFX10-CU-NEXT:    s_endpgm
5075;
5076; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5077; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5078; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5079; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5080; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5081; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5082; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5083; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5084; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5085; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5086; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5087; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5088; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5089; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5090; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5091; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5092; SKIP-CACHE-INV-NEXT:    s_endpgm
5093;
5094; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5095; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5096; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5097; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5098; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5099; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5100; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5101; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5102; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5103; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5104; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5105;
5106; GFX90A-TGSPLIT-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5107; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5108; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5109; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5110; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5111; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5112; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5113; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5114; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5115; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5116; GFX90A-TGSPLIT-NEXT:    s_endpgm
5117;
5118; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5119; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5120; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5121; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5122; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5123; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5124; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5125; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5126; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5127; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5128; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5129;
5130; GFX940-TGSPLIT-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
5131; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5132; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5133; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5134; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5135; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5136; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5137; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5138; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5139; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5140; GFX940-TGSPLIT-NEXT:    s_endpgm
5141    i32* %out, i32 %in, i32 %old) {
5142entry:
5143  %gep = getelementptr i32, i32* %out, i32 4
5144  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst seq_cst
5145  %val0 = extractvalue { i32, i1 } %val, 0
5146  store i32 %val0, i32* %out, align 4
5147  ret void
5148}
5149
5150define amdgpu_kernel void @flat_singlethread_one_as_unordered_load(
5151; GFX7-LABEL: flat_singlethread_one_as_unordered_load:
5152; GFX7:       ; %bb.0: ; %entry
5153; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5154; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5155; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5156; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5157; GFX7-NEXT:    flat_load_dword v2, v[0:1]
5158; GFX7-NEXT:    v_mov_b32_e32 v0, s2
5159; GFX7-NEXT:    v_mov_b32_e32 v1, s3
5160; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5161; GFX7-NEXT:    flat_store_dword v[0:1], v2
5162; GFX7-NEXT:    s_endpgm
5163;
5164; GFX10-WGP-LABEL: flat_singlethread_one_as_unordered_load:
5165; GFX10-WGP:       ; %bb.0: ; %entry
5166; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5167; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5168; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5169; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5170; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
5171; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
5172; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
5173; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5174; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5175; GFX10-WGP-NEXT:    s_endpgm
5176;
5177; GFX10-CU-LABEL: flat_singlethread_one_as_unordered_load:
5178; GFX10-CU:       ; %bb.0: ; %entry
5179; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5180; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5181; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5182; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5183; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
5184; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
5185; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
5186; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5187; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5188; GFX10-CU-NEXT:    s_endpgm
5189;
5190; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_unordered_load:
5191; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5192; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5193; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5194; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5195; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
5196; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
5197; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5198; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5199; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5200; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5201; SKIP-CACHE-INV-NEXT:    s_endpgm
5202;
5203; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_unordered_load:
5204; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5205; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5206; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5207; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5208; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
5209; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
5210; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
5211; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
5212; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5213; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5214; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5215;
5216; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_unordered_load:
5217; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5218; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5219; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5220; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5221; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
5222; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
5223; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
5224; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
5225; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5226; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5227; GFX90A-TGSPLIT-NEXT:    s_endpgm
5228;
5229; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_unordered_load:
5230; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5231; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5232; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5233; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5234; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
5235; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
5236; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
5237; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
5238; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5239; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5240; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5241;
5242; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_unordered_load:
5243; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5244; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5245; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5246; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5247; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
5248; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
5249; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
5250; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
5251; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5252; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5253; GFX940-TGSPLIT-NEXT:    s_endpgm
5254    i32* %in, i32* %out) {
5255entry:
5256  %val = load atomic i32, i32* %in syncscope("singlethread-one-as") unordered, align 4
5257  store i32 %val, i32* %out
5258  ret void
5259}
5260
5261define amdgpu_kernel void @flat_singlethread_one_as_monotonic_load(
5262; GFX7-LABEL: flat_singlethread_one_as_monotonic_load:
5263; GFX7:       ; %bb.0: ; %entry
5264; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5265; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5266; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5267; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5268; GFX7-NEXT:    flat_load_dword v2, v[0:1]
5269; GFX7-NEXT:    v_mov_b32_e32 v0, s2
5270; GFX7-NEXT:    v_mov_b32_e32 v1, s3
5271; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5272; GFX7-NEXT:    flat_store_dword v[0:1], v2
5273; GFX7-NEXT:    s_endpgm
5274;
5275; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_load:
5276; GFX10-WGP:       ; %bb.0: ; %entry
5277; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5278; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5279; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5280; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5281; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
5282; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
5283; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
5284; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5285; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5286; GFX10-WGP-NEXT:    s_endpgm
5287;
5288; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_load:
5289; GFX10-CU:       ; %bb.0: ; %entry
5290; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5291; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5292; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5293; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5294; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
5295; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
5296; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
5297; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5298; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5299; GFX10-CU-NEXT:    s_endpgm
5300;
5301; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_monotonic_load:
5302; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5303; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5304; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5305; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5306; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
5307; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
5308; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5309; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5310; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5311; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5312; SKIP-CACHE-INV-NEXT:    s_endpgm
5313;
5314; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_load:
5315; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5316; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5317; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5318; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5319; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
5320; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
5321; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
5322; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
5323; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5324; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5325; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5326;
5327; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_load:
5328; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5329; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5330; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5331; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5332; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
5333; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
5334; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
5335; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
5336; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5337; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5338; GFX90A-TGSPLIT-NEXT:    s_endpgm
5339;
5340; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_load:
5341; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5342; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5343; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5344; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5345; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
5346; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
5347; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
5348; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
5349; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5350; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5351; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5352;
5353; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_load:
5354; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5355; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5356; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5357; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5358; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
5359; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
5360; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
5361; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
5362; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5363; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5364; GFX940-TGSPLIT-NEXT:    s_endpgm
5365    i32* %in, i32* %out) {
5366entry:
5367  %val = load atomic i32, i32* %in syncscope("singlethread-one-as") monotonic, align 4
5368  store i32 %val, i32* %out
5369  ret void
5370}
5371
5372define amdgpu_kernel void @flat_singlethread_one_as_acquire_load(
5373; GFX7-LABEL: flat_singlethread_one_as_acquire_load:
5374; GFX7:       ; %bb.0: ; %entry
5375; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5376; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5377; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5378; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5379; GFX7-NEXT:    flat_load_dword v2, v[0:1]
5380; GFX7-NEXT:    v_mov_b32_e32 v0, s2
5381; GFX7-NEXT:    v_mov_b32_e32 v1, s3
5382; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5383; GFX7-NEXT:    flat_store_dword v[0:1], v2
5384; GFX7-NEXT:    s_endpgm
5385;
5386; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_load:
5387; GFX10-WGP:       ; %bb.0: ; %entry
5388; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5389; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5390; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5391; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5392; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
5393; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
5394; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
5395; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5396; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5397; GFX10-WGP-NEXT:    s_endpgm
5398;
5399; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_load:
5400; GFX10-CU:       ; %bb.0: ; %entry
5401; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5402; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5403; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5404; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5405; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
5406; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
5407; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
5408; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5409; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5410; GFX10-CU-NEXT:    s_endpgm
5411;
5412; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acquire_load:
5413; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5414; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5415; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5416; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5417; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
5418; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
5419; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5420; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5421; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5422; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5423; SKIP-CACHE-INV-NEXT:    s_endpgm
5424;
5425; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_load:
5426; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5427; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5428; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5429; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5430; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
5431; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
5432; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
5433; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
5434; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5435; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5436; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5437;
5438; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_load:
5439; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5440; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5441; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5442; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5443; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
5444; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
5445; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
5446; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
5447; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5448; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5449; GFX90A-TGSPLIT-NEXT:    s_endpgm
5450;
5451; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_load:
5452; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5453; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5454; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5455; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5456; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
5457; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
5458; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
5459; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
5460; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5461; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5462; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5463;
5464; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_load:
5465; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5466; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5467; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5468; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5469; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
5470; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
5471; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
5472; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
5473; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5474; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5475; GFX940-TGSPLIT-NEXT:    s_endpgm
5476    i32* %in, i32* %out) {
5477entry:
5478  %val = load atomic i32, i32* %in syncscope("singlethread-one-as") acquire, align 4
5479  store i32 %val, i32* %out
5480  ret void
5481}
5482
5483define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_load(
5484; GFX7-LABEL: flat_singlethread_one_as_seq_cst_load:
5485; GFX7:       ; %bb.0: ; %entry
5486; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5487; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5488; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5489; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5490; GFX7-NEXT:    flat_load_dword v2, v[0:1]
5491; GFX7-NEXT:    v_mov_b32_e32 v0, s2
5492; GFX7-NEXT:    v_mov_b32_e32 v1, s3
5493; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5494; GFX7-NEXT:    flat_store_dword v[0:1], v2
5495; GFX7-NEXT:    s_endpgm
5496;
5497; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_load:
5498; GFX10-WGP:       ; %bb.0: ; %entry
5499; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5500; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5501; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5502; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5503; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
5504; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
5505; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
5506; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5507; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5508; GFX10-WGP-NEXT:    s_endpgm
5509;
5510; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_load:
5511; GFX10-CU:       ; %bb.0: ; %entry
5512; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5513; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5514; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5515; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5516; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
5517; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
5518; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
5519; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5520; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5521; GFX10-CU-NEXT:    s_endpgm
5522;
5523; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_load:
5524; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5525; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5526; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5527; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5528; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
5529; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
5530; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5531; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5532; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5533; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5534; SKIP-CACHE-INV-NEXT:    s_endpgm
5535;
5536; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_load:
5537; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5538; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5539; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5540; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5541; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
5542; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
5543; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
5544; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
5545; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5546; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5547; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5548;
5549; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_load:
5550; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5551; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5552; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5553; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5554; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
5555; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
5556; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
5557; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
5558; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5559; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5560; GFX90A-TGSPLIT-NEXT:    s_endpgm
5561;
5562; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_load:
5563; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5564; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5565; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5566; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5567; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
5568; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
5569; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
5570; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
5571; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5572; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5573; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5574;
5575; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_load:
5576; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5577; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5578; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5579; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5580; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
5581; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
5582; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
5583; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
5584; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5585; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5586; GFX940-TGSPLIT-NEXT:    s_endpgm
5587    i32* %in, i32* %out) {
5588entry:
5589  %val = load atomic i32, i32* %in syncscope("singlethread-one-as") seq_cst, align 4
5590  store i32 %val, i32* %out
5591  ret void
5592}
5593
5594define amdgpu_kernel void @flat_singlethread_one_as_unordered_store(
5595; GFX7-LABEL: flat_singlethread_one_as_unordered_store:
5596; GFX7:       ; %bb.0: ; %entry
5597; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
5598; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
5599; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5600; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5601; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5602; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5603; GFX7-NEXT:    flat_store_dword v[0:1], v2
5604; GFX7-NEXT:    s_endpgm
5605;
5606; GFX10-WGP-LABEL: flat_singlethread_one_as_unordered_store:
5607; GFX10-WGP:       ; %bb.0: ; %entry
5608; GFX10-WGP-NEXT:    s_clause 0x1
5609; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5610; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
5611; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5612; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5613; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5614; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5615; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5616; GFX10-WGP-NEXT:    s_endpgm
5617;
5618; GFX10-CU-LABEL: flat_singlethread_one_as_unordered_store:
5619; GFX10-CU:       ; %bb.0: ; %entry
5620; GFX10-CU-NEXT:    s_clause 0x1
5621; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5622; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
5623; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5624; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5625; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5626; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5627; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5628; GFX10-CU-NEXT:    s_endpgm
5629;
5630; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_unordered_store:
5631; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5632; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
5633; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
5634; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5635; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5636; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5637; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5638; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5639; SKIP-CACHE-INV-NEXT:    s_endpgm
5640;
5641; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_unordered_store:
5642; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5643; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5644; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
5645; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5646; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5647; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5648; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5649; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5650;
5651; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_unordered_store:
5652; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5653; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5654; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
5655; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5656; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5657; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5658; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5659; GFX90A-TGSPLIT-NEXT:    s_endpgm
5660;
5661; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_unordered_store:
5662; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5663; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5664; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
5665; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5666; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5667; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
5668; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5669; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5670;
5671; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_unordered_store:
5672; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5673; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5674; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
5675; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5676; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5677; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
5678; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5679; GFX940-TGSPLIT-NEXT:    s_endpgm
5680    i32 %in, i32* %out) {
5681entry:
5682  store atomic i32 %in, i32* %out syncscope("singlethread-one-as") unordered, align 4
5683  ret void
5684}
5685
5686define amdgpu_kernel void @flat_singlethread_one_as_monotonic_store(
5687; GFX7-LABEL: flat_singlethread_one_as_monotonic_store:
5688; GFX7:       ; %bb.0: ; %entry
5689; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
5690; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
5691; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5692; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5693; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5694; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5695; GFX7-NEXT:    flat_store_dword v[0:1], v2
5696; GFX7-NEXT:    s_endpgm
5697;
5698; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_store:
5699; GFX10-WGP:       ; %bb.0: ; %entry
5700; GFX10-WGP-NEXT:    s_clause 0x1
5701; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5702; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
5703; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5704; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5705; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5706; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5707; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5708; GFX10-WGP-NEXT:    s_endpgm
5709;
5710; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_store:
5711; GFX10-CU:       ; %bb.0: ; %entry
5712; GFX10-CU-NEXT:    s_clause 0x1
5713; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5714; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
5715; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5716; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5717; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5718; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5719; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5720; GFX10-CU-NEXT:    s_endpgm
5721;
5722; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_monotonic_store:
5723; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5724; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
5725; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
5726; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5727; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5728; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5729; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5730; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5731; SKIP-CACHE-INV-NEXT:    s_endpgm
5732;
5733; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_store:
5734; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5735; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5736; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
5737; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5738; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5739; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5740; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5741; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5742;
5743; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_store:
5744; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5745; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5746; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
5747; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5748; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5749; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5750; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5751; GFX90A-TGSPLIT-NEXT:    s_endpgm
5752;
5753; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_store:
5754; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5755; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5756; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
5757; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5758; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5759; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
5760; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5761; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5762;
5763; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_store:
5764; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5765; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5766; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
5767; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5768; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5769; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
5770; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5771; GFX940-TGSPLIT-NEXT:    s_endpgm
5772    i32 %in, i32* %out) {
5773entry:
5774  store atomic i32 %in, i32* %out syncscope("singlethread-one-as") monotonic, align 4
5775  ret void
5776}
5777
5778define amdgpu_kernel void @flat_singlethread_one_as_release_store(
5779; GFX7-LABEL: flat_singlethread_one_as_release_store:
5780; GFX7:       ; %bb.0: ; %entry
5781; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
5782; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
5783; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5784; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5785; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5786; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5787; GFX7-NEXT:    flat_store_dword v[0:1], v2
5788; GFX7-NEXT:    s_endpgm
5789;
5790; GFX10-WGP-LABEL: flat_singlethread_one_as_release_store:
5791; GFX10-WGP:       ; %bb.0: ; %entry
5792; GFX10-WGP-NEXT:    s_clause 0x1
5793; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5794; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
5795; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5796; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5797; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5798; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5799; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5800; GFX10-WGP-NEXT:    s_endpgm
5801;
5802; GFX10-CU-LABEL: flat_singlethread_one_as_release_store:
5803; GFX10-CU:       ; %bb.0: ; %entry
5804; GFX10-CU-NEXT:    s_clause 0x1
5805; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5806; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
5807; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5808; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5809; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5810; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5811; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5812; GFX10-CU-NEXT:    s_endpgm
5813;
5814; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_release_store:
5815; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5816; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
5817; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
5818; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5819; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5820; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5821; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5822; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5823; SKIP-CACHE-INV-NEXT:    s_endpgm
5824;
5825; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_store:
5826; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5827; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5828; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
5829; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5830; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5831; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5832; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5833; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5834;
5835; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_release_store:
5836; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5837; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5838; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
5839; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5840; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5841; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5842; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5843; GFX90A-TGSPLIT-NEXT:    s_endpgm
5844;
5845; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_store:
5846; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5847; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5848; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
5849; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5850; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5851; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
5852; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5853; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5854;
5855; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_release_store:
5856; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5857; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5858; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
5859; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5860; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5861; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
5862; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5863; GFX940-TGSPLIT-NEXT:    s_endpgm
5864    i32 %in, i32* %out) {
5865entry:
5866  store atomic i32 %in, i32* %out syncscope("singlethread-one-as") release, align 4
5867  ret void
5868}
5869
5870define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_store(
5871; GFX7-LABEL: flat_singlethread_one_as_seq_cst_store:
5872; GFX7:       ; %bb.0: ; %entry
5873; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
5874; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
5875; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5876; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5877; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5878; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5879; GFX7-NEXT:    flat_store_dword v[0:1], v2
5880; GFX7-NEXT:    s_endpgm
5881;
5882; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_store:
5883; GFX10-WGP:       ; %bb.0: ; %entry
5884; GFX10-WGP-NEXT:    s_clause 0x1
5885; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5886; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
5887; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5888; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5889; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5890; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5891; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5892; GFX10-WGP-NEXT:    s_endpgm
5893;
5894; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_store:
5895; GFX10-CU:       ; %bb.0: ; %entry
5896; GFX10-CU-NEXT:    s_clause 0x1
5897; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5898; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
5899; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5900; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5901; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5902; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5903; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5904; GFX10-CU-NEXT:    s_endpgm
5905;
5906; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_store:
5907; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5908; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
5909; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
5910; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5911; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5912; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5913; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5914; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5915; SKIP-CACHE-INV-NEXT:    s_endpgm
5916;
5917; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_store:
5918; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5919; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5920; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
5921; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5922; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5923; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5924; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5925; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5926;
5927; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_store:
5928; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5929; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5930; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
5931; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5932; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5933; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5934; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5935; GFX90A-TGSPLIT-NEXT:    s_endpgm
5936;
5937; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_store:
5938; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5939; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5940; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
5941; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5942; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5943; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
5944; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5945; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5946;
5947; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_store:
5948; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5949; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
5950; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
5951; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5952; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5953; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
5954; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5955; GFX940-TGSPLIT-NEXT:    s_endpgm
5956    i32 %in, i32* %out) {
5957entry:
5958  store atomic i32 %in, i32* %out syncscope("singlethread-one-as") seq_cst, align 4
5959  ret void
5960}
5961
5962define amdgpu_kernel void @flat_singlethread_one_as_monotonic_atomicrmw(
5963; GFX7-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
5964; GFX7:       ; %bb.0: ; %entry
5965; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5966; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
5967; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5968; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5969; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5970; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5971; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
5972; GFX7-NEXT:    s_endpgm
5973;
5974; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
5975; GFX10-WGP:       ; %bb.0: ; %entry
5976; GFX10-WGP-NEXT:    s_clause 0x1
5977; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5978; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
5979; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5980; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5981; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5982; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5983; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
5984; GFX10-WGP-NEXT:    s_endpgm
5985;
5986; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
5987; GFX10-CU:       ; %bb.0: ; %entry
5988; GFX10-CU-NEXT:    s_clause 0x1
5989; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5990; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
5991; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5992; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5993; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5994; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5995; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
5996; GFX10-CU-NEXT:    s_endpgm
5997;
5998; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
5999; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6000; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6001; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
6002; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6003; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6004; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6005; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6006; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
6007; SKIP-CACHE-INV-NEXT:    s_endpgm
6008;
6009; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
6010; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6011; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6012; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6013; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6014; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6015; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6016; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
6017; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6018;
6019; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
6020; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6021; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6022; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6023; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6024; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6025; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6026; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
6027; GFX90A-TGSPLIT-NEXT:    s_endpgm
6028;
6029; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
6030; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6031; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6032; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
6033; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6034; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6035; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
6036; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
6037; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6038;
6039; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
6040; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6041; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6042; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
6043; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6044; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6045; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
6046; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
6047; GFX940-TGSPLIT-NEXT:    s_endpgm
6048    i32* %out, i32 %in) {
6049entry:
6050  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") monotonic
6051  ret void
6052}
6053
6054define amdgpu_kernel void @flat_singlethread_one_as_acquire_atomicrmw(
6055; GFX7-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
6056; GFX7:       ; %bb.0: ; %entry
6057; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6058; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
6059; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6060; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6061; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6062; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6063; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
6064; GFX7-NEXT:    s_endpgm
6065;
6066; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
6067; GFX10-WGP:       ; %bb.0: ; %entry
6068; GFX10-WGP-NEXT:    s_clause 0x1
6069; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6070; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
6071; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6072; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6073; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6074; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6075; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
6076; GFX10-WGP-NEXT:    s_endpgm
6077;
6078; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
6079; GFX10-CU:       ; %bb.0: ; %entry
6080; GFX10-CU-NEXT:    s_clause 0x1
6081; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6082; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
6083; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6084; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6085; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6086; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6087; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
6088; GFX10-CU-NEXT:    s_endpgm
6089;
6090; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
6091; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6092; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6093; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
6094; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6095; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6096; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6097; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6098; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
6099; SKIP-CACHE-INV-NEXT:    s_endpgm
6100;
6101; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
6102; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6103; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6104; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6105; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6106; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6107; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6108; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
6109; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6110;
6111; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
6112; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6113; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6114; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6115; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6116; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6117; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6118; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
6119; GFX90A-TGSPLIT-NEXT:    s_endpgm
6120;
6121; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
6122; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6123; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6124; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
6125; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6126; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6127; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
6128; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
6129; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6130;
6131; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
6132; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6133; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6134; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
6135; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6136; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6137; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
6138; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
6139; GFX940-TGSPLIT-NEXT:    s_endpgm
6140    i32* %out, i32 %in) {
6141entry:
6142  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") acquire
6143  ret void
6144}
6145
6146define amdgpu_kernel void @flat_singlethread_one_as_release_atomicrmw(
6147; GFX7-LABEL: flat_singlethread_one_as_release_atomicrmw:
6148; GFX7:       ; %bb.0: ; %entry
6149; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6150; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
6151; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6152; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6153; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6154; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6155; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
6156; GFX7-NEXT:    s_endpgm
6157;
6158; GFX10-WGP-LABEL: flat_singlethread_one_as_release_atomicrmw:
6159; GFX10-WGP:       ; %bb.0: ; %entry
6160; GFX10-WGP-NEXT:    s_clause 0x1
6161; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6162; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
6163; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6164; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6165; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6166; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6167; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
6168; GFX10-WGP-NEXT:    s_endpgm
6169;
6170; GFX10-CU-LABEL: flat_singlethread_one_as_release_atomicrmw:
6171; GFX10-CU:       ; %bb.0: ; %entry
6172; GFX10-CU-NEXT:    s_clause 0x1
6173; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6174; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
6175; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6176; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6177; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6178; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6179; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
6180; GFX10-CU-NEXT:    s_endpgm
6181;
6182; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_release_atomicrmw:
6183; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6184; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6185; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
6186; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6187; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6188; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6189; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6190; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
6191; SKIP-CACHE-INV-NEXT:    s_endpgm
6192;
6193; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_atomicrmw:
6194; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6195; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6196; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6197; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6198; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6199; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6200; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
6201; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6202;
6203; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_release_atomicrmw:
6204; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6205; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6206; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6207; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6208; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6209; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6210; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
6211; GFX90A-TGSPLIT-NEXT:    s_endpgm
6212;
6213; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_atomicrmw:
6214; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6215; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6216; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
6217; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6218; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6219; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
6220; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
6221; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6222;
6223; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_release_atomicrmw:
6224; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6225; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6226; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
6227; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6228; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6229; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
6230; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
6231; GFX940-TGSPLIT-NEXT:    s_endpgm
6232    i32* %out, i32 %in) {
6233entry:
6234  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") release
6235  ret void
6236}
6237
6238define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_atomicrmw(
6239; GFX7-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
6240; GFX7:       ; %bb.0: ; %entry
6241; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6242; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
6243; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6244; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6245; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6246; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6247; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
6248; GFX7-NEXT:    s_endpgm
6249;
6250; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
6251; GFX10-WGP:       ; %bb.0: ; %entry
6252; GFX10-WGP-NEXT:    s_clause 0x1
6253; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6254; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
6255; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6256; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6257; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6258; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6259; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
6260; GFX10-WGP-NEXT:    s_endpgm
6261;
6262; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
6263; GFX10-CU:       ; %bb.0: ; %entry
6264; GFX10-CU-NEXT:    s_clause 0x1
6265; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6266; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
6267; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6268; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6269; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6270; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6271; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
6272; GFX10-CU-NEXT:    s_endpgm
6273;
6274; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
6275; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6276; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6277; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
6278; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6279; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6280; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6281; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6282; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
6283; SKIP-CACHE-INV-NEXT:    s_endpgm
6284;
6285; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
6286; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6287; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6288; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6289; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6290; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6291; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6292; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
6293; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6294;
6295; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
6296; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6297; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6298; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6299; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6300; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6301; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6302; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
6303; GFX90A-TGSPLIT-NEXT:    s_endpgm
6304;
6305; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
6306; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6307; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6308; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
6309; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6310; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6311; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
6312; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
6313; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6314;
6315; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
6316; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6317; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6318; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
6319; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6320; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6321; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
6322; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
6323; GFX940-TGSPLIT-NEXT:    s_endpgm
6324    i32* %out, i32 %in) {
6325entry:
6326  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") acq_rel
6327  ret void
6328}
6329
6330define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_atomicrmw(
6331; GFX7-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
6332; GFX7:       ; %bb.0: ; %entry
6333; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6334; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
6335; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6336; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6337; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6338; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6339; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
6340; GFX7-NEXT:    s_endpgm
6341;
6342; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
6343; GFX10-WGP:       ; %bb.0: ; %entry
6344; GFX10-WGP-NEXT:    s_clause 0x1
6345; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6346; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
6347; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6348; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6349; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6350; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6351; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
6352; GFX10-WGP-NEXT:    s_endpgm
6353;
6354; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
6355; GFX10-CU:       ; %bb.0: ; %entry
6356; GFX10-CU-NEXT:    s_clause 0x1
6357; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6358; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
6359; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6360; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6361; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6362; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6363; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
6364; GFX10-CU-NEXT:    s_endpgm
6365;
6366; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
6367; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6368; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6369; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
6370; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6371; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6372; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6373; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6374; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
6375; SKIP-CACHE-INV-NEXT:    s_endpgm
6376;
6377; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
6378; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6379; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6380; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6381; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6382; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6383; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6384; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
6385; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6386;
6387; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
6388; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6389; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6390; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6391; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6392; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6393; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6394; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
6395; GFX90A-TGSPLIT-NEXT:    s_endpgm
6396;
6397; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
6398; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6399; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6400; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
6401; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6402; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6403; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
6404; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
6405; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6406;
6407; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
6408; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6409; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6410; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
6411; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6412; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6413; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
6414; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
6415; GFX940-TGSPLIT-NEXT:    s_endpgm
6416    i32* %out, i32 %in) {
6417entry:
6418  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") seq_cst
6419  ret void
6420}
6421
6422define amdgpu_kernel void @flat_singlethread_one_as_acquire_ret_atomicrmw(
6423; GFX7-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
6424; GFX7:       ; %bb.0: ; %entry
6425; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6426; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
6427; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6428; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6429; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6430; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6431; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
6432; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6433; GFX7-NEXT:    flat_store_dword v[0:1], v2
6434; GFX7-NEXT:    s_endpgm
6435;
6436; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
6437; GFX10-WGP:       ; %bb.0: ; %entry
6438; GFX10-WGP-NEXT:    s_clause 0x1
6439; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6440; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
6441; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6442; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6443; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6444; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6445; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
6446; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6447; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6448; GFX10-WGP-NEXT:    s_endpgm
6449;
6450; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
6451; GFX10-CU:       ; %bb.0: ; %entry
6452; GFX10-CU-NEXT:    s_clause 0x1
6453; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6454; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
6455; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6456; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6457; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6458; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6459; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
6460; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6461; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6462; GFX10-CU-NEXT:    s_endpgm
6463;
6464; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
6465; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6466; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6467; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
6468; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6469; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6470; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6471; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6472; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
6473; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6474; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6475; SKIP-CACHE-INV-NEXT:    s_endpgm
6476;
6477; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
6478; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6479; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6480; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6481; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6482; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6483; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6484; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
6485; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6486; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6487; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6488;
6489; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
6490; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6491; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6492; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6493; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6494; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6495; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6496; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
6497; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6498; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6499; GFX90A-TGSPLIT-NEXT:    s_endpgm
6500;
6501; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
6502; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6503; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6504; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
6505; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6506; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6507; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
6508; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
6509; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6510; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6511; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6512;
6513; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
6514; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6515; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6516; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
6517; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6518; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6519; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
6520; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
6521; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6522; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6523; GFX940-TGSPLIT-NEXT:    s_endpgm
6524    i32* %out, i32 %in) {
6525entry:
6526  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") acquire
6527  store i32 %val, i32* %out, align 4
6528  ret void
6529}
6530
6531define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_ret_atomicrmw(
6532; GFX7-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
6533; GFX7:       ; %bb.0: ; %entry
6534; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6535; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
6536; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6537; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6538; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6539; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6540; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
6541; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6542; GFX7-NEXT:    flat_store_dword v[0:1], v2
6543; GFX7-NEXT:    s_endpgm
6544;
6545; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
6546; GFX10-WGP:       ; %bb.0: ; %entry
6547; GFX10-WGP-NEXT:    s_clause 0x1
6548; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6549; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
6550; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6551; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6552; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6553; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6554; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
6555; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6556; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6557; GFX10-WGP-NEXT:    s_endpgm
6558;
6559; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
6560; GFX10-CU:       ; %bb.0: ; %entry
6561; GFX10-CU-NEXT:    s_clause 0x1
6562; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6563; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
6564; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6565; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6566; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6567; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6568; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
6569; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6570; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6571; GFX10-CU-NEXT:    s_endpgm
6572;
6573; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
6574; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6575; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6576; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
6577; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6578; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6579; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6580; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6581; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
6582; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6583; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6584; SKIP-CACHE-INV-NEXT:    s_endpgm
6585;
6586; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
6587; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6588; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6589; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6590; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6591; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6592; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6593; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
6594; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6595; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6596; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6597;
6598; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
6599; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6600; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6601; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6602; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6603; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6604; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6605; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
6606; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6607; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6608; GFX90A-TGSPLIT-NEXT:    s_endpgm
6609;
6610; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
6611; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6612; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6613; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
6614; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6615; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6616; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
6617; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
6618; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6619; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6620; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6621;
6622; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
6623; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6624; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6625; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
6626; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6627; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6628; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
6629; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
6630; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6631; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6632; GFX940-TGSPLIT-NEXT:    s_endpgm
6633    i32* %out, i32 %in) {
6634entry:
6635  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") acq_rel
6636  store i32 %val, i32* %out, align 4
6637  ret void
6638}
6639
6640define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_ret_atomicrmw(
6641; GFX7-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
6642; GFX7:       ; %bb.0: ; %entry
6643; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6644; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
6645; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6646; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6647; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6648; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6649; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
6650; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6651; GFX7-NEXT:    flat_store_dword v[0:1], v2
6652; GFX7-NEXT:    s_endpgm
6653;
6654; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
6655; GFX10-WGP:       ; %bb.0: ; %entry
6656; GFX10-WGP-NEXT:    s_clause 0x1
6657; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6658; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
6659; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6660; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6661; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6662; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6663; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
6664; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6665; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6666; GFX10-WGP-NEXT:    s_endpgm
6667;
6668; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
6669; GFX10-CU:       ; %bb.0: ; %entry
6670; GFX10-CU-NEXT:    s_clause 0x1
6671; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6672; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
6673; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6674; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6675; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6676; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6677; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
6678; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6679; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6680; GFX10-CU-NEXT:    s_endpgm
6681;
6682; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
6683; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6684; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6685; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
6686; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6687; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6688; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6689; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6690; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
6691; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6692; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6693; SKIP-CACHE-INV-NEXT:    s_endpgm
6694;
6695; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
6696; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6697; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6698; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6699; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6700; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6701; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6702; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
6703; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6704; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6705; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6706;
6707; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
6708; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6709; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6710; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6711; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6712; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6713; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6714; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
6715; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6716; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6717; GFX90A-TGSPLIT-NEXT:    s_endpgm
6718;
6719; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
6720; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6721; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6722; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
6723; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6724; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6725; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
6726; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
6727; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6728; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6729; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6730;
6731; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
6732; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6733; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6734; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
6735; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6736; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6737; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
6738; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
6739; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6740; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6741; GFX940-TGSPLIT-NEXT:    s_endpgm
6742    i32* %out, i32 %in) {
6743entry:
6744  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") seq_cst
6745  store i32 %val, i32* %out, align 4
6746  ret void
6747}
6748
6749define amdgpu_kernel void @flat_singlethread_one_as_monotonic_monotonic_cmpxchg(
6750; GFX7-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
6751; GFX7:       ; %bb.0: ; %entry
6752; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6753; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6754; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6755; GFX7-NEXT:    s_add_u32 s0, s0, 16
6756; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6757; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6758; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6759; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6760; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6761; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6762; GFX7-NEXT:    s_endpgm
6763;
6764; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
6765; GFX10-WGP:       ; %bb.0: ; %entry
6766; GFX10-WGP-NEXT:    s_clause 0x1
6767; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6768; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6769; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6770; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
6771; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
6772; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6773; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6774; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6775; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6776; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6777; GFX10-WGP-NEXT:    s_endpgm
6778;
6779; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
6780; GFX10-CU:       ; %bb.0: ; %entry
6781; GFX10-CU-NEXT:    s_clause 0x1
6782; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6783; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6784; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6785; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
6786; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
6787; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6788; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6789; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6790; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6791; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6792; GFX10-CU-NEXT:    s_endpgm
6793;
6794; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
6795; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6796; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6797; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6798; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6799; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
6800; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
6801; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6802; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6803; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6804; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6805; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6806; SKIP-CACHE-INV-NEXT:    s_endpgm
6807;
6808; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
6809; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6810; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6811; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6812; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6813; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6814; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6815; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6816; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6817;
6818; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
6819; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6820; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6821; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6822; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6823; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6824; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6825; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6826; GFX90A-TGSPLIT-NEXT:    s_endpgm
6827;
6828; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
6829; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6830; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6831; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6832; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6833; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6834; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6835; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6836; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6837;
6838; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
6839; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6840; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6841; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6842; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6843; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6844; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6845; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6846; GFX940-TGSPLIT-NEXT:    s_endpgm
6847    i32* %out, i32 %in, i32 %old) {
6848entry:
6849  %gep = getelementptr i32, i32* %out, i32 4
6850  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic monotonic
6851  ret void
6852}
6853
6854define amdgpu_kernel void @flat_singlethread_one_as_acquire_monotonic_cmpxchg(
6855; GFX7-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
6856; GFX7:       ; %bb.0: ; %entry
6857; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6858; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6859; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6860; GFX7-NEXT:    s_add_u32 s0, s0, 16
6861; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6862; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6863; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6864; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6865; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6866; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6867; GFX7-NEXT:    s_endpgm
6868;
6869; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
6870; GFX10-WGP:       ; %bb.0: ; %entry
6871; GFX10-WGP-NEXT:    s_clause 0x1
6872; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6873; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6874; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6875; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
6876; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
6877; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6878; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6879; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6880; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6881; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6882; GFX10-WGP-NEXT:    s_endpgm
6883;
6884; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
6885; GFX10-CU:       ; %bb.0: ; %entry
6886; GFX10-CU-NEXT:    s_clause 0x1
6887; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6888; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6889; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6890; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
6891; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
6892; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6893; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6894; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6895; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6896; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6897; GFX10-CU-NEXT:    s_endpgm
6898;
6899; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
6900; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6901; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6902; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6903; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6904; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
6905; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
6906; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6907; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6908; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6909; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6910; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6911; SKIP-CACHE-INV-NEXT:    s_endpgm
6912;
6913; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
6914; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6915; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6916; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6917; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6918; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6919; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6920; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6921; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6922;
6923; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
6924; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6925; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6926; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6927; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6928; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6929; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6930; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6931; GFX90A-TGSPLIT-NEXT:    s_endpgm
6932;
6933; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
6934; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6935; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6936; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6937; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6938; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6939; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6940; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6941; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6942;
6943; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
6944; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6945; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6946; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6947; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6948; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6949; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6950; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6951; GFX940-TGSPLIT-NEXT:    s_endpgm
6952    i32* %out, i32 %in, i32 %old) {
6953entry:
6954  %gep = getelementptr i32, i32* %out, i32 4
6955  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire monotonic
6956  ret void
6957}
6958
6959define amdgpu_kernel void @flat_singlethread_one_as_release_monotonic_cmpxchg(
6960; GFX7-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
6961; GFX7:       ; %bb.0: ; %entry
6962; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6963; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6964; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6965; GFX7-NEXT:    s_add_u32 s0, s0, 16
6966; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6967; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6968; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6969; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6970; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6971; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6972; GFX7-NEXT:    s_endpgm
6973;
6974; GFX10-WGP-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
6975; GFX10-WGP:       ; %bb.0: ; %entry
6976; GFX10-WGP-NEXT:    s_clause 0x1
6977; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6978; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6979; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6980; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
6981; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
6982; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6983; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6984; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6985; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6986; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6987; GFX10-WGP-NEXT:    s_endpgm
6988;
6989; GFX10-CU-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
6990; GFX10-CU:       ; %bb.0: ; %entry
6991; GFX10-CU-NEXT:    s_clause 0x1
6992; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6993; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6994; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6995; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
6996; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
6997; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6998; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6999; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7000; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7001; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7002; GFX10-CU-NEXT:    s_endpgm
7003;
7004; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
7005; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7006; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7007; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7008; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7009; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
7010; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
7011; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7012; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7013; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7014; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7015; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7016; SKIP-CACHE-INV-NEXT:    s_endpgm
7017;
7018; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
7019; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7020; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7021; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7022; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7023; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7024; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7025; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7026; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7027;
7028; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
7029; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7030; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7031; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7032; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7033; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7034; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7035; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7036; GFX90A-TGSPLIT-NEXT:    s_endpgm
7037;
7038; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
7039; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7040; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7041; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7042; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7043; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7044; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7045; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7046; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7047;
7048; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
7049; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7050; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7051; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7052; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7053; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7054; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7055; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7056; GFX940-TGSPLIT-NEXT:    s_endpgm
7057    i32* %out, i32 %in, i32 %old) {
7058entry:
7059  %gep = getelementptr i32, i32* %out, i32 4
7060  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release monotonic
7061  ret void
7062}
7063
7064define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_monotonic_cmpxchg(
7065; GFX7-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
7066; GFX7:       ; %bb.0: ; %entry
7067; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7068; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7069; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7070; GFX7-NEXT:    s_add_u32 s0, s0, 16
7071; GFX7-NEXT:    s_addc_u32 s1, s1, 0
7072; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7073; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7074; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7075; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7076; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7077; GFX7-NEXT:    s_endpgm
7078;
7079; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
7080; GFX10-WGP:       ; %bb.0: ; %entry
7081; GFX10-WGP-NEXT:    s_clause 0x1
7082; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7083; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7084; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7085; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
7086; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
7087; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7088; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7089; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7090; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7091; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7092; GFX10-WGP-NEXT:    s_endpgm
7093;
7094; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
7095; GFX10-CU:       ; %bb.0: ; %entry
7096; GFX10-CU-NEXT:    s_clause 0x1
7097; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7098; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7099; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7100; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
7101; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
7102; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7103; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7104; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7105; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7106; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7107; GFX10-CU-NEXT:    s_endpgm
7108;
7109; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
7110; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7111; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7112; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7113; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7114; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
7115; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
7116; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7117; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7118; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7119; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7120; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7121; SKIP-CACHE-INV-NEXT:    s_endpgm
7122;
7123; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
7124; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7125; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7126; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7127; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7128; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7129; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7130; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7131; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7132;
7133; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
7134; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7135; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7136; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7137; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7138; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7139; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7140; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7141; GFX90A-TGSPLIT-NEXT:    s_endpgm
7142;
7143; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
7144; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7145; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7146; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7147; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7148; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7149; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7150; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7151; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7152;
7153; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
7154; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7155; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7156; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7157; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7158; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7159; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7160; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7161; GFX940-TGSPLIT-NEXT:    s_endpgm
7162    i32* %out, i32 %in, i32 %old) {
7163entry:
7164  %gep = getelementptr i32, i32* %out, i32 4
7165  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel monotonic
7166  ret void
7167}
7168
7169define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_monotonic_cmpxchg(
7170; GFX7-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
7171; GFX7:       ; %bb.0: ; %entry
7172; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7173; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7174; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7175; GFX7-NEXT:    s_add_u32 s0, s0, 16
7176; GFX7-NEXT:    s_addc_u32 s1, s1, 0
7177; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7178; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7179; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7180; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7181; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7182; GFX7-NEXT:    s_endpgm
7183;
7184; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
7185; GFX10-WGP:       ; %bb.0: ; %entry
7186; GFX10-WGP-NEXT:    s_clause 0x1
7187; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7188; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7189; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7190; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
7191; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
7192; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7193; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7194; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7195; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7196; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7197; GFX10-WGP-NEXT:    s_endpgm
7198;
7199; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
7200; GFX10-CU:       ; %bb.0: ; %entry
7201; GFX10-CU-NEXT:    s_clause 0x1
7202; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7203; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7204; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7205; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
7206; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
7207; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7208; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7209; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7210; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7211; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7212; GFX10-CU-NEXT:    s_endpgm
7213;
7214; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
7215; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7216; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7217; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7218; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7219; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
7220; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
7221; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7222; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7223; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7224; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7225; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7226; SKIP-CACHE-INV-NEXT:    s_endpgm
7227;
7228; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
7229; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7230; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7231; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7232; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7233; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7234; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7235; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7236; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7237;
7238; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
7239; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7240; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7241; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7242; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7243; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7244; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7245; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7246; GFX90A-TGSPLIT-NEXT:    s_endpgm
7247;
7248; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
7249; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7250; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7251; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7252; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7253; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7254; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7255; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7256; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7257;
7258; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
7259; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7260; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7261; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7262; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7263; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7264; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7265; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7266; GFX940-TGSPLIT-NEXT:    s_endpgm
7267    i32* %out, i32 %in, i32 %old) {
7268entry:
7269  %gep = getelementptr i32, i32* %out, i32 4
7270  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst monotonic
7271  ret void
7272}
7273
7274define amdgpu_kernel void @flat_singlethread_one_as_monotonic_acquire_cmpxchg(
7275; GFX7-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
7276; GFX7:       ; %bb.0: ; %entry
7277; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7278; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7279; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7280; GFX7-NEXT:    s_add_u32 s0, s0, 16
7281; GFX7-NEXT:    s_addc_u32 s1, s1, 0
7282; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7283; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7284; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7285; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7286; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7287; GFX7-NEXT:    s_endpgm
7288;
7289; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
7290; GFX10-WGP:       ; %bb.0: ; %entry
7291; GFX10-WGP-NEXT:    s_clause 0x1
7292; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7293; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7294; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7295; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
7296; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
7297; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7298; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7299; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7300; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7301; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7302; GFX10-WGP-NEXT:    s_endpgm
7303;
7304; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
7305; GFX10-CU:       ; %bb.0: ; %entry
7306; GFX10-CU-NEXT:    s_clause 0x1
7307; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7308; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7309; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7310; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
7311; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
7312; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7313; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7314; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7315; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7316; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7317; GFX10-CU-NEXT:    s_endpgm
7318;
7319; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
7320; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7321; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7322; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7323; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7324; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
7325; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
7326; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7327; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7328; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7329; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7330; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7331; SKIP-CACHE-INV-NEXT:    s_endpgm
7332;
7333; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
7334; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7335; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7336; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7337; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7338; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7339; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7340; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7341; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7342;
7343; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
7344; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7345; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7346; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7347; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7348; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7349; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7350; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7351; GFX90A-TGSPLIT-NEXT:    s_endpgm
7352;
7353; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
7354; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7355; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7356; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7357; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7358; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7359; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7360; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7361; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7362;
7363; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
7364; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7365; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7366; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7367; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7368; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7369; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7370; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7371; GFX940-TGSPLIT-NEXT:    s_endpgm
7372    i32* %out, i32 %in, i32 %old) {
7373entry:
7374  %gep = getelementptr i32, i32* %out, i32 4
7375  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic acquire
7376  ret void
7377}
7378
7379define amdgpu_kernel void @flat_singlethread_one_as_acquire_acquire_cmpxchg(
7380; GFX7-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
7381; GFX7:       ; %bb.0: ; %entry
7382; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7383; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7384; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7385; GFX7-NEXT:    s_add_u32 s0, s0, 16
7386; GFX7-NEXT:    s_addc_u32 s1, s1, 0
7387; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7388; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7389; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7390; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7391; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7392; GFX7-NEXT:    s_endpgm
7393;
7394; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
7395; GFX10-WGP:       ; %bb.0: ; %entry
7396; GFX10-WGP-NEXT:    s_clause 0x1
7397; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7398; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7399; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7400; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
7401; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
7402; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7403; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7404; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7405; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7406; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7407; GFX10-WGP-NEXT:    s_endpgm
7408;
7409; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
7410; GFX10-CU:       ; %bb.0: ; %entry
7411; GFX10-CU-NEXT:    s_clause 0x1
7412; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7413; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7414; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7415; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
7416; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
7417; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7418; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7419; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7420; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7421; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7422; GFX10-CU-NEXT:    s_endpgm
7423;
7424; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
7425; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7426; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7427; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7428; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7429; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
7430; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
7431; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7432; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7433; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7434; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7435; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7436; SKIP-CACHE-INV-NEXT:    s_endpgm
7437;
7438; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
7439; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7440; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7441; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7442; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7443; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7444; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7445; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7446; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7447;
7448; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
7449; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7450; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7451; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7452; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7453; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7454; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7455; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7456; GFX90A-TGSPLIT-NEXT:    s_endpgm
7457;
7458; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
7459; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7460; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7461; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7462; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7463; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7464; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7465; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7466; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7467;
7468; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
7469; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7470; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7471; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7472; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7473; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7474; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7475; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7476; GFX940-TGSPLIT-NEXT:    s_endpgm
7477    i32* %out, i32 %in, i32 %old) {
7478entry:
7479  %gep = getelementptr i32, i32* %out, i32 4
7480  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire acquire
7481  ret void
7482}
7483
7484define amdgpu_kernel void @flat_singlethread_one_as_release_acquire_cmpxchg(
7485; GFX7-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
7486; GFX7:       ; %bb.0: ; %entry
7487; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7488; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7489; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7490; GFX7-NEXT:    s_add_u32 s0, s0, 16
7491; GFX7-NEXT:    s_addc_u32 s1, s1, 0
7492; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7493; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7494; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7495; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7496; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7497; GFX7-NEXT:    s_endpgm
7498;
7499; GFX10-WGP-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
7500; GFX10-WGP:       ; %bb.0: ; %entry
7501; GFX10-WGP-NEXT:    s_clause 0x1
7502; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7503; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7504; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7505; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
7506; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
7507; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7508; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7509; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7510; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7511; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7512; GFX10-WGP-NEXT:    s_endpgm
7513;
7514; GFX10-CU-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
7515; GFX10-CU:       ; %bb.0: ; %entry
7516; GFX10-CU-NEXT:    s_clause 0x1
7517; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7518; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7519; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7520; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
7521; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
7522; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7523; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7524; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7525; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7526; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7527; GFX10-CU-NEXT:    s_endpgm
7528;
7529; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
7530; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7531; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7532; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7533; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7534; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
7535; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
7536; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7537; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7538; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7539; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7540; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7541; SKIP-CACHE-INV-NEXT:    s_endpgm
7542;
7543; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
7544; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7545; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7546; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7547; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7548; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7549; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7550; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7551; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7552;
7553; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
7554; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7555; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7556; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7557; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7558; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7559; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7560; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7561; GFX90A-TGSPLIT-NEXT:    s_endpgm
7562;
7563; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
7564; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7565; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7566; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7567; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7568; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7569; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7570; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7571; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7572;
7573; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
7574; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7575; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7576; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7577; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7578; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7579; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7580; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7581; GFX940-TGSPLIT-NEXT:    s_endpgm
7582    i32* %out, i32 %in, i32 %old) {
7583entry:
7584  %gep = getelementptr i32, i32* %out, i32 4
7585  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release acquire
7586  ret void
7587}
7588
7589define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_acquire_cmpxchg(
7590; GFX7-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
7591; GFX7:       ; %bb.0: ; %entry
7592; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7593; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7594; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7595; GFX7-NEXT:    s_add_u32 s0, s0, 16
7596; GFX7-NEXT:    s_addc_u32 s1, s1, 0
7597; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7598; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7599; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7600; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7601; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7602; GFX7-NEXT:    s_endpgm
7603;
7604; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
7605; GFX10-WGP:       ; %bb.0: ; %entry
7606; GFX10-WGP-NEXT:    s_clause 0x1
7607; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7608; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7609; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7610; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
7611; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
7612; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7613; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7614; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7615; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7616; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7617; GFX10-WGP-NEXT:    s_endpgm
7618;
7619; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
7620; GFX10-CU:       ; %bb.0: ; %entry
7621; GFX10-CU-NEXT:    s_clause 0x1
7622; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7623; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7624; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7625; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
7626; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
7627; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7628; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7629; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7630; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7631; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7632; GFX10-CU-NEXT:    s_endpgm
7633;
7634; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
7635; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7636; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7637; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7638; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7639; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
7640; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
7641; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7642; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7643; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7644; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7645; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7646; SKIP-CACHE-INV-NEXT:    s_endpgm
7647;
7648; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
7649; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7650; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7651; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7652; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7653; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7654; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7655; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7656; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7657;
7658; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
7659; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7660; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7661; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7662; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7663; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7664; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7665; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7666; GFX90A-TGSPLIT-NEXT:    s_endpgm
7667;
7668; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
7669; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7670; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7671; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7672; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7673; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7674; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7675; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7676; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7677;
7678; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
7679; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7680; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7681; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7682; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7683; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7684; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7685; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7686; GFX940-TGSPLIT-NEXT:    s_endpgm
7687    i32* %out, i32 %in, i32 %old) {
7688entry:
7689  %gep = getelementptr i32, i32* %out, i32 4
7690  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel acquire
7691  ret void
7692}
7693
7694define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_acquire_cmpxchg(
7695; GFX7-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
7696; GFX7:       ; %bb.0: ; %entry
7697; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7698; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7699; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7700; GFX7-NEXT:    s_add_u32 s0, s0, 16
7701; GFX7-NEXT:    s_addc_u32 s1, s1, 0
7702; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7703; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7704; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7705; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7706; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7707; GFX7-NEXT:    s_endpgm
7708;
7709; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
7710; GFX10-WGP:       ; %bb.0: ; %entry
7711; GFX10-WGP-NEXT:    s_clause 0x1
7712; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7713; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7714; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7715; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
7716; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
7717; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7718; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7719; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7720; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7721; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7722; GFX10-WGP-NEXT:    s_endpgm
7723;
7724; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
7725; GFX10-CU:       ; %bb.0: ; %entry
7726; GFX10-CU-NEXT:    s_clause 0x1
7727; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7728; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7729; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7730; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
7731; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
7732; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7733; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7734; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7735; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7736; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7737; GFX10-CU-NEXT:    s_endpgm
7738;
7739; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
7740; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7741; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7742; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7743; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7744; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
7745; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
7746; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7747; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7748; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7749; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7750; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7751; SKIP-CACHE-INV-NEXT:    s_endpgm
7752;
7753; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
7754; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7755; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7756; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7757; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7758; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7759; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7760; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7761; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7762;
7763; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
7764; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7765; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7766; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7767; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7768; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7769; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7770; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7771; GFX90A-TGSPLIT-NEXT:    s_endpgm
7772;
7773; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
7774; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7775; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7776; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7777; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7778; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7779; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7780; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7781; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7782;
7783; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
7784; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7785; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7786; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7787; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7788; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7789; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7790; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7791; GFX940-TGSPLIT-NEXT:    s_endpgm
7792    i32* %out, i32 %in, i32 %old) {
7793entry:
7794  %gep = getelementptr i32, i32* %out, i32 4
7795  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst acquire
7796  ret void
7797}
7798
7799define amdgpu_kernel void @flat_singlethread_one_as_monotonic_seq_cst_cmpxchg(
7800; GFX7-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
7801; GFX7:       ; %bb.0: ; %entry
7802; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7803; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7804; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7805; GFX7-NEXT:    s_add_u32 s0, s0, 16
7806; GFX7-NEXT:    s_addc_u32 s1, s1, 0
7807; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7808; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7809; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7810; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7811; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7812; GFX7-NEXT:    s_endpgm
7813;
7814; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
7815; GFX10-WGP:       ; %bb.0: ; %entry
7816; GFX10-WGP-NEXT:    s_clause 0x1
7817; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7818; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7819; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7820; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
7821; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
7822; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7823; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7824; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7825; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7826; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7827; GFX10-WGP-NEXT:    s_endpgm
7828;
7829; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
7830; GFX10-CU:       ; %bb.0: ; %entry
7831; GFX10-CU-NEXT:    s_clause 0x1
7832; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7833; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7834; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7835; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
7836; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
7837; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7838; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7839; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7840; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7841; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7842; GFX10-CU-NEXT:    s_endpgm
7843;
7844; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
7845; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7846; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7847; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7848; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7849; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
7850; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
7851; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7852; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7853; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7854; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7855; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7856; SKIP-CACHE-INV-NEXT:    s_endpgm
7857;
7858; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
7859; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7860; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7861; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7862; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7863; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7864; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7865; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7866; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7867;
7868; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
7869; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7870; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7871; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7872; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7873; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7874; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7875; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7876; GFX90A-TGSPLIT-NEXT:    s_endpgm
7877;
7878; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
7879; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7880; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7881; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7882; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7883; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7884; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7885; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7886; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7887;
7888; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
7889; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7890; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7891; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7892; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7893; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7894; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7895; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7896; GFX940-TGSPLIT-NEXT:    s_endpgm
7897    i32* %out, i32 %in, i32 %old) {
7898entry:
7899  %gep = getelementptr i32, i32* %out, i32 4
7900  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic seq_cst
7901  ret void
7902}
7903
7904define amdgpu_kernel void @flat_singlethread_one_as_acquire_seq_cst_cmpxchg(
7905; GFX7-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
7906; GFX7:       ; %bb.0: ; %entry
7907; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7908; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7909; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7910; GFX7-NEXT:    s_add_u32 s0, s0, 16
7911; GFX7-NEXT:    s_addc_u32 s1, s1, 0
7912; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7913; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7914; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7915; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7916; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7917; GFX7-NEXT:    s_endpgm
7918;
7919; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
7920; GFX10-WGP:       ; %bb.0: ; %entry
7921; GFX10-WGP-NEXT:    s_clause 0x1
7922; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7923; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7924; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7925; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
7926; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
7927; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7928; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7929; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7930; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7931; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7932; GFX10-WGP-NEXT:    s_endpgm
7933;
7934; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
7935; GFX10-CU:       ; %bb.0: ; %entry
7936; GFX10-CU-NEXT:    s_clause 0x1
7937; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7938; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7939; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7940; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
7941; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
7942; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7943; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7944; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7945; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7946; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7947; GFX10-CU-NEXT:    s_endpgm
7948;
7949; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
7950; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7951; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7952; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7953; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7954; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
7955; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
7956; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7957; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7958; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7959; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7960; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7961; SKIP-CACHE-INV-NEXT:    s_endpgm
7962;
7963; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
7964; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7965; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7966; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7967; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7968; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7969; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7970; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7971; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7972;
7973; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
7974; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7975; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7976; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7977; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7978; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7979; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7980; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7981; GFX90A-TGSPLIT-NEXT:    s_endpgm
7982;
7983; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
7984; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7985; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7986; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7987; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7988; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7989; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7990; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7991; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7992;
7993; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
7994; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7995; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7996; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7997; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7998; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7999; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8000; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8001; GFX940-TGSPLIT-NEXT:    s_endpgm
8002    i32* %out, i32 %in, i32 %old) {
8003entry:
8004  %gep = getelementptr i32, i32* %out, i32 4
8005  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire seq_cst
8006  ret void
8007}
8008
8009define amdgpu_kernel void @flat_singlethread_one_as_release_seq_cst_cmpxchg(
8010; GFX7-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
8011; GFX7:       ; %bb.0: ; %entry
8012; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8013; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8014; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8015; GFX7-NEXT:    s_add_u32 s0, s0, 16
8016; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8017; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8018; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8019; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8020; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8021; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8022; GFX7-NEXT:    s_endpgm
8023;
8024; GFX10-WGP-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
8025; GFX10-WGP:       ; %bb.0: ; %entry
8026; GFX10-WGP-NEXT:    s_clause 0x1
8027; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8028; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8029; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8030; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
8031; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
8032; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8033; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8034; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8035; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8036; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8037; GFX10-WGP-NEXT:    s_endpgm
8038;
8039; GFX10-CU-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
8040; GFX10-CU:       ; %bb.0: ; %entry
8041; GFX10-CU-NEXT:    s_clause 0x1
8042; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8043; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8044; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8045; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8046; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8047; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8048; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8049; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8050; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8051; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8052; GFX10-CU-NEXT:    s_endpgm
8053;
8054; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
8055; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8056; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8057; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8058; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8059; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
8060; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
8061; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8062; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8063; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8064; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8065; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8066; SKIP-CACHE-INV-NEXT:    s_endpgm
8067;
8068; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
8069; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8070; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8071; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8072; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8073; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8074; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8075; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8076; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8077;
8078; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
8079; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8080; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8081; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8082; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8083; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8084; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8085; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8086; GFX90A-TGSPLIT-NEXT:    s_endpgm
8087;
8088; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
8089; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8090; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8091; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8092; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8093; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8094; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8095; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8096; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8097;
8098; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
8099; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8100; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8101; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8102; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8103; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8104; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8105; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8106; GFX940-TGSPLIT-NEXT:    s_endpgm
8107    i32* %out, i32 %in, i32 %old) {
8108entry:
8109  %gep = getelementptr i32, i32* %out, i32 4
8110  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release seq_cst
8111  ret void
8112}
8113
8114define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg(
8115; GFX7-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
8116; GFX7:       ; %bb.0: ; %entry
8117; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8118; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8119; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8120; GFX7-NEXT:    s_add_u32 s0, s0, 16
8121; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8122; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8123; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8124; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8125; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8126; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8127; GFX7-NEXT:    s_endpgm
8128;
8129; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
8130; GFX10-WGP:       ; %bb.0: ; %entry
8131; GFX10-WGP-NEXT:    s_clause 0x1
8132; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8133; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8134; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8135; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
8136; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
8137; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8138; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8139; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8140; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8141; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8142; GFX10-WGP-NEXT:    s_endpgm
8143;
8144; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
8145; GFX10-CU:       ; %bb.0: ; %entry
8146; GFX10-CU-NEXT:    s_clause 0x1
8147; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8148; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8149; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8150; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8151; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8152; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8153; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8154; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8155; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8156; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8157; GFX10-CU-NEXT:    s_endpgm
8158;
8159; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
8160; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8161; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8162; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8163; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8164; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
8165; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
8166; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8167; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8168; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8169; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8170; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8171; SKIP-CACHE-INV-NEXT:    s_endpgm
8172;
8173; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
8174; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8175; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8176; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8177; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8178; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8179; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8180; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8181; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8182;
8183; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
8184; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8185; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8186; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8187; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8188; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8189; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8190; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8191; GFX90A-TGSPLIT-NEXT:    s_endpgm
8192;
8193; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
8194; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8195; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8196; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8197; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8198; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8199; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8200; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8201; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8202;
8203; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
8204; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8205; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8206; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8207; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8208; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8209; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8210; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8211; GFX940-TGSPLIT-NEXT:    s_endpgm
8212    i32* %out, i32 %in, i32 %old) {
8213entry:
8214  %gep = getelementptr i32, i32* %out, i32 4
8215  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel seq_cst
8216  ret void
8217}
8218
8219define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg(
8220; GFX7-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
8221; GFX7:       ; %bb.0: ; %entry
8222; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8223; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8224; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8225; GFX7-NEXT:    s_add_u32 s0, s0, 16
8226; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8227; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8228; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8229; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8230; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8231; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8232; GFX7-NEXT:    s_endpgm
8233;
8234; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
8235; GFX10-WGP:       ; %bb.0: ; %entry
8236; GFX10-WGP-NEXT:    s_clause 0x1
8237; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8238; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8239; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8240; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
8241; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
8242; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8243; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8244; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8245; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8246; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8247; GFX10-WGP-NEXT:    s_endpgm
8248;
8249; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
8250; GFX10-CU:       ; %bb.0: ; %entry
8251; GFX10-CU-NEXT:    s_clause 0x1
8252; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8253; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8254; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8255; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8256; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8257; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8258; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8259; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8260; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8261; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8262; GFX10-CU-NEXT:    s_endpgm
8263;
8264; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
8265; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8266; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8267; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8268; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8269; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
8270; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
8271; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8272; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8273; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8274; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8275; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8276; SKIP-CACHE-INV-NEXT:    s_endpgm
8277;
8278; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
8279; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8280; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8281; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8282; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8283; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8284; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8285; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8286; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8287;
8288; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
8289; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8290; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8291; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8292; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8293; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8294; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8295; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8296; GFX90A-TGSPLIT-NEXT:    s_endpgm
8297;
8298; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
8299; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8300; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8301; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8302; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8303; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8304; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8305; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8306; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8307;
8308; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
8309; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8310; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8311; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8312; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8313; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8314; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8315; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8316; GFX940-TGSPLIT-NEXT:    s_endpgm
8317    i32* %out, i32 %in, i32 %old) {
8318entry:
8319  %gep = getelementptr i32, i32* %out, i32 4
8320  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst seq_cst
8321  ret void
8322}
8323
8324define amdgpu_kernel void @flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg(
8325; GFX7-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
8326; GFX7:       ; %bb.0: ; %entry
8327; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8328; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8329; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8330; GFX7-NEXT:    s_add_u32 s4, s0, 16
8331; GFX7-NEXT:    s_addc_u32 s5, s1, 0
8332; GFX7-NEXT:    v_mov_b32_e32 v0, s4
8333; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8334; GFX7-NEXT:    v_mov_b32_e32 v1, s5
8335; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8336; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8337; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8338; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8339; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8340; GFX7-NEXT:    flat_store_dword v[0:1], v2
8341; GFX7-NEXT:    s_endpgm
8342;
8343; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
8344; GFX10-WGP:       ; %bb.0: ; %entry
8345; GFX10-WGP-NEXT:    s_clause 0x1
8346; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8347; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8348; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8349; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
8350; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
8351; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
8352; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8353; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
8354; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8355; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8356; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8357; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8358; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8359; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8360; GFX10-WGP-NEXT:    s_endpgm
8361;
8362; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
8363; GFX10-CU:       ; %bb.0: ; %entry
8364; GFX10-CU-NEXT:    s_clause 0x1
8365; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8366; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8367; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8368; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
8369; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
8370; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
8371; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8372; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
8373; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8374; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8375; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8376; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8377; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8378; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8379; GFX10-CU-NEXT:    s_endpgm
8380;
8381; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
8382; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8383; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8384; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8385; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8386; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
8387; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
8388; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8389; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8390; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8391; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8392; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8393; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8394; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8395; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8396; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8397; SKIP-CACHE-INV-NEXT:    s_endpgm
8398;
8399; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
8400; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8401; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8402; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8403; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8404; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8405; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8406; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8407; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8408; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8409; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8410;
8411; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
8412; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8413; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8414; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8415; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8416; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8417; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8418; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8419; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8420; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8421; GFX90A-TGSPLIT-NEXT:    s_endpgm
8422;
8423; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
8424; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8425; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8426; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8427; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8428; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8429; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8430; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
8431; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8432; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8433; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8434;
8435; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
8436; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8437; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8438; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8439; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8440; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8441; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8442; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
8443; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8444; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8445; GFX940-TGSPLIT-NEXT:    s_endpgm
8446    i32* %out, i32 %in, i32 %old) {
8447entry:
8448  %gep = getelementptr i32, i32* %out, i32 4
8449  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic monotonic
8450  %val0 = extractvalue { i32, i1 } %val, 0
8451  store i32 %val0, i32* %out, align 4
8452  ret void
8453}
8454
8455define amdgpu_kernel void @flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg(
8456; GFX7-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
8457; GFX7:       ; %bb.0: ; %entry
8458; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8459; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8460; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8461; GFX7-NEXT:    s_add_u32 s4, s0, 16
8462; GFX7-NEXT:    s_addc_u32 s5, s1, 0
8463; GFX7-NEXT:    v_mov_b32_e32 v0, s4
8464; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8465; GFX7-NEXT:    v_mov_b32_e32 v1, s5
8466; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8467; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8468; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8469; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8470; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8471; GFX7-NEXT:    flat_store_dword v[0:1], v2
8472; GFX7-NEXT:    s_endpgm
8473;
8474; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
8475; GFX10-WGP:       ; %bb.0: ; %entry
8476; GFX10-WGP-NEXT:    s_clause 0x1
8477; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8478; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8479; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8480; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
8481; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
8482; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
8483; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8484; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
8485; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8486; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8487; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8488; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8489; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8490; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8491; GFX10-WGP-NEXT:    s_endpgm
8492;
8493; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
8494; GFX10-CU:       ; %bb.0: ; %entry
8495; GFX10-CU-NEXT:    s_clause 0x1
8496; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8497; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8498; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8499; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
8500; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
8501; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
8502; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8503; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
8504; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8505; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8506; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8507; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8508; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8509; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8510; GFX10-CU-NEXT:    s_endpgm
8511;
8512; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
8513; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8514; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8515; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8516; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8517; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
8518; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
8519; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8520; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8521; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8522; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8523; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8524; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8525; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8526; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8527; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8528; SKIP-CACHE-INV-NEXT:    s_endpgm
8529;
8530; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
8531; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8532; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8533; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8534; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8535; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8536; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8537; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8538; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8539; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8540; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8541;
8542; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
8543; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8544; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8545; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8546; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8547; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8548; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8549; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8550; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8551; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8552; GFX90A-TGSPLIT-NEXT:    s_endpgm
8553;
8554; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
8555; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8556; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8557; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8558; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8559; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8560; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8561; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
8562; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8563; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8564; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8565;
8566; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
8567; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8568; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8569; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8570; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8571; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8572; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8573; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
8574; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8575; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8576; GFX940-TGSPLIT-NEXT:    s_endpgm
8577    i32* %out, i32 %in, i32 %old) {
8578entry:
8579  %gep = getelementptr i32, i32* %out, i32 4
8580  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire monotonic
8581  %val0 = extractvalue { i32, i1 } %val, 0
8582  store i32 %val0, i32* %out, align 4
8583  ret void
8584}
8585
8586define amdgpu_kernel void @flat_singlethread_one_as_release_monotonic_ret_cmpxchg(
8587; GFX7-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
8588; GFX7:       ; %bb.0: ; %entry
8589; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8590; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8591; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8592; GFX7-NEXT:    s_add_u32 s4, s0, 16
8593; GFX7-NEXT:    s_addc_u32 s5, s1, 0
8594; GFX7-NEXT:    v_mov_b32_e32 v0, s4
8595; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8596; GFX7-NEXT:    v_mov_b32_e32 v1, s5
8597; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8598; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8599; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8600; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8601; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8602; GFX7-NEXT:    flat_store_dword v[0:1], v2
8603; GFX7-NEXT:    s_endpgm
8604;
8605; GFX10-WGP-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
8606; GFX10-WGP:       ; %bb.0: ; %entry
8607; GFX10-WGP-NEXT:    s_clause 0x1
8608; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8609; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8610; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8611; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
8612; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
8613; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
8614; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8615; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
8616; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8617; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8618; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8619; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8620; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8621; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8622; GFX10-WGP-NEXT:    s_endpgm
8623;
8624; GFX10-CU-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
8625; GFX10-CU:       ; %bb.0: ; %entry
8626; GFX10-CU-NEXT:    s_clause 0x1
8627; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8628; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8629; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8630; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
8631; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
8632; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
8633; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8634; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
8635; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8636; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8637; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8638; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8639; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8640; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8641; GFX10-CU-NEXT:    s_endpgm
8642;
8643; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
8644; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8645; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8646; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8647; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8648; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
8649; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
8650; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8651; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8652; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8653; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8654; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8655; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8656; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8657; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8658; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8659; SKIP-CACHE-INV-NEXT:    s_endpgm
8660;
8661; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
8662; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8663; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8664; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8665; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8666; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8667; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8668; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8669; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8670; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8671; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8672;
8673; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
8674; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8675; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8676; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8677; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8678; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8679; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8680; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8681; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8682; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8683; GFX90A-TGSPLIT-NEXT:    s_endpgm
8684;
8685; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
8686; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8687; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8688; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8689; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8690; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8691; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8692; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
8693; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8694; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8695; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8696;
8697; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
8698; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8699; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8700; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8701; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8702; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8703; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8704; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
8705; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8706; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8707; GFX940-TGSPLIT-NEXT:    s_endpgm
8708    i32* %out, i32 %in, i32 %old) {
8709entry:
8710  %gep = getelementptr i32, i32* %out, i32 4
8711  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release monotonic
8712  %val0 = extractvalue { i32, i1 } %val, 0
8713  store i32 %val0, i32* %out, align 4
8714  ret void
8715}
8716
8717define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg(
8718; GFX7-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
8719; GFX7:       ; %bb.0: ; %entry
8720; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8721; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8722; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8723; GFX7-NEXT:    s_add_u32 s4, s0, 16
8724; GFX7-NEXT:    s_addc_u32 s5, s1, 0
8725; GFX7-NEXT:    v_mov_b32_e32 v0, s4
8726; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8727; GFX7-NEXT:    v_mov_b32_e32 v1, s5
8728; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8729; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8730; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8731; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8732; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8733; GFX7-NEXT:    flat_store_dword v[0:1], v2
8734; GFX7-NEXT:    s_endpgm
8735;
8736; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
8737; GFX10-WGP:       ; %bb.0: ; %entry
8738; GFX10-WGP-NEXT:    s_clause 0x1
8739; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8740; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8741; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8742; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
8743; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
8744; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
8745; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8746; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
8747; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8748; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8749; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8750; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8751; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8752; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8753; GFX10-WGP-NEXT:    s_endpgm
8754;
8755; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
8756; GFX10-CU:       ; %bb.0: ; %entry
8757; GFX10-CU-NEXT:    s_clause 0x1
8758; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8759; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8760; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8761; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
8762; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
8763; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
8764; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8765; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
8766; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8767; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8768; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8769; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8770; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8771; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8772; GFX10-CU-NEXT:    s_endpgm
8773;
8774; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
8775; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8776; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8777; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8778; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8779; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
8780; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
8781; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8782; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8783; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8784; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8785; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8786; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8787; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8788; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8789; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8790; SKIP-CACHE-INV-NEXT:    s_endpgm
8791;
8792; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
8793; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8794; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8795; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8796; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8797; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8798; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8799; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8800; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8801; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8802; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8803;
8804; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
8805; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8806; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8807; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8808; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8809; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8810; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8811; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8812; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8813; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8814; GFX90A-TGSPLIT-NEXT:    s_endpgm
8815;
8816; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
8817; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8818; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8819; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8820; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8821; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8822; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8823; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
8824; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8825; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8826; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8827;
8828; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
8829; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8830; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8831; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8832; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8833; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8834; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8835; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
8836; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8837; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8838; GFX940-TGSPLIT-NEXT:    s_endpgm
8839    i32* %out, i32 %in, i32 %old) {
8840entry:
8841  %gep = getelementptr i32, i32* %out, i32 4
8842  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel monotonic
8843  %val0 = extractvalue { i32, i1 } %val, 0
8844  store i32 %val0, i32* %out, align 4
8845  ret void
8846}
8847
8848define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg(
8849; GFX7-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
8850; GFX7:       ; %bb.0: ; %entry
8851; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8852; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8853; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8854; GFX7-NEXT:    s_add_u32 s4, s0, 16
8855; GFX7-NEXT:    s_addc_u32 s5, s1, 0
8856; GFX7-NEXT:    v_mov_b32_e32 v0, s4
8857; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8858; GFX7-NEXT:    v_mov_b32_e32 v1, s5
8859; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8860; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8861; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8862; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8863; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8864; GFX7-NEXT:    flat_store_dword v[0:1], v2
8865; GFX7-NEXT:    s_endpgm
8866;
8867; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
8868; GFX10-WGP:       ; %bb.0: ; %entry
8869; GFX10-WGP-NEXT:    s_clause 0x1
8870; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8871; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8872; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8873; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
8874; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
8875; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
8876; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8877; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
8878; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8879; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8880; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8881; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8882; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8883; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8884; GFX10-WGP-NEXT:    s_endpgm
8885;
8886; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
8887; GFX10-CU:       ; %bb.0: ; %entry
8888; GFX10-CU-NEXT:    s_clause 0x1
8889; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8890; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8891; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8892; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
8893; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
8894; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
8895; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8896; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
8897; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8898; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8899; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8900; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8901; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8902; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8903; GFX10-CU-NEXT:    s_endpgm
8904;
8905; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
8906; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8907; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8908; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8909; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8910; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
8911; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
8912; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8913; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8914; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8915; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8916; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8917; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8918; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8919; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8920; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8921; SKIP-CACHE-INV-NEXT:    s_endpgm
8922;
8923; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
8924; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8925; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8926; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8927; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8928; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8929; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8930; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8931; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8932; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8933; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8934;
8935; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
8936; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8937; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8938; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8939; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8940; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8941; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8942; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8943; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8944; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8945; GFX90A-TGSPLIT-NEXT:    s_endpgm
8946;
8947; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
8948; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8949; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8950; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8951; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8952; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8953; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8954; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
8955; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8956; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8957; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8958;
8959; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
8960; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8961; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8962; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8963; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8964; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8965; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8966; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
8967; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8968; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8969; GFX940-TGSPLIT-NEXT:    s_endpgm
8970    i32* %out, i32 %in, i32 %old) {
8971entry:
8972  %gep = getelementptr i32, i32* %out, i32 4
8973  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst monotonic
8974  %val0 = extractvalue { i32, i1 } %val, 0
8975  store i32 %val0, i32* %out, align 4
8976  ret void
8977}
8978
8979define amdgpu_kernel void @flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg(
8980; GFX7-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
8981; GFX7:       ; %bb.0: ; %entry
8982; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8983; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8984; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8985; GFX7-NEXT:    s_add_u32 s4, s0, 16
8986; GFX7-NEXT:    s_addc_u32 s5, s1, 0
8987; GFX7-NEXT:    v_mov_b32_e32 v0, s4
8988; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8989; GFX7-NEXT:    v_mov_b32_e32 v1, s5
8990; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8991; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8992; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8993; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8994; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
8995; GFX7-NEXT:    flat_store_dword v[0:1], v2
8996; GFX7-NEXT:    s_endpgm
8997;
8998; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
8999; GFX10-WGP:       ; %bb.0: ; %entry
9000; GFX10-WGP-NEXT:    s_clause 0x1
9001; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9002; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9003; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9004; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
9005; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
9006; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
9007; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9008; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
9009; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9010; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9011; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9012; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9013; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9014; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
9015; GFX10-WGP-NEXT:    s_endpgm
9016;
9017; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
9018; GFX10-CU:       ; %bb.0: ; %entry
9019; GFX10-CU-NEXT:    s_clause 0x1
9020; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9021; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9022; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9023; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
9024; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
9025; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
9026; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9027; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
9028; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9029; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9030; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9031; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9032; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9033; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
9034; GFX10-CU-NEXT:    s_endpgm
9035;
9036; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
9037; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9038; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9039; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9040; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9041; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
9042; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
9043; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
9044; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9045; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
9046; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9047; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9048; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9049; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9050; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9051; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
9052; SKIP-CACHE-INV-NEXT:    s_endpgm
9053;
9054; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
9055; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9056; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9057; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9058; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9059; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9060; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9061; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9062; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9063; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9064; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9065;
9066; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
9067; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9068; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9069; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9070; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9071; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9072; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9073; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9074; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9075; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9076; GFX90A-TGSPLIT-NEXT:    s_endpgm
9077;
9078; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
9079; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9080; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9081; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9082; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9083; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9084; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9085; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9086; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9087; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9088; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9089;
9090; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
9091; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9092; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9093; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9094; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9095; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9096; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9097; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9098; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9099; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9100; GFX940-TGSPLIT-NEXT:    s_endpgm
9101    i32* %out, i32 %in, i32 %old) {
9102entry:
9103  %gep = getelementptr i32, i32* %out, i32 4
9104  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic acquire
9105  %val0 = extractvalue { i32, i1 } %val, 0
9106  store i32 %val0, i32* %out, align 4
9107  ret void
9108}
9109
9110define amdgpu_kernel void @flat_singlethread_one_as_acquire_acquire_ret_cmpxchg(
9111; GFX7-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
9112; GFX7:       ; %bb.0: ; %entry
9113; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9114; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9115; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9116; GFX7-NEXT:    s_add_u32 s4, s0, 16
9117; GFX7-NEXT:    s_addc_u32 s5, s1, 0
9118; GFX7-NEXT:    v_mov_b32_e32 v0, s4
9119; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9120; GFX7-NEXT:    v_mov_b32_e32 v1, s5
9121; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9122; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9123; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9124; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9125; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9126; GFX7-NEXT:    flat_store_dword v[0:1], v2
9127; GFX7-NEXT:    s_endpgm
9128;
9129; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
9130; GFX10-WGP:       ; %bb.0: ; %entry
9131; GFX10-WGP-NEXT:    s_clause 0x1
9132; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9133; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9134; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9135; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
9136; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
9137; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
9138; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9139; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
9140; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9141; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9142; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9143; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9144; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9145; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
9146; GFX10-WGP-NEXT:    s_endpgm
9147;
9148; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
9149; GFX10-CU:       ; %bb.0: ; %entry
9150; GFX10-CU-NEXT:    s_clause 0x1
9151; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9152; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9153; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9154; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
9155; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
9156; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
9157; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9158; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
9159; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9160; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9161; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9162; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9163; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9164; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
9165; GFX10-CU-NEXT:    s_endpgm
9166;
9167; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
9168; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9169; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9170; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9171; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9172; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
9173; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
9174; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
9175; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9176; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
9177; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9178; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9179; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9180; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9181; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9182; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
9183; SKIP-CACHE-INV-NEXT:    s_endpgm
9184;
9185; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
9186; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9187; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9188; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9189; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9190; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9191; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9192; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9193; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9194; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9195; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9196;
9197; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
9198; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9199; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9200; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9201; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9202; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9203; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9204; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9205; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9206; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9207; GFX90A-TGSPLIT-NEXT:    s_endpgm
9208;
9209; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
9210; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9211; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9212; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9213; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9214; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9215; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9216; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9217; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9218; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9219; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9220;
9221; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
9222; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9223; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9224; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9225; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9226; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9227; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9228; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9229; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9230; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9231; GFX940-TGSPLIT-NEXT:    s_endpgm
9232    i32* %out, i32 %in, i32 %old) {
9233entry:
9234  %gep = getelementptr i32, i32* %out, i32 4
9235  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire acquire
9236  %val0 = extractvalue { i32, i1 } %val, 0
9237  store i32 %val0, i32* %out, align 4
9238  ret void
9239}
9240
9241define amdgpu_kernel void @flat_singlethread_one_as_release_acquire_ret_cmpxchg(
9242; GFX7-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
9243; GFX7:       ; %bb.0: ; %entry
9244; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9245; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9246; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9247; GFX7-NEXT:    s_add_u32 s4, s0, 16
9248; GFX7-NEXT:    s_addc_u32 s5, s1, 0
9249; GFX7-NEXT:    v_mov_b32_e32 v0, s4
9250; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9251; GFX7-NEXT:    v_mov_b32_e32 v1, s5
9252; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9253; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9254; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9255; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9256; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9257; GFX7-NEXT:    flat_store_dword v[0:1], v2
9258; GFX7-NEXT:    s_endpgm
9259;
9260; GFX10-WGP-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
9261; GFX10-WGP:       ; %bb.0: ; %entry
9262; GFX10-WGP-NEXT:    s_clause 0x1
9263; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9264; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9265; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9266; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
9267; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
9268; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
9269; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9270; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
9271; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9272; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9273; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9274; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9275; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9276; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
9277; GFX10-WGP-NEXT:    s_endpgm
9278;
9279; GFX10-CU-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
9280; GFX10-CU:       ; %bb.0: ; %entry
9281; GFX10-CU-NEXT:    s_clause 0x1
9282; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9283; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9284; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9285; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
9286; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
9287; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
9288; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9289; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
9290; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9291; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9292; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9293; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9294; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9295; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
9296; GFX10-CU-NEXT:    s_endpgm
9297;
9298; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
9299; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9300; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9301; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9302; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9303; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
9304; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
9305; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
9306; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9307; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
9308; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9309; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9310; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9311; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9312; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9313; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
9314; SKIP-CACHE-INV-NEXT:    s_endpgm
9315;
9316; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
9317; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9318; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9319; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9320; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9321; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9322; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9323; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9324; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9325; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9326; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9327;
9328; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
9329; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9330; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9331; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9332; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9333; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9334; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9335; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9336; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9337; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9338; GFX90A-TGSPLIT-NEXT:    s_endpgm
9339;
9340; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
9341; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9342; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9343; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9344; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9345; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9346; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9347; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9348; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9349; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9350; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9351;
9352; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
9353; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9354; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9355; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9356; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9357; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9358; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9359; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9360; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9361; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9362; GFX940-TGSPLIT-NEXT:    s_endpgm
9363    i32* %out, i32 %in, i32 %old) {
9364entry:
9365  %gep = getelementptr i32, i32* %out, i32 4
9366  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release acquire
9367  %val0 = extractvalue { i32, i1 } %val, 0
9368  store i32 %val0, i32* %out, align 4
9369  ret void
9370}
9371
9372define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg(
9373; GFX7-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
9374; GFX7:       ; %bb.0: ; %entry
9375; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9376; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9377; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9378; GFX7-NEXT:    s_add_u32 s4, s0, 16
9379; GFX7-NEXT:    s_addc_u32 s5, s1, 0
9380; GFX7-NEXT:    v_mov_b32_e32 v0, s4
9381; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9382; GFX7-NEXT:    v_mov_b32_e32 v1, s5
9383; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9384; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9385; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9386; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9387; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9388; GFX7-NEXT:    flat_store_dword v[0:1], v2
9389; GFX7-NEXT:    s_endpgm
9390;
9391; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
9392; GFX10-WGP:       ; %bb.0: ; %entry
9393; GFX10-WGP-NEXT:    s_clause 0x1
9394; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9395; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9396; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9397; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
9398; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
9399; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
9400; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9401; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
9402; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9403; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9404; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9405; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9406; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9407; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
9408; GFX10-WGP-NEXT:    s_endpgm
9409;
9410; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
9411; GFX10-CU:       ; %bb.0: ; %entry
9412; GFX10-CU-NEXT:    s_clause 0x1
9413; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9414; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9415; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9416; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
9417; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
9418; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
9419; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9420; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
9421; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9422; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9423; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9424; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9425; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9426; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
9427; GFX10-CU-NEXT:    s_endpgm
9428;
9429; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
9430; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9431; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9432; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9433; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9434; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
9435; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
9436; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
9437; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9438; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
9439; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9440; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9441; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9442; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9443; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9444; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
9445; SKIP-CACHE-INV-NEXT:    s_endpgm
9446;
9447; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
9448; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9449; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9450; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9451; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9452; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9453; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9454; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9455; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9456; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9457; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9458;
9459; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
9460; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9461; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9462; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9463; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9464; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9465; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9466; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9467; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9468; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9469; GFX90A-TGSPLIT-NEXT:    s_endpgm
9470;
9471; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
9472; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9473; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9474; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9475; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9476; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9477; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9478; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9479; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9480; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9481; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9482;
9483; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
9484; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9485; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9486; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9487; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9488; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9489; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9490; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9491; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9492; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9493; GFX940-TGSPLIT-NEXT:    s_endpgm
9494    i32* %out, i32 %in, i32 %old) {
9495entry:
9496  %gep = getelementptr i32, i32* %out, i32 4
9497  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel acquire
9498  %val0 = extractvalue { i32, i1 } %val, 0
9499  store i32 %val0, i32* %out, align 4
9500  ret void
9501}
9502
9503define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg(
9504; GFX7-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
9505; GFX7:       ; %bb.0: ; %entry
9506; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9507; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9508; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9509; GFX7-NEXT:    s_add_u32 s4, s0, 16
9510; GFX7-NEXT:    s_addc_u32 s5, s1, 0
9511; GFX7-NEXT:    v_mov_b32_e32 v0, s4
9512; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9513; GFX7-NEXT:    v_mov_b32_e32 v1, s5
9514; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9515; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9516; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9517; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9518; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9519; GFX7-NEXT:    flat_store_dword v[0:1], v2
9520; GFX7-NEXT:    s_endpgm
9521;
9522; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
9523; GFX10-WGP:       ; %bb.0: ; %entry
9524; GFX10-WGP-NEXT:    s_clause 0x1
9525; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9526; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9527; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9528; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
9529; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
9530; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
9531; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9532; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
9533; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9534; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9535; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9536; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9537; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9538; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
9539; GFX10-WGP-NEXT:    s_endpgm
9540;
9541; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
9542; GFX10-CU:       ; %bb.0: ; %entry
9543; GFX10-CU-NEXT:    s_clause 0x1
9544; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9545; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9546; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9547; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
9548; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
9549; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
9550; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9551; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
9552; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9553; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9554; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9555; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9556; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9557; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
9558; GFX10-CU-NEXT:    s_endpgm
9559;
9560; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
9561; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9562; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9563; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9564; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9565; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
9566; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
9567; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
9568; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9569; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
9570; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9571; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9572; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9573; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9574; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9575; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
9576; SKIP-CACHE-INV-NEXT:    s_endpgm
9577;
9578; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
9579; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9580; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9581; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9582; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9583; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9584; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9585; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9586; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9587; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9588; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9589;
9590; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
9591; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9592; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9593; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9594; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9595; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9596; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9597; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9598; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9599; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9600; GFX90A-TGSPLIT-NEXT:    s_endpgm
9601;
9602; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
9603; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9604; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9605; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9606; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9607; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9608; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9609; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9610; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9611; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9612; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9613;
9614; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
9615; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9616; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9617; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9618; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9619; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9620; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9621; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9622; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9623; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9624; GFX940-TGSPLIT-NEXT:    s_endpgm
9625    i32* %out, i32 %in, i32 %old) {
9626entry:
9627  %gep = getelementptr i32, i32* %out, i32 4
9628  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst acquire
9629  %val0 = extractvalue { i32, i1 } %val, 0
9630  store i32 %val0, i32* %out, align 4
9631  ret void
9632}
9633
9634define amdgpu_kernel void @flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg(
9635; GFX7-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
9636; GFX7:       ; %bb.0: ; %entry
9637; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9638; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9639; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9640; GFX7-NEXT:    s_add_u32 s4, s0, 16
9641; GFX7-NEXT:    s_addc_u32 s5, s1, 0
9642; GFX7-NEXT:    v_mov_b32_e32 v0, s4
9643; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9644; GFX7-NEXT:    v_mov_b32_e32 v1, s5
9645; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9646; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9647; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9648; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9649; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9650; GFX7-NEXT:    flat_store_dword v[0:1], v2
9651; GFX7-NEXT:    s_endpgm
9652;
9653; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
9654; GFX10-WGP:       ; %bb.0: ; %entry
9655; GFX10-WGP-NEXT:    s_clause 0x1
9656; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9657; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9658; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9659; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
9660; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
9661; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
9662; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9663; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
9664; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9665; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9666; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9667; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9668; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9669; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
9670; GFX10-WGP-NEXT:    s_endpgm
9671;
9672; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
9673; GFX10-CU:       ; %bb.0: ; %entry
9674; GFX10-CU-NEXT:    s_clause 0x1
9675; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9676; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9677; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9678; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
9679; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
9680; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
9681; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9682; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
9683; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9684; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9685; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9686; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9687; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9688; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
9689; GFX10-CU-NEXT:    s_endpgm
9690;
9691; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
9692; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9693; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9694; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9695; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9696; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
9697; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
9698; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
9699; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9700; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
9701; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9702; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9703; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9704; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9705; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9706; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
9707; SKIP-CACHE-INV-NEXT:    s_endpgm
9708;
9709; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
9710; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9711; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9712; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9713; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9714; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9715; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9716; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9717; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9718; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9719; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9720;
9721; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
9722; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9723; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9724; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9725; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9726; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9727; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9728; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9729; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9730; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9731; GFX90A-TGSPLIT-NEXT:    s_endpgm
9732;
9733; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
9734; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9735; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9736; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9737; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9738; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9739; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9740; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9741; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9742; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9743; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9744;
9745; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
9746; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9747; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9748; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9749; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9750; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9751; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9752; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9753; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9754; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9755; GFX940-TGSPLIT-NEXT:    s_endpgm
9756    i32* %out, i32 %in, i32 %old) {
9757entry:
9758  %gep = getelementptr i32, i32* %out, i32 4
9759  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic seq_cst
9760  %val0 = extractvalue { i32, i1 } %val, 0
9761  store i32 %val0, i32* %out, align 4
9762  ret void
9763}
9764
9765define amdgpu_kernel void @flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg(
9766; GFX7-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
9767; GFX7:       ; %bb.0: ; %entry
9768; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9769; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9770; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9771; GFX7-NEXT:    s_add_u32 s4, s0, 16
9772; GFX7-NEXT:    s_addc_u32 s5, s1, 0
9773; GFX7-NEXT:    v_mov_b32_e32 v0, s4
9774; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9775; GFX7-NEXT:    v_mov_b32_e32 v1, s5
9776; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9777; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9778; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9779; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9780; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9781; GFX7-NEXT:    flat_store_dword v[0:1], v2
9782; GFX7-NEXT:    s_endpgm
9783;
9784; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
9785; GFX10-WGP:       ; %bb.0: ; %entry
9786; GFX10-WGP-NEXT:    s_clause 0x1
9787; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9788; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9789; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9790; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
9791; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
9792; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
9793; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9794; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
9795; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9796; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9797; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9798; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9799; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9800; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
9801; GFX10-WGP-NEXT:    s_endpgm
9802;
9803; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
9804; GFX10-CU:       ; %bb.0: ; %entry
9805; GFX10-CU-NEXT:    s_clause 0x1
9806; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9807; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9808; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9809; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
9810; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
9811; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
9812; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9813; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
9814; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9815; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9816; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9817; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9818; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9819; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
9820; GFX10-CU-NEXT:    s_endpgm
9821;
9822; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
9823; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9824; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9825; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9826; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9827; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
9828; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
9829; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
9830; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9831; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
9832; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9833; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9834; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9835; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9836; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9837; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
9838; SKIP-CACHE-INV-NEXT:    s_endpgm
9839;
9840; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
9841; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9842; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9843; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9844; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9845; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9846; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9847; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9848; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9849; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9850; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9851;
9852; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
9853; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9854; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9855; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9856; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9857; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9858; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9859; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9860; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9861; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9862; GFX90A-TGSPLIT-NEXT:    s_endpgm
9863;
9864; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
9865; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9866; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9867; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9868; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9869; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9870; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9871; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9872; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9873; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9874; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9875;
9876; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
9877; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9878; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9879; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9880; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9881; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9882; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9883; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9884; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9885; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9886; GFX940-TGSPLIT-NEXT:    s_endpgm
9887    i32* %out, i32 %in, i32 %old) {
9888entry:
9889  %gep = getelementptr i32, i32* %out, i32 4
9890  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire seq_cst
9891  %val0 = extractvalue { i32, i1 } %val, 0
9892  store i32 %val0, i32* %out, align 4
9893  ret void
9894}
9895
9896define amdgpu_kernel void @flat_singlethread_one_as_release_seq_cst_ret_cmpxchg(
9897; GFX7-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
9898; GFX7:       ; %bb.0: ; %entry
9899; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9900; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9901; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9902; GFX7-NEXT:    s_add_u32 s4, s0, 16
9903; GFX7-NEXT:    s_addc_u32 s5, s1, 0
9904; GFX7-NEXT:    v_mov_b32_e32 v0, s4
9905; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9906; GFX7-NEXT:    v_mov_b32_e32 v1, s5
9907; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9908; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9909; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9910; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9911; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9912; GFX7-NEXT:    flat_store_dword v[0:1], v2
9913; GFX7-NEXT:    s_endpgm
9914;
9915; GFX10-WGP-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
9916; GFX10-WGP:       ; %bb.0: ; %entry
9917; GFX10-WGP-NEXT:    s_clause 0x1
9918; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9919; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9920; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9921; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
9922; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
9923; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
9924; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9925; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
9926; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9927; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9928; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9929; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9930; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9931; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
9932; GFX10-WGP-NEXT:    s_endpgm
9933;
9934; GFX10-CU-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
9935; GFX10-CU:       ; %bb.0: ; %entry
9936; GFX10-CU-NEXT:    s_clause 0x1
9937; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9938; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9939; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9940; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
9941; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
9942; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
9943; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9944; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
9945; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9946; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9947; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9948; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9949; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9950; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
9951; GFX10-CU-NEXT:    s_endpgm
9952;
9953; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
9954; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9955; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9956; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9957; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9958; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
9959; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
9960; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
9961; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9962; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
9963; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9964; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9965; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9966; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9967; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9968; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
9969; SKIP-CACHE-INV-NEXT:    s_endpgm
9970;
9971; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
9972; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9973; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9974; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9975; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9976; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9977; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9978; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9979; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9980; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9981; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9982;
9983; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
9984; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9985; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9986; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9987; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9988; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9989; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9990; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9991; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9992; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9993; GFX90A-TGSPLIT-NEXT:    s_endpgm
9994;
9995; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
9996; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9997; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9998; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9999; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10000; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10001; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10002; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10003; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10004; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10005; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10006;
10007; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
10008; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10009; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10010; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10011; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10012; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10013; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10014; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10015; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10016; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10017; GFX940-TGSPLIT-NEXT:    s_endpgm
10018    i32* %out, i32 %in, i32 %old) {
10019entry:
10020  %gep = getelementptr i32, i32* %out, i32 4
10021  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release seq_cst
10022  %val0 = extractvalue { i32, i1 } %val, 0
10023  store i32 %val0, i32* %out, align 4
10024  ret void
10025}
10026
10027define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg(
10028; GFX7-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
10029; GFX7:       ; %bb.0: ; %entry
10030; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10031; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10032; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10033; GFX7-NEXT:    s_add_u32 s4, s0, 16
10034; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10035; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10036; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10037; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10038; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10039; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10040; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10041; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10042; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10043; GFX7-NEXT:    flat_store_dword v[0:1], v2
10044; GFX7-NEXT:    s_endpgm
10045;
10046; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
10047; GFX10-WGP:       ; %bb.0: ; %entry
10048; GFX10-WGP-NEXT:    s_clause 0x1
10049; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10050; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10051; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10052; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
10053; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
10054; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
10055; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10056; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
10057; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10058; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10059; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10060; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10061; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10062; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
10063; GFX10-WGP-NEXT:    s_endpgm
10064;
10065; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
10066; GFX10-CU:       ; %bb.0: ; %entry
10067; GFX10-CU-NEXT:    s_clause 0x1
10068; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10069; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10070; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10071; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
10072; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
10073; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
10074; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10075; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
10076; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10077; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10078; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10079; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10080; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10081; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
10082; GFX10-CU-NEXT:    s_endpgm
10083;
10084; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
10085; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10086; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10087; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10088; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10089; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
10090; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
10091; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10092; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10093; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10094; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10095; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10096; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10097; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10098; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10099; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
10100; SKIP-CACHE-INV-NEXT:    s_endpgm
10101;
10102; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
10103; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10104; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10105; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10106; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10107; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10108; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10109; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10110; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10111; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10112; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10113;
10114; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
10115; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10116; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10117; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10118; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10119; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10120; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10121; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10122; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10123; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10124; GFX90A-TGSPLIT-NEXT:    s_endpgm
10125;
10126; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
10127; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10128; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10129; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10130; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10131; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10132; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10133; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10134; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10135; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10136; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10137;
10138; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
10139; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10140; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10141; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10142; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10143; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10144; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10145; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10146; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10147; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10148; GFX940-TGSPLIT-NEXT:    s_endpgm
10149    i32* %out, i32 %in, i32 %old) {
10150entry:
10151  %gep = getelementptr i32, i32* %out, i32 4
10152  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel seq_cst
10153  %val0 = extractvalue { i32, i1 } %val, 0
10154  store i32 %val0, i32* %out, align 4
10155  ret void
10156}
10157
10158define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg(
10159; GFX7-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
10160; GFX7:       ; %bb.0: ; %entry
10161; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10162; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10163; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10164; GFX7-NEXT:    s_add_u32 s4, s0, 16
10165; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10166; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10167; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10168; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10169; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10170; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10171; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10172; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10173; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10174; GFX7-NEXT:    flat_store_dword v[0:1], v2
10175; GFX7-NEXT:    s_endpgm
10176;
10177; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
10178; GFX10-WGP:       ; %bb.0: ; %entry
10179; GFX10-WGP-NEXT:    s_clause 0x1
10180; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10181; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10182; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10183; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
10184; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
10185; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
10186; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10187; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
10188; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10189; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10190; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10191; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10192; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10193; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
10194; GFX10-WGP-NEXT:    s_endpgm
10195;
10196; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
10197; GFX10-CU:       ; %bb.0: ; %entry
10198; GFX10-CU-NEXT:    s_clause 0x1
10199; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10200; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10201; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10202; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
10203; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
10204; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
10205; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10206; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
10207; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10208; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10209; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10210; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10211; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10212; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
10213; GFX10-CU-NEXT:    s_endpgm
10214;
10215; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
10216; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10217; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10218; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10219; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10220; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
10221; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
10222; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10223; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10224; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10225; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10226; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10227; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10228; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10229; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10230; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
10231; SKIP-CACHE-INV-NEXT:    s_endpgm
10232;
10233; GFX90A-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
10234; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10235; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10236; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10237; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10238; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10239; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10240; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10241; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10242; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10243; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10244;
10245; GFX90A-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
10246; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10247; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10248; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10249; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10250; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10251; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10252; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10253; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10254; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10255; GFX90A-TGSPLIT-NEXT:    s_endpgm
10256;
10257; GFX940-NOTTGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
10258; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10259; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10260; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10261; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10262; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10263; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10264; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10265; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10266; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10267; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10268;
10269; GFX940-TGSPLIT-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
10270; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10271; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10272; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10273; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10274; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10275; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10276; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10277; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10278; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10279; GFX940-TGSPLIT-NEXT:    s_endpgm
10280    i32* %out, i32 %in, i32 %old) {
10281entry:
10282  %gep = getelementptr i32, i32* %out, i32 4
10283  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst seq_cst
10284  %val0 = extractvalue { i32, i1 } %val, 0
10285  store i32 %val0, i32* %out, align 4
10286  ret void
10287}
10288
10289