1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
3; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
4; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
5; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
6; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-NOTTGSPLIT %s
7; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-TGSPLIT %s
8; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-NOTTGSPLIT %s
9; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
10; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11-WGP %s
11; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11-CU %s
12
13define amdgpu_kernel void @flat_system_unordered_load(
14; GFX7-LABEL: flat_system_unordered_load:
15; GFX7:       ; %bb.0: ; %entry
16; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
17; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
18; GFX7-NEXT:    v_mov_b32_e32 v0, s0
19; GFX7-NEXT:    v_mov_b32_e32 v1, s1
20; GFX7-NEXT:    flat_load_dword v2, v[0:1]
21; GFX7-NEXT:    v_mov_b32_e32 v0, s2
22; GFX7-NEXT:    v_mov_b32_e32 v1, s3
23; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
24; GFX7-NEXT:    flat_store_dword v[0:1], v2
25; GFX7-NEXT:    s_endpgm
26;
27; GFX10-WGP-LABEL: flat_system_unordered_load:
28; GFX10-WGP:       ; %bb.0: ; %entry
29; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
30; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
31; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
32; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
33; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
34; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
35; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
36; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
37; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
38; GFX10-WGP-NEXT:    s_endpgm
39;
40; GFX10-CU-LABEL: flat_system_unordered_load:
41; GFX10-CU:       ; %bb.0: ; %entry
42; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
43; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
44; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
45; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
46; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
47; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
48; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
49; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
50; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
51; GFX10-CU-NEXT:    s_endpgm
52;
53; SKIP-CACHE-INV-LABEL: flat_system_unordered_load:
54; SKIP-CACHE-INV:       ; %bb.0: ; %entry
55; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
56; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
57; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
58; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
59; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
60; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
61; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
62; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
63; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
64; SKIP-CACHE-INV-NEXT:    s_endpgm
65;
66; GFX90A-NOTTGSPLIT-LABEL: flat_system_unordered_load:
67; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
68; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
69; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
70; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
71; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
72; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
73; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
74; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
75; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
76; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
77; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
78;
79; GFX90A-TGSPLIT-LABEL: flat_system_unordered_load:
80; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
81; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
82; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
83; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
84; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
85; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
86; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
87; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
88; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
89; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
90; GFX90A-TGSPLIT-NEXT:    s_endpgm
91;
92; GFX940-NOTTGSPLIT-LABEL: flat_system_unordered_load:
93; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
94; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
95; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
96; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
97; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
98; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
99; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
100; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
101; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
102; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
103; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
104;
105; GFX940-TGSPLIT-LABEL: flat_system_unordered_load:
106; GFX940-TGSPLIT:       ; %bb.0: ; %entry
107; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
108; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
109; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
110; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
111; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
112; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
113; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
114; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
115; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
116; GFX940-TGSPLIT-NEXT:    s_endpgm
117;
118; GFX11-WGP-LABEL: flat_system_unordered_load:
119; GFX11-WGP:       ; %bb.0: ; %entry
120; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
121; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
122; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
123; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1]
124; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
125; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
126; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
127; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
128; GFX11-WGP-NEXT:    s_endpgm
129;
130; GFX11-CU-LABEL: flat_system_unordered_load:
131; GFX11-CU:       ; %bb.0: ; %entry
132; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
133; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
134; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
135; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
136; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
137; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
138; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
139; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
140; GFX11-CU-NEXT:    s_endpgm
141    i32* %in, i32* %out) {
142entry:
143  %val = load atomic i32, i32* %in unordered, align 4
144  store i32 %val, i32* %out
145  ret void
146}
147
148define amdgpu_kernel void @flat_system_monotonic_load(
149; GFX7-LABEL: flat_system_monotonic_load:
150; GFX7:       ; %bb.0: ; %entry
151; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
152; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
153; GFX7-NEXT:    v_mov_b32_e32 v0, s0
154; GFX7-NEXT:    v_mov_b32_e32 v1, s1
155; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
156; GFX7-NEXT:    v_mov_b32_e32 v0, s2
157; GFX7-NEXT:    v_mov_b32_e32 v1, s3
158; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
159; GFX7-NEXT:    flat_store_dword v[0:1], v2
160; GFX7-NEXT:    s_endpgm
161;
162; GFX10-WGP-LABEL: flat_system_monotonic_load:
163; GFX10-WGP:       ; %bb.0: ; %entry
164; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
165; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
166; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
167; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
168; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
169; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
170; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
171; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
172; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
173; GFX10-WGP-NEXT:    s_endpgm
174;
175; GFX10-CU-LABEL: flat_system_monotonic_load:
176; GFX10-CU:       ; %bb.0: ; %entry
177; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
178; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
179; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
180; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
181; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
182; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
183; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
184; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
185; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
186; GFX10-CU-NEXT:    s_endpgm
187;
188; SKIP-CACHE-INV-LABEL: flat_system_monotonic_load:
189; SKIP-CACHE-INV:       ; %bb.0: ; %entry
190; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
191; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
192; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
193; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
194; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
195; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
196; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
197; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
198; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
199; SKIP-CACHE-INV-NEXT:    s_endpgm
200;
201; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_load:
202; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
203; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
204; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
205; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
206; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
207; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
208; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
209; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
210; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
211; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
212; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
213;
214; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_load:
215; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
216; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
217; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
218; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
219; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
220; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
221; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
222; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
223; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
224; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
225; GFX90A-TGSPLIT-NEXT:    s_endpgm
226;
227; GFX940-NOTTGSPLIT-LABEL: flat_system_monotonic_load:
228; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
229; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
230; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
231; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
232; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
233; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0 sc1
234; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
235; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
236; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
237; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
238; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
239;
240; GFX940-TGSPLIT-LABEL: flat_system_monotonic_load:
241; GFX940-TGSPLIT:       ; %bb.0: ; %entry
242; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
243; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
244; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
245; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
246; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0 sc1
247; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
248; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
249; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
250; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
251; GFX940-TGSPLIT-NEXT:    s_endpgm
252;
253; GFX11-WGP-LABEL: flat_system_monotonic_load:
254; GFX11-WGP:       ; %bb.0: ; %entry
255; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
256; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
257; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
258; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc
259; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
260; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
261; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
262; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
263; GFX11-WGP-NEXT:    s_endpgm
264;
265; GFX11-CU-LABEL: flat_system_monotonic_load:
266; GFX11-CU:       ; %bb.0: ; %entry
267; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
268; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
269; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
270; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1] glc
271; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
272; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
273; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
274; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
275; GFX11-CU-NEXT:    s_endpgm
276    i32* %in, i32* %out) {
277entry:
278  %val = load atomic i32, i32* %in monotonic, align 4
279  store i32 %val, i32* %out
280  ret void
281}
282
283define amdgpu_kernel void @flat_system_acquire_load(
284; GFX7-LABEL: flat_system_acquire_load:
285; GFX7:       ; %bb.0: ; %entry
286; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
287; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
288; GFX7-NEXT:    v_mov_b32_e32 v0, s0
289; GFX7-NEXT:    v_mov_b32_e32 v1, s1
290; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
291; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
292; GFX7-NEXT:    buffer_wbinvl1_vol
293; GFX7-NEXT:    v_mov_b32_e32 v0, s2
294; GFX7-NEXT:    v_mov_b32_e32 v1, s3
295; GFX7-NEXT:    flat_store_dword v[0:1], v2
296; GFX7-NEXT:    s_endpgm
297;
298; GFX10-WGP-LABEL: flat_system_acquire_load:
299; GFX10-WGP:       ; %bb.0: ; %entry
300; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
301; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
302; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
303; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
304; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
305; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
306; GFX10-WGP-NEXT:    buffer_gl0_inv
307; GFX10-WGP-NEXT:    buffer_gl1_inv
308; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
309; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
310; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
311; GFX10-WGP-NEXT:    s_endpgm
312;
313; GFX10-CU-LABEL: flat_system_acquire_load:
314; GFX10-CU:       ; %bb.0: ; %entry
315; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
316; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
317; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
318; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
319; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
320; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
321; GFX10-CU-NEXT:    buffer_gl0_inv
322; GFX10-CU-NEXT:    buffer_gl1_inv
323; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
324; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
325; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
326; GFX10-CU-NEXT:    s_endpgm
327;
328; SKIP-CACHE-INV-LABEL: flat_system_acquire_load:
329; SKIP-CACHE-INV:       ; %bb.0: ; %entry
330; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
331; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
332; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
333; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
334; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
335; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
336; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
337; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
338; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
339; SKIP-CACHE-INV-NEXT:    s_endpgm
340;
341; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_load:
342; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
343; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
344; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
345; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
346; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
347; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
348; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
349; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
350; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
351; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
352; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
353; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
354; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
355;
356; GFX90A-TGSPLIT-LABEL: flat_system_acquire_load:
357; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
358; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
359; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
360; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
361; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
362; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
363; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
364; GFX90A-TGSPLIT-NEXT:    buffer_invl2
365; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
366; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
367; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
368; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
369; GFX90A-TGSPLIT-NEXT:    s_endpgm
370;
371; GFX940-NOTTGSPLIT-LABEL: flat_system_acquire_load:
372; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
373; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
374; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
375; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
376; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
377; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0 sc1
378; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
379; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
380; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
381; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
382; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
383; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
384;
385; GFX940-TGSPLIT-LABEL: flat_system_acquire_load:
386; GFX940-TGSPLIT:       ; %bb.0: ; %entry
387; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
388; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
389; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
390; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
391; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0 sc1
392; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
393; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
394; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
395; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
396; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
397; GFX940-TGSPLIT-NEXT:    s_endpgm
398;
399; GFX11-WGP-LABEL: flat_system_acquire_load:
400; GFX11-WGP:       ; %bb.0: ; %entry
401; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
402; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
403; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
404; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc
405; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
406; GFX11-WGP-NEXT:    buffer_gl0_inv
407; GFX11-WGP-NEXT:    buffer_gl1_inv
408; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
409; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
410; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
411; GFX11-WGP-NEXT:    s_endpgm
412;
413; GFX11-CU-LABEL: flat_system_acquire_load:
414; GFX11-CU:       ; %bb.0: ; %entry
415; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
416; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
417; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
418; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1] glc
419; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
420; GFX11-CU-NEXT:    buffer_gl0_inv
421; GFX11-CU-NEXT:    buffer_gl1_inv
422; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
423; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
424; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
425; GFX11-CU-NEXT:    s_endpgm
426    i32* %in, i32* %out) {
427entry:
428  %val = load atomic i32, i32* %in acquire, align 4
429  store i32 %val, i32* %out
430  ret void
431}
432
433define amdgpu_kernel void @flat_system_seq_cst_load(
434; GFX7-LABEL: flat_system_seq_cst_load:
435; GFX7:       ; %bb.0: ; %entry
436; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
437; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
438; GFX7-NEXT:    v_mov_b32_e32 v0, s0
439; GFX7-NEXT:    v_mov_b32_e32 v1, s1
440; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
441; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
442; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
443; GFX7-NEXT:    buffer_wbinvl1_vol
444; GFX7-NEXT:    v_mov_b32_e32 v0, s2
445; GFX7-NEXT:    v_mov_b32_e32 v1, s3
446; GFX7-NEXT:    flat_store_dword v[0:1], v2
447; GFX7-NEXT:    s_endpgm
448;
449; GFX10-WGP-LABEL: flat_system_seq_cst_load:
450; GFX10-WGP:       ; %bb.0: ; %entry
451; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
452; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
453; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
454; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
455; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
456; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
457; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
458; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
459; GFX10-WGP-NEXT:    buffer_gl0_inv
460; GFX10-WGP-NEXT:    buffer_gl1_inv
461; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
462; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
463; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
464; GFX10-WGP-NEXT:    s_endpgm
465;
466; GFX10-CU-LABEL: flat_system_seq_cst_load:
467; GFX10-CU:       ; %bb.0: ; %entry
468; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
469; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
470; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
471; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
472; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
473; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
474; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
475; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
476; GFX10-CU-NEXT:    buffer_gl0_inv
477; GFX10-CU-NEXT:    buffer_gl1_inv
478; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
479; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
480; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
481; GFX10-CU-NEXT:    s_endpgm
482;
483; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_load:
484; SKIP-CACHE-INV:       ; %bb.0: ; %entry
485; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
486; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
487; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
488; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
489; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
490; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
491; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
492; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
493; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
494; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
495; SKIP-CACHE-INV-NEXT:    s_endpgm
496;
497; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_load:
498; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
499; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
500; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
501; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
502; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
503; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
504; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
505; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
506; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
507; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
508; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
509; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
510; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
511; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
512;
513; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_load:
514; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
515; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
516; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
517; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
518; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
519; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
520; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
521; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
522; GFX90A-TGSPLIT-NEXT:    buffer_invl2
523; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
524; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
525; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
526; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
527; GFX90A-TGSPLIT-NEXT:    s_endpgm
528;
529; GFX940-NOTTGSPLIT-LABEL: flat_system_seq_cst_load:
530; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
531; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
532; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
533; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
534; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
535; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
536; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0 sc1
537; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
538; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
539; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
540; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
541; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
542; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
543;
544; GFX940-TGSPLIT-LABEL: flat_system_seq_cst_load:
545; GFX940-TGSPLIT:       ; %bb.0: ; %entry
546; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
547; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
548; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
549; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
550; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
551; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0 sc1
552; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
553; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
554; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
555; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
556; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
557; GFX940-TGSPLIT-NEXT:    s_endpgm
558;
559; GFX11-WGP-LABEL: flat_system_seq_cst_load:
560; GFX11-WGP:       ; %bb.0: ; %entry
561; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
562; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
563; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
564; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
565; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
566; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc
567; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
568; GFX11-WGP-NEXT:    buffer_gl0_inv
569; GFX11-WGP-NEXT:    buffer_gl1_inv
570; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
571; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
572; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
573; GFX11-WGP-NEXT:    s_endpgm
574;
575; GFX11-CU-LABEL: flat_system_seq_cst_load:
576; GFX11-CU:       ; %bb.0: ; %entry
577; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
578; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
579; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
580; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
581; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
582; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1] glc
583; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
584; GFX11-CU-NEXT:    buffer_gl0_inv
585; GFX11-CU-NEXT:    buffer_gl1_inv
586; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
587; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
588; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
589; GFX11-CU-NEXT:    s_endpgm
590    i32* %in, i32* %out) {
591entry:
592  %val = load atomic i32, i32* %in seq_cst, align 4
593  store i32 %val, i32* %out
594  ret void
595}
596
597define amdgpu_kernel void @flat_system_unordered_store(
598; GFX7-LABEL: flat_system_unordered_store:
599; GFX7:       ; %bb.0: ; %entry
600; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
601; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
602; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
603; GFX7-NEXT:    v_mov_b32_e32 v0, s0
604; GFX7-NEXT:    v_mov_b32_e32 v1, s1
605; GFX7-NEXT:    v_mov_b32_e32 v2, s2
606; GFX7-NEXT:    flat_store_dword v[0:1], v2
607; GFX7-NEXT:    s_endpgm
608;
609; GFX10-WGP-LABEL: flat_system_unordered_store:
610; GFX10-WGP:       ; %bb.0: ; %entry
611; GFX10-WGP-NEXT:    s_clause 0x1
612; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
613; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
614; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
615; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
616; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
617; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
618; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
619; GFX10-WGP-NEXT:    s_endpgm
620;
621; GFX10-CU-LABEL: flat_system_unordered_store:
622; GFX10-CU:       ; %bb.0: ; %entry
623; GFX10-CU-NEXT:    s_clause 0x1
624; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
625; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
626; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
627; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
628; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
629; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
630; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
631; GFX10-CU-NEXT:    s_endpgm
632;
633; SKIP-CACHE-INV-LABEL: flat_system_unordered_store:
634; SKIP-CACHE-INV:       ; %bb.0: ; %entry
635; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
636; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
637; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
638; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
639; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
640; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
641; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
642; SKIP-CACHE-INV-NEXT:    s_endpgm
643;
644; GFX90A-NOTTGSPLIT-LABEL: flat_system_unordered_store:
645; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
646; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
647; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
648; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
649; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
650; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
651; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
652; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
653;
654; GFX90A-TGSPLIT-LABEL: flat_system_unordered_store:
655; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
656; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
657; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
658; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
659; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
660; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
661; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
662; GFX90A-TGSPLIT-NEXT:    s_endpgm
663;
664; GFX940-NOTTGSPLIT-LABEL: flat_system_unordered_store:
665; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
666; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
667; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
668; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
669; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
670; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
671; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
672; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
673;
674; GFX940-TGSPLIT-LABEL: flat_system_unordered_store:
675; GFX940-TGSPLIT:       ; %bb.0: ; %entry
676; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
677; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
678; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
679; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
680; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
681; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
682; GFX940-TGSPLIT-NEXT:    s_endpgm
683;
684; GFX11-WGP-LABEL: flat_system_unordered_store:
685; GFX11-WGP:       ; %bb.0: ; %entry
686; GFX11-WGP-NEXT:    s_clause 0x1
687; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
688; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
689; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
690; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
691; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
692; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
693; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
694; GFX11-WGP-NEXT:    s_endpgm
695;
696; GFX11-CU-LABEL: flat_system_unordered_store:
697; GFX11-CU:       ; %bb.0: ; %entry
698; GFX11-CU-NEXT:    s_clause 0x1
699; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
700; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
701; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
702; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
703; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
704; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
705; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
706; GFX11-CU-NEXT:    s_endpgm
707    i32 %in, i32* %out) {
708entry:
709  store atomic i32 %in, i32* %out unordered, align 4
710  ret void
711}
712
713define amdgpu_kernel void @flat_system_monotonic_store(
714; GFX7-LABEL: flat_system_monotonic_store:
715; GFX7:       ; %bb.0: ; %entry
716; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
717; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
718; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
719; GFX7-NEXT:    v_mov_b32_e32 v0, s0
720; GFX7-NEXT:    v_mov_b32_e32 v1, s1
721; GFX7-NEXT:    v_mov_b32_e32 v2, s2
722; GFX7-NEXT:    flat_store_dword v[0:1], v2
723; GFX7-NEXT:    s_endpgm
724;
725; GFX10-WGP-LABEL: flat_system_monotonic_store:
726; GFX10-WGP:       ; %bb.0: ; %entry
727; GFX10-WGP-NEXT:    s_clause 0x1
728; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
729; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
730; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
731; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
732; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
733; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
734; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
735; GFX10-WGP-NEXT:    s_endpgm
736;
737; GFX10-CU-LABEL: flat_system_monotonic_store:
738; GFX10-CU:       ; %bb.0: ; %entry
739; GFX10-CU-NEXT:    s_clause 0x1
740; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
741; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
742; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
743; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
744; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
745; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
746; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
747; GFX10-CU-NEXT:    s_endpgm
748;
749; SKIP-CACHE-INV-LABEL: flat_system_monotonic_store:
750; SKIP-CACHE-INV:       ; %bb.0: ; %entry
751; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
752; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
753; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
754; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
755; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
756; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
757; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
758; SKIP-CACHE-INV-NEXT:    s_endpgm
759;
760; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_store:
761; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
762; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
763; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
764; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
765; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
766; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
767; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
768; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
769;
770; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_store:
771; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
772; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
773; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
774; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
775; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
776; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
777; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
778; GFX90A-TGSPLIT-NEXT:    s_endpgm
779;
780; GFX940-NOTTGSPLIT-LABEL: flat_system_monotonic_store:
781; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
782; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
783; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
784; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
785; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
786; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
787; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
788; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
789;
790; GFX940-TGSPLIT-LABEL: flat_system_monotonic_store:
791; GFX940-TGSPLIT:       ; %bb.0: ; %entry
792; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
793; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
794; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
795; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
796; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
797; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
798; GFX940-TGSPLIT-NEXT:    s_endpgm
799;
800; GFX11-WGP-LABEL: flat_system_monotonic_store:
801; GFX11-WGP:       ; %bb.0: ; %entry
802; GFX11-WGP-NEXT:    s_clause 0x1
803; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
804; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
805; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
806; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
807; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
808; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
809; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
810; GFX11-WGP-NEXT:    s_endpgm
811;
812; GFX11-CU-LABEL: flat_system_monotonic_store:
813; GFX11-CU:       ; %bb.0: ; %entry
814; GFX11-CU-NEXT:    s_clause 0x1
815; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
816; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
817; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
818; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
819; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
820; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
821; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
822; GFX11-CU-NEXT:    s_endpgm
823    i32 %in, i32* %out) {
824entry:
825  store atomic i32 %in, i32* %out monotonic, align 4
826  ret void
827}
828
829define amdgpu_kernel void @flat_system_release_store(
830; GFX7-LABEL: flat_system_release_store:
831; GFX7:       ; %bb.0: ; %entry
832; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
833; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
834; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
835; GFX7-NEXT:    v_mov_b32_e32 v0, s0
836; GFX7-NEXT:    v_mov_b32_e32 v1, s1
837; GFX7-NEXT:    v_mov_b32_e32 v2, s2
838; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
839; GFX7-NEXT:    flat_store_dword v[0:1], v2
840; GFX7-NEXT:    s_endpgm
841;
842; GFX10-WGP-LABEL: flat_system_release_store:
843; GFX10-WGP:       ; %bb.0: ; %entry
844; GFX10-WGP-NEXT:    s_clause 0x1
845; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
846; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
847; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
848; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
849; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
850; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
851; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
852; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
853; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
854; GFX10-WGP-NEXT:    s_endpgm
855;
856; GFX10-CU-LABEL: flat_system_release_store:
857; GFX10-CU:       ; %bb.0: ; %entry
858; GFX10-CU-NEXT:    s_clause 0x1
859; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
860; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
861; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
862; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
863; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
864; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
865; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
866; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
867; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
868; GFX10-CU-NEXT:    s_endpgm
869;
870; SKIP-CACHE-INV-LABEL: flat_system_release_store:
871; SKIP-CACHE-INV:       ; %bb.0: ; %entry
872; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
873; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
874; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
875; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
876; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
877; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
878; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
879; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
880; SKIP-CACHE-INV-NEXT:    s_endpgm
881;
882; GFX90A-NOTTGSPLIT-LABEL: flat_system_release_store:
883; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
884; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
885; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
886; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
887; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
888; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
889; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
890; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
891; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
892; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
893;
894; GFX90A-TGSPLIT-LABEL: flat_system_release_store:
895; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
896; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
897; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
898; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
899; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
900; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
901; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
902; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
903; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
904; GFX90A-TGSPLIT-NEXT:    s_endpgm
905;
906; GFX940-NOTTGSPLIT-LABEL: flat_system_release_store:
907; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
908; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
909; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
910; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
911; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
912; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
913; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
914; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
915; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
916; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
917;
918; GFX940-TGSPLIT-LABEL: flat_system_release_store:
919; GFX940-TGSPLIT:       ; %bb.0: ; %entry
920; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
921; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
922; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
923; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
924; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
925; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
926; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
927; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
928; GFX940-TGSPLIT-NEXT:    s_endpgm
929;
930; GFX11-WGP-LABEL: flat_system_release_store:
931; GFX11-WGP:       ; %bb.0: ; %entry
932; GFX11-WGP-NEXT:    s_clause 0x1
933; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
934; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
935; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
936; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
937; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
938; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
939; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
940; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
941; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
942; GFX11-WGP-NEXT:    s_endpgm
943;
944; GFX11-CU-LABEL: flat_system_release_store:
945; GFX11-CU:       ; %bb.0: ; %entry
946; GFX11-CU-NEXT:    s_clause 0x1
947; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
948; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
949; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
950; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
951; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
952; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
953; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
954; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
955; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
956; GFX11-CU-NEXT:    s_endpgm
957    i32 %in, i32* %out) {
958entry:
959  store atomic i32 %in, i32* %out release, align 4
960  ret void
961}
962
963define amdgpu_kernel void @flat_system_seq_cst_store(
964; GFX7-LABEL: flat_system_seq_cst_store:
965; GFX7:       ; %bb.0: ; %entry
966; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
967; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
968; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
969; GFX7-NEXT:    v_mov_b32_e32 v0, s0
970; GFX7-NEXT:    v_mov_b32_e32 v1, s1
971; GFX7-NEXT:    v_mov_b32_e32 v2, s2
972; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
973; GFX7-NEXT:    flat_store_dword v[0:1], v2
974; GFX7-NEXT:    s_endpgm
975;
976; GFX10-WGP-LABEL: flat_system_seq_cst_store:
977; GFX10-WGP:       ; %bb.0: ; %entry
978; GFX10-WGP-NEXT:    s_clause 0x1
979; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
980; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
981; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
982; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
983; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
984; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
985; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
986; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
987; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
988; GFX10-WGP-NEXT:    s_endpgm
989;
990; GFX10-CU-LABEL: flat_system_seq_cst_store:
991; GFX10-CU:       ; %bb.0: ; %entry
992; GFX10-CU-NEXT:    s_clause 0x1
993; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
994; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
995; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
996; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
997; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
998; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
999; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1000; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1001; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1002; GFX10-CU-NEXT:    s_endpgm
1003;
1004; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_store:
1005; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1006; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
1007; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
1008; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1009; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1010; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1011; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1012; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1013; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1014; SKIP-CACHE-INV-NEXT:    s_endpgm
1015;
1016; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_store:
1017; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1018; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1019; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
1020; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1021; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1022; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1023; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
1024; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1025; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1026; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1027;
1028; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_store:
1029; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1030; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1031; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
1032; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1033; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1034; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1035; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
1036; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1037; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1038; GFX90A-TGSPLIT-NEXT:    s_endpgm
1039;
1040; GFX940-NOTTGSPLIT-LABEL: flat_system_seq_cst_store:
1041; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1042; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
1043; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
1044; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1045; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1046; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1047; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
1048; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1049; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
1050; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1051;
1052; GFX940-TGSPLIT-LABEL: flat_system_seq_cst_store:
1053; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1054; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
1055; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
1056; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1057; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1058; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1059; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
1060; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1061; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
1062; GFX940-TGSPLIT-NEXT:    s_endpgm
1063;
1064; GFX11-WGP-LABEL: flat_system_seq_cst_store:
1065; GFX11-WGP:       ; %bb.0: ; %entry
1066; GFX11-WGP-NEXT:    s_clause 0x1
1067; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
1068; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
1069; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1070; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1071; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1072; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1073; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1074; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
1075; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1076; GFX11-WGP-NEXT:    s_endpgm
1077;
1078; GFX11-CU-LABEL: flat_system_seq_cst_store:
1079; GFX11-CU:       ; %bb.0: ; %entry
1080; GFX11-CU-NEXT:    s_clause 0x1
1081; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
1082; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
1083; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1084; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1085; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1086; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1087; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1088; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
1089; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1090; GFX11-CU-NEXT:    s_endpgm
1091    i32 %in, i32* %out) {
1092entry:
1093  store atomic i32 %in, i32* %out seq_cst, align 4
1094  ret void
1095}
1096
1097define amdgpu_kernel void @flat_system_monotonic_atomicrmw(
1098; GFX7-LABEL: flat_system_monotonic_atomicrmw:
1099; GFX7:       ; %bb.0: ; %entry
1100; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1101; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1102; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1103; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1104; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1105; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1106; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1107; GFX7-NEXT:    s_endpgm
1108;
1109; GFX10-WGP-LABEL: flat_system_monotonic_atomicrmw:
1110; GFX10-WGP:       ; %bb.0: ; %entry
1111; GFX10-WGP-NEXT:    s_clause 0x1
1112; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1113; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1114; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1115; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1116; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1117; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1118; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1119; GFX10-WGP-NEXT:    s_endpgm
1120;
1121; GFX10-CU-LABEL: flat_system_monotonic_atomicrmw:
1122; GFX10-CU:       ; %bb.0: ; %entry
1123; GFX10-CU-NEXT:    s_clause 0x1
1124; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1125; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1126; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1127; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1128; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1129; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1130; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1131; GFX10-CU-NEXT:    s_endpgm
1132;
1133; SKIP-CACHE-INV-LABEL: flat_system_monotonic_atomicrmw:
1134; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1135; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1136; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1137; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1138; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1139; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1140; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1141; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1142; SKIP-CACHE-INV-NEXT:    s_endpgm
1143;
1144; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_atomicrmw:
1145; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1146; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1147; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1148; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1149; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1150; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1151; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1152; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1153;
1154; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_atomicrmw:
1155; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1156; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1157; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1158; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1159; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1160; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1161; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1162; GFX90A-TGSPLIT-NEXT:    s_endpgm
1163;
1164; GFX940-NOTTGSPLIT-LABEL: flat_system_monotonic_atomicrmw:
1165; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1166; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1167; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1168; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1169; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1170; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1171; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
1172; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1173;
1174; GFX940-TGSPLIT-LABEL: flat_system_monotonic_atomicrmw:
1175; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1176; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1177; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1178; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1179; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1180; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1181; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
1182; GFX940-TGSPLIT-NEXT:    s_endpgm
1183;
1184; GFX11-WGP-LABEL: flat_system_monotonic_atomicrmw:
1185; GFX11-WGP:       ; %bb.0: ; %entry
1186; GFX11-WGP-NEXT:    s_clause 0x1
1187; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1188; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1189; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1190; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1191; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1192; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1193; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1194; GFX11-WGP-NEXT:    s_endpgm
1195;
1196; GFX11-CU-LABEL: flat_system_monotonic_atomicrmw:
1197; GFX11-CU:       ; %bb.0: ; %entry
1198; GFX11-CU-NEXT:    s_clause 0x1
1199; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1200; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1201; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1202; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1203; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1204; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1205; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1206; GFX11-CU-NEXT:    s_endpgm
1207    i32* %out, i32 %in) {
1208entry:
1209  %val = atomicrmw volatile xchg i32* %out, i32 %in monotonic
1210  ret void
1211}
1212
1213define amdgpu_kernel void @flat_system_acquire_atomicrmw(
1214; GFX7-LABEL: flat_system_acquire_atomicrmw:
1215; GFX7:       ; %bb.0: ; %entry
1216; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1217; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1218; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1219; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1220; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1221; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1222; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1223; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1224; GFX7-NEXT:    buffer_wbinvl1_vol
1225; GFX7-NEXT:    s_endpgm
1226;
1227; GFX10-WGP-LABEL: flat_system_acquire_atomicrmw:
1228; GFX10-WGP:       ; %bb.0: ; %entry
1229; GFX10-WGP-NEXT:    s_clause 0x1
1230; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1231; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1232; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1233; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1234; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1235; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1236; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1237; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1238; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1239; GFX10-WGP-NEXT:    buffer_gl0_inv
1240; GFX10-WGP-NEXT:    buffer_gl1_inv
1241; GFX10-WGP-NEXT:    s_endpgm
1242;
1243; GFX10-CU-LABEL: flat_system_acquire_atomicrmw:
1244; GFX10-CU:       ; %bb.0: ; %entry
1245; GFX10-CU-NEXT:    s_clause 0x1
1246; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1247; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1248; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1249; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1250; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1251; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1252; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1253; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1254; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1255; GFX10-CU-NEXT:    buffer_gl0_inv
1256; GFX10-CU-NEXT:    buffer_gl1_inv
1257; GFX10-CU-NEXT:    s_endpgm
1258;
1259; SKIP-CACHE-INV-LABEL: flat_system_acquire_atomicrmw:
1260; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1261; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1262; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1263; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1264; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1265; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1266; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1267; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1268; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1269; SKIP-CACHE-INV-NEXT:    s_endpgm
1270;
1271; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_atomicrmw:
1272; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1273; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1274; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1275; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1276; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1277; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1278; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1279; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1280; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
1281; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1282; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1283;
1284; GFX90A-TGSPLIT-LABEL: flat_system_acquire_atomicrmw:
1285; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1286; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1287; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1288; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1289; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1290; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1291; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1292; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1293; GFX90A-TGSPLIT-NEXT:    buffer_invl2
1294; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1295; GFX90A-TGSPLIT-NEXT:    s_endpgm
1296;
1297; GFX940-NOTTGSPLIT-LABEL: flat_system_acquire_atomicrmw:
1298; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1299; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1300; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1301; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1302; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1303; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1304; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
1305; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1306; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
1307; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1308;
1309; GFX940-TGSPLIT-LABEL: flat_system_acquire_atomicrmw:
1310; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1311; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1312; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1313; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1314; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1315; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1316; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
1317; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1318; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
1319; GFX940-TGSPLIT-NEXT:    s_endpgm
1320;
1321; GFX11-WGP-LABEL: flat_system_acquire_atomicrmw:
1322; GFX11-WGP:       ; %bb.0: ; %entry
1323; GFX11-WGP-NEXT:    s_clause 0x1
1324; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1325; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1326; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1327; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1328; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1329; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1330; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1331; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1332; GFX11-WGP-NEXT:    buffer_gl0_inv
1333; GFX11-WGP-NEXT:    buffer_gl1_inv
1334; GFX11-WGP-NEXT:    s_endpgm
1335;
1336; GFX11-CU-LABEL: flat_system_acquire_atomicrmw:
1337; GFX11-CU:       ; %bb.0: ; %entry
1338; GFX11-CU-NEXT:    s_clause 0x1
1339; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1340; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1341; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1342; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1343; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1344; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1345; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1346; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1347; GFX11-CU-NEXT:    buffer_gl0_inv
1348; GFX11-CU-NEXT:    buffer_gl1_inv
1349; GFX11-CU-NEXT:    s_endpgm
1350    i32* %out, i32 %in) {
1351entry:
1352  %val = atomicrmw volatile xchg i32* %out, i32 %in acquire
1353  ret void
1354}
1355
1356define amdgpu_kernel void @flat_system_release_atomicrmw(
1357; GFX7-LABEL: flat_system_release_atomicrmw:
1358; GFX7:       ; %bb.0: ; %entry
1359; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1360; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1361; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1362; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1363; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1364; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1365; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1366; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1367; GFX7-NEXT:    s_endpgm
1368;
1369; GFX10-WGP-LABEL: flat_system_release_atomicrmw:
1370; GFX10-WGP:       ; %bb.0: ; %entry
1371; GFX10-WGP-NEXT:    s_clause 0x1
1372; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1373; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1374; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1375; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1376; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1377; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1378; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1379; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1380; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1381; GFX10-WGP-NEXT:    s_endpgm
1382;
1383; GFX10-CU-LABEL: flat_system_release_atomicrmw:
1384; GFX10-CU:       ; %bb.0: ; %entry
1385; GFX10-CU-NEXT:    s_clause 0x1
1386; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1387; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1388; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1389; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1390; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1391; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1392; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1393; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1394; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1395; GFX10-CU-NEXT:    s_endpgm
1396;
1397; SKIP-CACHE-INV-LABEL: flat_system_release_atomicrmw:
1398; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1399; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1400; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1401; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1402; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1403; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1404; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1405; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1406; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1407; SKIP-CACHE-INV-NEXT:    s_endpgm
1408;
1409; GFX90A-NOTTGSPLIT-LABEL: flat_system_release_atomicrmw:
1410; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1411; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1412; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1413; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1414; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1415; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1416; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
1417; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1418; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1419; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1420;
1421; GFX90A-TGSPLIT-LABEL: flat_system_release_atomicrmw:
1422; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1423; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1424; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1425; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1426; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1427; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1428; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
1429; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1430; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1431; GFX90A-TGSPLIT-NEXT:    s_endpgm
1432;
1433; GFX940-NOTTGSPLIT-LABEL: flat_system_release_atomicrmw:
1434; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1435; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1436; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1437; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1438; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1439; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1440; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
1441; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1442; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
1443; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1444;
1445; GFX940-TGSPLIT-LABEL: flat_system_release_atomicrmw:
1446; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1447; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1448; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1449; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1450; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1451; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1452; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
1453; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1454; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
1455; GFX940-TGSPLIT-NEXT:    s_endpgm
1456;
1457; GFX11-WGP-LABEL: flat_system_release_atomicrmw:
1458; GFX11-WGP:       ; %bb.0: ; %entry
1459; GFX11-WGP-NEXT:    s_clause 0x1
1460; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1461; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1462; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1463; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1464; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1465; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1466; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1467; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1468; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1469; GFX11-WGP-NEXT:    s_endpgm
1470;
1471; GFX11-CU-LABEL: flat_system_release_atomicrmw:
1472; GFX11-CU:       ; %bb.0: ; %entry
1473; GFX11-CU-NEXT:    s_clause 0x1
1474; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1475; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1476; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1477; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1478; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1479; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1480; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1481; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1482; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1483; GFX11-CU-NEXT:    s_endpgm
1484    i32* %out, i32 %in) {
1485entry:
1486  %val = atomicrmw volatile xchg i32* %out, i32 %in release
1487  ret void
1488}
1489
1490define amdgpu_kernel void @flat_system_acq_rel_atomicrmw(
1491; GFX7-LABEL: flat_system_acq_rel_atomicrmw:
1492; GFX7:       ; %bb.0: ; %entry
1493; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1494; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1495; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1496; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1497; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1498; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1499; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1500; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1501; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1502; GFX7-NEXT:    buffer_wbinvl1_vol
1503; GFX7-NEXT:    s_endpgm
1504;
1505; GFX10-WGP-LABEL: flat_system_acq_rel_atomicrmw:
1506; GFX10-WGP:       ; %bb.0: ; %entry
1507; GFX10-WGP-NEXT:    s_clause 0x1
1508; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1509; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1510; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1511; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1512; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1513; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1514; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1515; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1516; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1517; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1518; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1519; GFX10-WGP-NEXT:    buffer_gl0_inv
1520; GFX10-WGP-NEXT:    buffer_gl1_inv
1521; GFX10-WGP-NEXT:    s_endpgm
1522;
1523; GFX10-CU-LABEL: flat_system_acq_rel_atomicrmw:
1524; GFX10-CU:       ; %bb.0: ; %entry
1525; GFX10-CU-NEXT:    s_clause 0x1
1526; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1527; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1528; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1529; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1530; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1531; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1532; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1533; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1534; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1535; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1536; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1537; GFX10-CU-NEXT:    buffer_gl0_inv
1538; GFX10-CU-NEXT:    buffer_gl1_inv
1539; GFX10-CU-NEXT:    s_endpgm
1540;
1541; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_atomicrmw:
1542; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1543; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1544; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1545; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1546; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1547; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1548; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1549; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1550; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1551; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1552; SKIP-CACHE-INV-NEXT:    s_endpgm
1553;
1554; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_atomicrmw:
1555; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1556; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1557; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1558; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1559; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1560; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1561; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
1562; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1563; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1564; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1565; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
1566; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1567; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1568;
1569; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_atomicrmw:
1570; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1571; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1572; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1573; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1574; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1575; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1576; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
1577; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1578; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1579; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1580; GFX90A-TGSPLIT-NEXT:    buffer_invl2
1581; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1582; GFX90A-TGSPLIT-NEXT:    s_endpgm
1583;
1584; GFX940-NOTTGSPLIT-LABEL: flat_system_acq_rel_atomicrmw:
1585; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1586; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1587; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1588; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1589; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1590; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1591; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
1592; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1593; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
1594; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1595; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
1596; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1597;
1598; GFX940-TGSPLIT-LABEL: flat_system_acq_rel_atomicrmw:
1599; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1600; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1601; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1602; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1603; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1604; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1605; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
1606; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1607; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
1608; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1609; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
1610; GFX940-TGSPLIT-NEXT:    s_endpgm
1611;
1612; GFX11-WGP-LABEL: flat_system_acq_rel_atomicrmw:
1613; GFX11-WGP:       ; %bb.0: ; %entry
1614; GFX11-WGP-NEXT:    s_clause 0x1
1615; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1616; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1617; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1618; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1619; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1620; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1621; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1622; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1623; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1624; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1625; GFX11-WGP-NEXT:    buffer_gl0_inv
1626; GFX11-WGP-NEXT:    buffer_gl1_inv
1627; GFX11-WGP-NEXT:    s_endpgm
1628;
1629; GFX11-CU-LABEL: flat_system_acq_rel_atomicrmw:
1630; GFX11-CU:       ; %bb.0: ; %entry
1631; GFX11-CU-NEXT:    s_clause 0x1
1632; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1633; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1634; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1635; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1636; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1637; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1638; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1639; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1640; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1641; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1642; GFX11-CU-NEXT:    buffer_gl0_inv
1643; GFX11-CU-NEXT:    buffer_gl1_inv
1644; GFX11-CU-NEXT:    s_endpgm
1645    i32* %out, i32 %in) {
1646entry:
1647  %val = atomicrmw volatile xchg i32* %out, i32 %in acq_rel
1648  ret void
1649}
1650
1651define amdgpu_kernel void @flat_system_seq_cst_atomicrmw(
1652; GFX7-LABEL: flat_system_seq_cst_atomicrmw:
1653; GFX7:       ; %bb.0: ; %entry
1654; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1655; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1656; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1657; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1658; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1659; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1660; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1661; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1662; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1663; GFX7-NEXT:    buffer_wbinvl1_vol
1664; GFX7-NEXT:    s_endpgm
1665;
1666; GFX10-WGP-LABEL: flat_system_seq_cst_atomicrmw:
1667; GFX10-WGP:       ; %bb.0: ; %entry
1668; GFX10-WGP-NEXT:    s_clause 0x1
1669; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1670; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1671; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1672; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1673; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1674; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1675; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1676; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1677; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1678; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1679; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1680; GFX10-WGP-NEXT:    buffer_gl0_inv
1681; GFX10-WGP-NEXT:    buffer_gl1_inv
1682; GFX10-WGP-NEXT:    s_endpgm
1683;
1684; GFX10-CU-LABEL: flat_system_seq_cst_atomicrmw:
1685; GFX10-CU:       ; %bb.0: ; %entry
1686; GFX10-CU-NEXT:    s_clause 0x1
1687; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1688; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1689; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1690; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1691; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1692; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1693; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1694; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1695; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1696; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1697; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1698; GFX10-CU-NEXT:    buffer_gl0_inv
1699; GFX10-CU-NEXT:    buffer_gl1_inv
1700; GFX10-CU-NEXT:    s_endpgm
1701;
1702; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_atomicrmw:
1703; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1704; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1705; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1706; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1707; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1708; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1709; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1710; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1711; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1712; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1713; SKIP-CACHE-INV-NEXT:    s_endpgm
1714;
1715; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_atomicrmw:
1716; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1717; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1718; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1719; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1720; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1721; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1722; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
1723; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1724; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1725; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1726; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
1727; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1728; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1729;
1730; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_atomicrmw:
1731; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1732; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1733; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1734; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1735; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1736; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1737; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
1738; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1739; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1740; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1741; GFX90A-TGSPLIT-NEXT:    buffer_invl2
1742; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1743; GFX90A-TGSPLIT-NEXT:    s_endpgm
1744;
1745; GFX940-NOTTGSPLIT-LABEL: flat_system_seq_cst_atomicrmw:
1746; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1747; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1748; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1749; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1750; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1751; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1752; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
1753; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1754; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
1755; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1756; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
1757; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1758;
1759; GFX940-TGSPLIT-LABEL: flat_system_seq_cst_atomicrmw:
1760; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1761; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1762; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1763; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1764; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1765; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1766; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
1767; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1768; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
1769; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1770; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
1771; GFX940-TGSPLIT-NEXT:    s_endpgm
1772;
1773; GFX11-WGP-LABEL: flat_system_seq_cst_atomicrmw:
1774; GFX11-WGP:       ; %bb.0: ; %entry
1775; GFX11-WGP-NEXT:    s_clause 0x1
1776; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1777; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1778; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1779; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1780; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1781; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1782; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1783; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1784; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1785; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1786; GFX11-WGP-NEXT:    buffer_gl0_inv
1787; GFX11-WGP-NEXT:    buffer_gl1_inv
1788; GFX11-WGP-NEXT:    s_endpgm
1789;
1790; GFX11-CU-LABEL: flat_system_seq_cst_atomicrmw:
1791; GFX11-CU:       ; %bb.0: ; %entry
1792; GFX11-CU-NEXT:    s_clause 0x1
1793; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1794; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1795; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1796; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1797; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1798; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1799; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1800; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1801; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1802; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1803; GFX11-CU-NEXT:    buffer_gl0_inv
1804; GFX11-CU-NEXT:    buffer_gl1_inv
1805; GFX11-CU-NEXT:    s_endpgm
1806    i32* %out, i32 %in) {
1807entry:
1808  %val = atomicrmw volatile xchg i32* %out, i32 %in seq_cst
1809  ret void
1810}
1811
1812define amdgpu_kernel void @flat_system_acquire_ret_atomicrmw(
1813; GFX7-LABEL: flat_system_acquire_ret_atomicrmw:
1814; GFX7:       ; %bb.0: ; %entry
1815; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1816; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1817; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1818; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1819; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1820; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1821; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1822; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1823; GFX7-NEXT:    buffer_wbinvl1_vol
1824; GFX7-NEXT:    flat_store_dword v[0:1], v2
1825; GFX7-NEXT:    s_endpgm
1826;
1827; GFX10-WGP-LABEL: flat_system_acquire_ret_atomicrmw:
1828; GFX10-WGP:       ; %bb.0: ; %entry
1829; GFX10-WGP-NEXT:    s_clause 0x1
1830; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1831; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1832; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1833; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1834; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1835; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1836; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1837; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1838; GFX10-WGP-NEXT:    buffer_gl0_inv
1839; GFX10-WGP-NEXT:    buffer_gl1_inv
1840; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1841; GFX10-WGP-NEXT:    s_endpgm
1842;
1843; GFX10-CU-LABEL: flat_system_acquire_ret_atomicrmw:
1844; GFX10-CU:       ; %bb.0: ; %entry
1845; GFX10-CU-NEXT:    s_clause 0x1
1846; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1847; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1848; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1849; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1850; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1851; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1852; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1853; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1854; GFX10-CU-NEXT:    buffer_gl0_inv
1855; GFX10-CU-NEXT:    buffer_gl1_inv
1856; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1857; GFX10-CU-NEXT:    s_endpgm
1858;
1859; SKIP-CACHE-INV-LABEL: flat_system_acquire_ret_atomicrmw:
1860; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1861; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1862; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1863; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1864; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1865; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1866; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1867; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1868; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1869; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1870; SKIP-CACHE-INV-NEXT:    s_endpgm
1871;
1872; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_ret_atomicrmw:
1873; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1874; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1875; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1876; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1877; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1878; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1879; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1880; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1881; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
1882; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1883; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1884; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1885;
1886; GFX90A-TGSPLIT-LABEL: flat_system_acquire_ret_atomicrmw:
1887; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1888; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1889; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1890; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1891; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1892; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1893; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1894; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1895; GFX90A-TGSPLIT-NEXT:    buffer_invl2
1896; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1897; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1898; GFX90A-TGSPLIT-NEXT:    s_endpgm
1899;
1900; GFX940-NOTTGSPLIT-LABEL: flat_system_acquire_ret_atomicrmw:
1901; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1902; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1903; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1904; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1905; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1906; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1907; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0 sc1
1908; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1909; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
1910; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1911; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1912;
1913; GFX940-TGSPLIT-LABEL: flat_system_acquire_ret_atomicrmw:
1914; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1915; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1916; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1917; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1918; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1919; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1920; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0 sc1
1921; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1922; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
1923; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1924; GFX940-TGSPLIT-NEXT:    s_endpgm
1925;
1926; GFX11-WGP-LABEL: flat_system_acquire_ret_atomicrmw:
1927; GFX11-WGP:       ; %bb.0: ; %entry
1928; GFX11-WGP-NEXT:    s_clause 0x1
1929; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1930; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1931; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1932; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1933; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1934; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
1935; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1936; GFX11-WGP-NEXT:    buffer_gl0_inv
1937; GFX11-WGP-NEXT:    buffer_gl1_inv
1938; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
1939; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1940; GFX11-WGP-NEXT:    s_endpgm
1941;
1942; GFX11-CU-LABEL: flat_system_acquire_ret_atomicrmw:
1943; GFX11-CU:       ; %bb.0: ; %entry
1944; GFX11-CU-NEXT:    s_clause 0x1
1945; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1946; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1947; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1948; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1949; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1950; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
1951; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1952; GFX11-CU-NEXT:    buffer_gl0_inv
1953; GFX11-CU-NEXT:    buffer_gl1_inv
1954; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
1955; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1956; GFX11-CU-NEXT:    s_endpgm
1957    i32* %out, i32 %in) {
1958entry:
1959  %val = atomicrmw volatile xchg i32* %out, i32 %in acquire
1960  store i32 %val, i32* %out, align 4
1961  ret void
1962}
1963
1964define amdgpu_kernel void @flat_system_acq_rel_ret_atomicrmw(
1965; GFX7-LABEL: flat_system_acq_rel_ret_atomicrmw:
1966; GFX7:       ; %bb.0: ; %entry
1967; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1968; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1969; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1970; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1971; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1972; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1973; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1974; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1975; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1976; GFX7-NEXT:    buffer_wbinvl1_vol
1977; GFX7-NEXT:    flat_store_dword v[0:1], v2
1978; GFX7-NEXT:    s_endpgm
1979;
1980; GFX10-WGP-LABEL: flat_system_acq_rel_ret_atomicrmw:
1981; GFX10-WGP:       ; %bb.0: ; %entry
1982; GFX10-WGP-NEXT:    s_clause 0x1
1983; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1984; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1985; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1986; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1987; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1988; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1989; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1990; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1991; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1992; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1993; GFX10-WGP-NEXT:    buffer_gl0_inv
1994; GFX10-WGP-NEXT:    buffer_gl1_inv
1995; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1996; GFX10-WGP-NEXT:    s_endpgm
1997;
1998; GFX10-CU-LABEL: flat_system_acq_rel_ret_atomicrmw:
1999; GFX10-CU:       ; %bb.0: ; %entry
2000; GFX10-CU-NEXT:    s_clause 0x1
2001; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2002; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
2003; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2004; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2005; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2006; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2007; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2008; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2009; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2010; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2011; GFX10-CU-NEXT:    buffer_gl0_inv
2012; GFX10-CU-NEXT:    buffer_gl1_inv
2013; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
2014; GFX10-CU-NEXT:    s_endpgm
2015;
2016; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_ret_atomicrmw:
2017; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2018; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2019; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
2020; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2021; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2022; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2023; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2024; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2025; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2026; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2027; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
2028; SKIP-CACHE-INV-NEXT:    s_endpgm
2029;
2030; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_ret_atomicrmw:
2031; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2032; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2033; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
2034; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2035; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2036; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
2037; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
2038; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2039; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2040; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2041; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
2042; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2043; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2044; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2045;
2046; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_ret_atomicrmw:
2047; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2048; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2049; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
2050; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2051; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2052; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
2053; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
2054; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2055; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2056; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2057; GFX90A-TGSPLIT-NEXT:    buffer_invl2
2058; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2059; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2060; GFX90A-TGSPLIT-NEXT:    s_endpgm
2061;
2062; GFX940-NOTTGSPLIT-LABEL: flat_system_acq_rel_ret_atomicrmw:
2063; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2064; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2065; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
2066; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2067; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2068; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
2069; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
2070; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2071; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0 sc1
2072; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2073; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
2074; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2075; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2076;
2077; GFX940-TGSPLIT-LABEL: flat_system_acq_rel_ret_atomicrmw:
2078; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2079; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2080; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
2081; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2082; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2083; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
2084; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
2085; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2086; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0 sc1
2087; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2088; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
2089; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2090; GFX940-TGSPLIT-NEXT:    s_endpgm
2091;
2092; GFX11-WGP-LABEL: flat_system_acq_rel_ret_atomicrmw:
2093; GFX11-WGP:       ; %bb.0: ; %entry
2094; GFX11-WGP-NEXT:    s_clause 0x1
2095; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2096; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
2097; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2098; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2099; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2100; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2101; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2102; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
2103; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2104; GFX11-WGP-NEXT:    buffer_gl0_inv
2105; GFX11-WGP-NEXT:    buffer_gl1_inv
2106; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
2107; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2108; GFX11-WGP-NEXT:    s_endpgm
2109;
2110; GFX11-CU-LABEL: flat_system_acq_rel_ret_atomicrmw:
2111; GFX11-CU:       ; %bb.0: ; %entry
2112; GFX11-CU-NEXT:    s_clause 0x1
2113; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2114; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
2115; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2116; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2117; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2118; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2119; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2120; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
2121; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2122; GFX11-CU-NEXT:    buffer_gl0_inv
2123; GFX11-CU-NEXT:    buffer_gl1_inv
2124; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
2125; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2126; GFX11-CU-NEXT:    s_endpgm
2127    i32* %out, i32 %in) {
2128entry:
2129  %val = atomicrmw volatile xchg i32* %out, i32 %in acq_rel
2130  store i32 %val, i32* %out, align 4
2131  ret void
2132}
2133
2134define amdgpu_kernel void @flat_system_seq_cst_ret_atomicrmw(
2135; GFX7-LABEL: flat_system_seq_cst_ret_atomicrmw:
2136; GFX7:       ; %bb.0: ; %entry
2137; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2138; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
2139; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2140; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2141; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2142; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2143; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2144; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2145; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2146; GFX7-NEXT:    buffer_wbinvl1_vol
2147; GFX7-NEXT:    flat_store_dword v[0:1], v2
2148; GFX7-NEXT:    s_endpgm
2149;
2150; GFX10-WGP-LABEL: flat_system_seq_cst_ret_atomicrmw:
2151; GFX10-WGP:       ; %bb.0: ; %entry
2152; GFX10-WGP-NEXT:    s_clause 0x1
2153; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2154; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
2155; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2156; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2157; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2158; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2159; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2160; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2161; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2162; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2163; GFX10-WGP-NEXT:    buffer_gl0_inv
2164; GFX10-WGP-NEXT:    buffer_gl1_inv
2165; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
2166; GFX10-WGP-NEXT:    s_endpgm
2167;
2168; GFX10-CU-LABEL: flat_system_seq_cst_ret_atomicrmw:
2169; GFX10-CU:       ; %bb.0: ; %entry
2170; GFX10-CU-NEXT:    s_clause 0x1
2171; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2172; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
2173; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2174; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2175; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2176; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2177; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2178; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2179; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2180; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2181; GFX10-CU-NEXT:    buffer_gl0_inv
2182; GFX10-CU-NEXT:    buffer_gl1_inv
2183; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
2184; GFX10-CU-NEXT:    s_endpgm
2185;
2186; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_ret_atomicrmw:
2187; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2188; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2189; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
2190; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2191; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2192; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2193; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2194; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2195; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2196; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2197; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
2198; SKIP-CACHE-INV-NEXT:    s_endpgm
2199;
2200; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_ret_atomicrmw:
2201; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2202; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2203; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
2204; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2205; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2206; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
2207; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
2208; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2209; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2210; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2211; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
2212; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2213; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2214; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2215;
2216; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_ret_atomicrmw:
2217; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2218; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2219; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
2220; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2221; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2222; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
2223; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
2224; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2225; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2226; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2227; GFX90A-TGSPLIT-NEXT:    buffer_invl2
2228; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2229; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2230; GFX90A-TGSPLIT-NEXT:    s_endpgm
2231;
2232; GFX940-NOTTGSPLIT-LABEL: flat_system_seq_cst_ret_atomicrmw:
2233; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2234; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2235; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
2236; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2237; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2238; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
2239; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
2240; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2241; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0 sc1
2242; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2243; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
2244; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2245; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2246;
2247; GFX940-TGSPLIT-LABEL: flat_system_seq_cst_ret_atomicrmw:
2248; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2249; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2250; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
2251; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2252; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2253; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
2254; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
2255; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2256; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0 sc1
2257; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2258; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
2259; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2260; GFX940-TGSPLIT-NEXT:    s_endpgm
2261;
2262; GFX11-WGP-LABEL: flat_system_seq_cst_ret_atomicrmw:
2263; GFX11-WGP:       ; %bb.0: ; %entry
2264; GFX11-WGP-NEXT:    s_clause 0x1
2265; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2266; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
2267; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2268; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2269; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2270; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2271; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2272; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
2273; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2274; GFX11-WGP-NEXT:    buffer_gl0_inv
2275; GFX11-WGP-NEXT:    buffer_gl1_inv
2276; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
2277; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2278; GFX11-WGP-NEXT:    s_endpgm
2279;
2280; GFX11-CU-LABEL: flat_system_seq_cst_ret_atomicrmw:
2281; GFX11-CU:       ; %bb.0: ; %entry
2282; GFX11-CU-NEXT:    s_clause 0x1
2283; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2284; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
2285; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2286; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2287; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2288; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2289; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2290; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
2291; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2292; GFX11-CU-NEXT:    buffer_gl0_inv
2293; GFX11-CU-NEXT:    buffer_gl1_inv
2294; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
2295; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2296; GFX11-CU-NEXT:    s_endpgm
2297    i32* %out, i32 %in) {
2298entry:
2299  %val = atomicrmw volatile xchg i32* %out, i32 %in seq_cst
2300  store i32 %val, i32* %out, align 4
2301  ret void
2302}
2303
2304define amdgpu_kernel void @flat_system_monotonic_monotonic_cmpxchg(
2305; GFX7-LABEL: flat_system_monotonic_monotonic_cmpxchg:
2306; GFX7:       ; %bb.0: ; %entry
2307; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2308; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2309; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2310; GFX7-NEXT:    s_add_u32 s0, s0, 16
2311; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2312; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2313; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2314; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2315; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2316; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2317; GFX7-NEXT:    s_endpgm
2318;
2319; GFX10-WGP-LABEL: flat_system_monotonic_monotonic_cmpxchg:
2320; GFX10-WGP:       ; %bb.0: ; %entry
2321; GFX10-WGP-NEXT:    s_clause 0x1
2322; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2323; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2324; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2325; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2326; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2327; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2328; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2329; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2330; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2331; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2332; GFX10-WGP-NEXT:    s_endpgm
2333;
2334; GFX10-CU-LABEL: flat_system_monotonic_monotonic_cmpxchg:
2335; GFX10-CU:       ; %bb.0: ; %entry
2336; GFX10-CU-NEXT:    s_clause 0x1
2337; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2338; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2339; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2340; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2341; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2342; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2343; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2344; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2345; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2346; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2347; GFX10-CU-NEXT:    s_endpgm
2348;
2349; SKIP-CACHE-INV-LABEL: flat_system_monotonic_monotonic_cmpxchg:
2350; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2351; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2352; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2353; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2354; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2355; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2356; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2357; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2358; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2359; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2360; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2361; SKIP-CACHE-INV-NEXT:    s_endpgm
2362;
2363; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_monotonic_cmpxchg:
2364; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2365; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2366; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2367; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2368; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2369; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2370; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2371; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2372;
2373; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_monotonic_cmpxchg:
2374; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2375; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2376; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2377; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2378; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2379; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2380; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2381; GFX90A-TGSPLIT-NEXT:    s_endpgm
2382;
2383; GFX940-NOTTGSPLIT-LABEL: flat_system_monotonic_monotonic_cmpxchg:
2384; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2385; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2386; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2387; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2388; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2389; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2390; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
2391; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2392;
2393; GFX940-TGSPLIT-LABEL: flat_system_monotonic_monotonic_cmpxchg:
2394; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2395; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2396; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2397; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2398; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2399; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2400; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
2401; GFX940-TGSPLIT-NEXT:    s_endpgm
2402;
2403; GFX11-WGP-LABEL: flat_system_monotonic_monotonic_cmpxchg:
2404; GFX11-WGP:       ; %bb.0: ; %entry
2405; GFX11-WGP-NEXT:    s_clause 0x1
2406; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2407; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2408; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2409; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2410; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
2411; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2412; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2413; GFX11-WGP-NEXT:    s_endpgm
2414;
2415; GFX11-CU-LABEL: flat_system_monotonic_monotonic_cmpxchg:
2416; GFX11-CU:       ; %bb.0: ; %entry
2417; GFX11-CU-NEXT:    s_clause 0x1
2418; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2419; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2420; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2421; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2422; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
2423; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2424; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2425; GFX11-CU-NEXT:    s_endpgm
2426    i32* %out, i32 %in, i32 %old) {
2427entry:
2428  %gep = getelementptr i32, i32* %out, i32 4
2429  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in monotonic monotonic
2430  ret void
2431}
2432
2433define amdgpu_kernel void @flat_system_acquire_monotonic_cmpxchg(
2434; GFX7-LABEL: flat_system_acquire_monotonic_cmpxchg:
2435; GFX7:       ; %bb.0: ; %entry
2436; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2437; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2438; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2439; GFX7-NEXT:    s_add_u32 s0, s0, 16
2440; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2441; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2442; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2443; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2444; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2445; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2446; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2447; GFX7-NEXT:    buffer_wbinvl1_vol
2448; GFX7-NEXT:    s_endpgm
2449;
2450; GFX10-WGP-LABEL: flat_system_acquire_monotonic_cmpxchg:
2451; GFX10-WGP:       ; %bb.0: ; %entry
2452; GFX10-WGP-NEXT:    s_clause 0x1
2453; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2454; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2455; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2456; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2457; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2458; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2459; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2460; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2461; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2462; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2463; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2464; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2465; GFX10-WGP-NEXT:    buffer_gl0_inv
2466; GFX10-WGP-NEXT:    buffer_gl1_inv
2467; GFX10-WGP-NEXT:    s_endpgm
2468;
2469; GFX10-CU-LABEL: flat_system_acquire_monotonic_cmpxchg:
2470; GFX10-CU:       ; %bb.0: ; %entry
2471; GFX10-CU-NEXT:    s_clause 0x1
2472; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2473; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2474; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2475; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2476; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2477; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2478; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2479; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2480; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2481; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2482; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2483; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2484; GFX10-CU-NEXT:    buffer_gl0_inv
2485; GFX10-CU-NEXT:    buffer_gl1_inv
2486; GFX10-CU-NEXT:    s_endpgm
2487;
2488; SKIP-CACHE-INV-LABEL: flat_system_acquire_monotonic_cmpxchg:
2489; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2490; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2491; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2492; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2493; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2494; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2495; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2496; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2497; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2498; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2499; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2500; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2501; SKIP-CACHE-INV-NEXT:    s_endpgm
2502;
2503; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_monotonic_cmpxchg:
2504; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2505; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2506; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2507; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2508; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2509; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2510; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2511; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2512; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
2513; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2514; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2515;
2516; GFX90A-TGSPLIT-LABEL: flat_system_acquire_monotonic_cmpxchg:
2517; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2518; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2519; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2520; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2521; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2522; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2523; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2524; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2525; GFX90A-TGSPLIT-NEXT:    buffer_invl2
2526; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2527; GFX90A-TGSPLIT-NEXT:    s_endpgm
2528;
2529; GFX940-NOTTGSPLIT-LABEL: flat_system_acquire_monotonic_cmpxchg:
2530; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2531; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2532; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2533; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2534; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2535; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2536; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
2537; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2538; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
2539; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2540;
2541; GFX940-TGSPLIT-LABEL: flat_system_acquire_monotonic_cmpxchg:
2542; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2543; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2544; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2545; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2546; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2547; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2548; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
2549; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2550; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
2551; GFX940-TGSPLIT-NEXT:    s_endpgm
2552;
2553; GFX11-WGP-LABEL: flat_system_acquire_monotonic_cmpxchg:
2554; GFX11-WGP:       ; %bb.0: ; %entry
2555; GFX11-WGP-NEXT:    s_clause 0x1
2556; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2557; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2558; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2559; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2560; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
2561; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2562; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2563; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2564; GFX11-WGP-NEXT:    buffer_gl0_inv
2565; GFX11-WGP-NEXT:    buffer_gl1_inv
2566; GFX11-WGP-NEXT:    s_endpgm
2567;
2568; GFX11-CU-LABEL: flat_system_acquire_monotonic_cmpxchg:
2569; GFX11-CU:       ; %bb.0: ; %entry
2570; GFX11-CU-NEXT:    s_clause 0x1
2571; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2572; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2573; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2574; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2575; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
2576; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2577; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2578; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2579; GFX11-CU-NEXT:    buffer_gl0_inv
2580; GFX11-CU-NEXT:    buffer_gl1_inv
2581; GFX11-CU-NEXT:    s_endpgm
2582    i32* %out, i32 %in, i32 %old) {
2583entry:
2584  %gep = getelementptr i32, i32* %out, i32 4
2585  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acquire monotonic
2586  ret void
2587}
2588
2589define amdgpu_kernel void @flat_system_release_monotonic_cmpxchg(
2590; GFX7-LABEL: flat_system_release_monotonic_cmpxchg:
2591; GFX7:       ; %bb.0: ; %entry
2592; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2593; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2594; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2595; GFX7-NEXT:    s_add_u32 s0, s0, 16
2596; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2597; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2598; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2599; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2600; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2601; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2602; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2603; GFX7-NEXT:    s_endpgm
2604;
2605; GFX10-WGP-LABEL: flat_system_release_monotonic_cmpxchg:
2606; GFX10-WGP:       ; %bb.0: ; %entry
2607; GFX10-WGP-NEXT:    s_clause 0x1
2608; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2609; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2610; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2611; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2612; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2613; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2614; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2615; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2616; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2617; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2618; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2619; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2620; GFX10-WGP-NEXT:    s_endpgm
2621;
2622; GFX10-CU-LABEL: flat_system_release_monotonic_cmpxchg:
2623; GFX10-CU:       ; %bb.0: ; %entry
2624; GFX10-CU-NEXT:    s_clause 0x1
2625; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2626; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2627; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2628; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2629; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2630; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2631; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2632; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2633; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2634; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2635; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2636; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2637; GFX10-CU-NEXT:    s_endpgm
2638;
2639; SKIP-CACHE-INV-LABEL: flat_system_release_monotonic_cmpxchg:
2640; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2641; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2642; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2643; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2644; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2645; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2646; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2647; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2648; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2649; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2650; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2651; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2652; SKIP-CACHE-INV-NEXT:    s_endpgm
2653;
2654; GFX90A-NOTTGSPLIT-LABEL: flat_system_release_monotonic_cmpxchg:
2655; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2656; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2657; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2658; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2659; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2660; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2661; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
2662; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2663; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2664; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2665;
2666; GFX90A-TGSPLIT-LABEL: flat_system_release_monotonic_cmpxchg:
2667; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2668; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2669; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2670; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2671; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2672; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2673; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
2674; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2675; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2676; GFX90A-TGSPLIT-NEXT:    s_endpgm
2677;
2678; GFX940-NOTTGSPLIT-LABEL: flat_system_release_monotonic_cmpxchg:
2679; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2680; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2681; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2682; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2683; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2684; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2685; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
2686; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2687; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
2688; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2689;
2690; GFX940-TGSPLIT-LABEL: flat_system_release_monotonic_cmpxchg:
2691; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2692; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2693; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2694; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2695; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2696; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2697; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
2698; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2699; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
2700; GFX940-TGSPLIT-NEXT:    s_endpgm
2701;
2702; GFX11-WGP-LABEL: flat_system_release_monotonic_cmpxchg:
2703; GFX11-WGP:       ; %bb.0: ; %entry
2704; GFX11-WGP-NEXT:    s_clause 0x1
2705; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2706; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2707; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2708; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2709; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
2710; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2711; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2712; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2713; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2714; GFX11-WGP-NEXT:    s_endpgm
2715;
2716; GFX11-CU-LABEL: flat_system_release_monotonic_cmpxchg:
2717; GFX11-CU:       ; %bb.0: ; %entry
2718; GFX11-CU-NEXT:    s_clause 0x1
2719; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2720; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2721; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2722; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2723; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
2724; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2725; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2726; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2727; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2728; GFX11-CU-NEXT:    s_endpgm
2729    i32* %out, i32 %in, i32 %old) {
2730entry:
2731  %gep = getelementptr i32, i32* %out, i32 4
2732  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in release monotonic
2733  ret void
2734}
2735
2736define amdgpu_kernel void @flat_system_acq_rel_monotonic_cmpxchg(
2737; GFX7-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
2738; GFX7:       ; %bb.0: ; %entry
2739; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2740; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2741; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2742; GFX7-NEXT:    s_add_u32 s0, s0, 16
2743; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2744; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2745; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2746; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2747; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2748; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2749; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2750; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2751; GFX7-NEXT:    buffer_wbinvl1_vol
2752; GFX7-NEXT:    s_endpgm
2753;
2754; GFX10-WGP-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
2755; GFX10-WGP:       ; %bb.0: ; %entry
2756; GFX10-WGP-NEXT:    s_clause 0x1
2757; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2758; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2759; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2760; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2761; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2762; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2763; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2764; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2765; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2766; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2767; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2768; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2769; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2770; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2771; GFX10-WGP-NEXT:    buffer_gl0_inv
2772; GFX10-WGP-NEXT:    buffer_gl1_inv
2773; GFX10-WGP-NEXT:    s_endpgm
2774;
2775; GFX10-CU-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
2776; GFX10-CU:       ; %bb.0: ; %entry
2777; GFX10-CU-NEXT:    s_clause 0x1
2778; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2779; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2780; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2781; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2782; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2783; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2784; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2785; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2786; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2787; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2788; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2789; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2790; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2791; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2792; GFX10-CU-NEXT:    buffer_gl0_inv
2793; GFX10-CU-NEXT:    buffer_gl1_inv
2794; GFX10-CU-NEXT:    s_endpgm
2795;
2796; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
2797; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2798; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2799; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2800; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2801; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2802; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2803; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2804; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2805; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2806; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2807; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2808; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2809; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2810; SKIP-CACHE-INV-NEXT:    s_endpgm
2811;
2812; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
2813; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2814; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2815; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2816; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2817; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2818; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2819; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
2820; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2821; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2822; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2823; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
2824; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2825; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2826;
2827; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
2828; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2829; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2830; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2831; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2832; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2833; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2834; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
2835; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2836; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2837; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2838; GFX90A-TGSPLIT-NEXT:    buffer_invl2
2839; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2840; GFX90A-TGSPLIT-NEXT:    s_endpgm
2841;
2842; GFX940-NOTTGSPLIT-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
2843; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2844; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2845; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2846; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2847; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2848; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2849; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
2850; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2851; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
2852; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2853; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
2854; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2855;
2856; GFX940-TGSPLIT-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
2857; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2858; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2859; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2860; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2861; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2862; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2863; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
2864; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2865; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
2866; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2867; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
2868; GFX940-TGSPLIT-NEXT:    s_endpgm
2869;
2870; GFX11-WGP-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
2871; GFX11-WGP:       ; %bb.0: ; %entry
2872; GFX11-WGP-NEXT:    s_clause 0x1
2873; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2874; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2875; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2876; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2877; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
2878; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2879; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2880; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2881; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2882; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2883; GFX11-WGP-NEXT:    buffer_gl0_inv
2884; GFX11-WGP-NEXT:    buffer_gl1_inv
2885; GFX11-WGP-NEXT:    s_endpgm
2886;
2887; GFX11-CU-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
2888; GFX11-CU:       ; %bb.0: ; %entry
2889; GFX11-CU-NEXT:    s_clause 0x1
2890; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2891; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2892; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2893; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2894; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
2895; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2896; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2897; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2898; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2899; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2900; GFX11-CU-NEXT:    buffer_gl0_inv
2901; GFX11-CU-NEXT:    buffer_gl1_inv
2902; GFX11-CU-NEXT:    s_endpgm
2903    i32* %out, i32 %in, i32 %old) {
2904entry:
2905  %gep = getelementptr i32, i32* %out, i32 4
2906  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acq_rel monotonic
2907  ret void
2908}
2909
2910define amdgpu_kernel void @flat_system_seq_cst_monotonic_cmpxchg(
2911; GFX7-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
2912; GFX7:       ; %bb.0: ; %entry
2913; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2914; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2915; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2916; GFX7-NEXT:    s_add_u32 s0, s0, 16
2917; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2918; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2919; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2920; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2921; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2922; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2923; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2924; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2925; GFX7-NEXT:    buffer_wbinvl1_vol
2926; GFX7-NEXT:    s_endpgm
2927;
2928; GFX10-WGP-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
2929; GFX10-WGP:       ; %bb.0: ; %entry
2930; GFX10-WGP-NEXT:    s_clause 0x1
2931; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2932; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2933; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2934; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2935; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2936; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2937; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2938; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2939; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2940; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2941; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2942; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2943; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2944; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2945; GFX10-WGP-NEXT:    buffer_gl0_inv
2946; GFX10-WGP-NEXT:    buffer_gl1_inv
2947; GFX10-WGP-NEXT:    s_endpgm
2948;
2949; GFX10-CU-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
2950; GFX10-CU:       ; %bb.0: ; %entry
2951; GFX10-CU-NEXT:    s_clause 0x1
2952; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2953; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2954; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2955; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2956; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2957; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2958; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2959; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2960; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2961; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2962; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2963; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2964; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2965; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2966; GFX10-CU-NEXT:    buffer_gl0_inv
2967; GFX10-CU-NEXT:    buffer_gl1_inv
2968; GFX10-CU-NEXT:    s_endpgm
2969;
2970; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
2971; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2972; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2973; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2974; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2975; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2976; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2977; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2978; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2979; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2980; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2981; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2982; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2983; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2984; SKIP-CACHE-INV-NEXT:    s_endpgm
2985;
2986; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
2987; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2988; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2989; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2990; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2991; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2992; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2993; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
2994; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2995; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2996; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2997; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
2998; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2999; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3000;
3001; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
3002; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3003; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3004; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3005; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3006; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3007; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3008; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
3009; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3010; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3011; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3012; GFX90A-TGSPLIT-NEXT:    buffer_invl2
3013; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3014; GFX90A-TGSPLIT-NEXT:    s_endpgm
3015;
3016; GFX940-NOTTGSPLIT-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
3017; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3018; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3019; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3020; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3021; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3022; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3023; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
3024; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3025; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
3026; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3027; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
3028; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3029;
3030; GFX940-TGSPLIT-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
3031; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3032; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3033; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3034; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3035; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3036; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3037; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
3038; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3039; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
3040; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3041; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
3042; GFX940-TGSPLIT-NEXT:    s_endpgm
3043;
3044; GFX11-WGP-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
3045; GFX11-WGP:       ; %bb.0: ; %entry
3046; GFX11-WGP-NEXT:    s_clause 0x1
3047; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3048; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3049; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3050; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3051; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3052; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3053; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3054; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3055; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3056; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3057; GFX11-WGP-NEXT:    buffer_gl0_inv
3058; GFX11-WGP-NEXT:    buffer_gl1_inv
3059; GFX11-WGP-NEXT:    s_endpgm
3060;
3061; GFX11-CU-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
3062; GFX11-CU:       ; %bb.0: ; %entry
3063; GFX11-CU-NEXT:    s_clause 0x1
3064; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3065; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3066; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3067; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3068; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3069; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3070; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3071; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3072; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3073; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3074; GFX11-CU-NEXT:    buffer_gl0_inv
3075; GFX11-CU-NEXT:    buffer_gl1_inv
3076; GFX11-CU-NEXT:    s_endpgm
3077    i32* %out, i32 %in, i32 %old) {
3078entry:
3079  %gep = getelementptr i32, i32* %out, i32 4
3080  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst monotonic
3081  ret void
3082}
3083
3084define amdgpu_kernel void @flat_system_monotonic_acquire_cmpxchg(
3085; GFX7-LABEL: flat_system_monotonic_acquire_cmpxchg:
3086; GFX7:       ; %bb.0: ; %entry
3087; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3088; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3089; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3090; GFX7-NEXT:    s_add_u32 s0, s0, 16
3091; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3092; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3093; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3094; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3095; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3096; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3097; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3098; GFX7-NEXT:    buffer_wbinvl1_vol
3099; GFX7-NEXT:    s_endpgm
3100;
3101; GFX10-WGP-LABEL: flat_system_monotonic_acquire_cmpxchg:
3102; GFX10-WGP:       ; %bb.0: ; %entry
3103; GFX10-WGP-NEXT:    s_clause 0x1
3104; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3105; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3106; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3107; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3108; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3109; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3110; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3111; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3112; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3113; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3114; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3115; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3116; GFX10-WGP-NEXT:    buffer_gl0_inv
3117; GFX10-WGP-NEXT:    buffer_gl1_inv
3118; GFX10-WGP-NEXT:    s_endpgm
3119;
3120; GFX10-CU-LABEL: flat_system_monotonic_acquire_cmpxchg:
3121; GFX10-CU:       ; %bb.0: ; %entry
3122; GFX10-CU-NEXT:    s_clause 0x1
3123; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3124; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3125; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3126; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3127; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3128; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3129; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3130; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3131; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3132; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3133; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3134; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3135; GFX10-CU-NEXT:    buffer_gl0_inv
3136; GFX10-CU-NEXT:    buffer_gl1_inv
3137; GFX10-CU-NEXT:    s_endpgm
3138;
3139; SKIP-CACHE-INV-LABEL: flat_system_monotonic_acquire_cmpxchg:
3140; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3141; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3142; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3143; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3144; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3145; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3146; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3147; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3148; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3149; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3150; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3151; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3152; SKIP-CACHE-INV-NEXT:    s_endpgm
3153;
3154; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_acquire_cmpxchg:
3155; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3156; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3157; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3158; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3159; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3160; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3161; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3162; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3163; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
3164; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3165; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3166;
3167; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_acquire_cmpxchg:
3168; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3169; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3170; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3171; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3172; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3173; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3174; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3175; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3176; GFX90A-TGSPLIT-NEXT:    buffer_invl2
3177; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3178; GFX90A-TGSPLIT-NEXT:    s_endpgm
3179;
3180; GFX940-NOTTGSPLIT-LABEL: flat_system_monotonic_acquire_cmpxchg:
3181; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3182; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3183; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3184; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3185; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3186; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3187; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
3188; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3189; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
3190; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3191;
3192; GFX940-TGSPLIT-LABEL: flat_system_monotonic_acquire_cmpxchg:
3193; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3194; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3195; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3196; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3197; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3198; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3199; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
3200; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3201; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
3202; GFX940-TGSPLIT-NEXT:    s_endpgm
3203;
3204; GFX11-WGP-LABEL: flat_system_monotonic_acquire_cmpxchg:
3205; GFX11-WGP:       ; %bb.0: ; %entry
3206; GFX11-WGP-NEXT:    s_clause 0x1
3207; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3208; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3209; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3210; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3211; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3212; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3213; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3214; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3215; GFX11-WGP-NEXT:    buffer_gl0_inv
3216; GFX11-WGP-NEXT:    buffer_gl1_inv
3217; GFX11-WGP-NEXT:    s_endpgm
3218;
3219; GFX11-CU-LABEL: flat_system_monotonic_acquire_cmpxchg:
3220; GFX11-CU:       ; %bb.0: ; %entry
3221; GFX11-CU-NEXT:    s_clause 0x1
3222; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3223; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3224; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3225; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3226; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3227; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3228; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3229; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3230; GFX11-CU-NEXT:    buffer_gl0_inv
3231; GFX11-CU-NEXT:    buffer_gl1_inv
3232; GFX11-CU-NEXT:    s_endpgm
3233    i32* %out, i32 %in, i32 %old) {
3234entry:
3235  %gep = getelementptr i32, i32* %out, i32 4
3236  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in monotonic acquire
3237  ret void
3238}
3239
3240define amdgpu_kernel void @flat_system_acquire_acquire_cmpxchg(
3241; GFX7-LABEL: flat_system_acquire_acquire_cmpxchg:
3242; GFX7:       ; %bb.0: ; %entry
3243; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3244; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3245; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3246; GFX7-NEXT:    s_add_u32 s0, s0, 16
3247; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3248; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3249; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3250; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3251; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3252; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3253; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3254; GFX7-NEXT:    buffer_wbinvl1_vol
3255; GFX7-NEXT:    s_endpgm
3256;
3257; GFX10-WGP-LABEL: flat_system_acquire_acquire_cmpxchg:
3258; GFX10-WGP:       ; %bb.0: ; %entry
3259; GFX10-WGP-NEXT:    s_clause 0x1
3260; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3261; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3262; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3263; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3264; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3265; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3266; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3267; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3268; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3269; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3270; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3271; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3272; GFX10-WGP-NEXT:    buffer_gl0_inv
3273; GFX10-WGP-NEXT:    buffer_gl1_inv
3274; GFX10-WGP-NEXT:    s_endpgm
3275;
3276; GFX10-CU-LABEL: flat_system_acquire_acquire_cmpxchg:
3277; GFX10-CU:       ; %bb.0: ; %entry
3278; GFX10-CU-NEXT:    s_clause 0x1
3279; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3280; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3281; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3282; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3283; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3284; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3285; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3286; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3287; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3288; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3289; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3290; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3291; GFX10-CU-NEXT:    buffer_gl0_inv
3292; GFX10-CU-NEXT:    buffer_gl1_inv
3293; GFX10-CU-NEXT:    s_endpgm
3294;
3295; SKIP-CACHE-INV-LABEL: flat_system_acquire_acquire_cmpxchg:
3296; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3297; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3298; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3299; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3300; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3301; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3302; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3303; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3304; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3305; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3306; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3307; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3308; SKIP-CACHE-INV-NEXT:    s_endpgm
3309;
3310; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_acquire_cmpxchg:
3311; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3312; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3313; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3314; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3315; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3316; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3317; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3318; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3319; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
3320; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3321; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3322;
3323; GFX90A-TGSPLIT-LABEL: flat_system_acquire_acquire_cmpxchg:
3324; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3325; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3326; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3327; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3328; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3329; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3330; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3331; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3332; GFX90A-TGSPLIT-NEXT:    buffer_invl2
3333; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3334; GFX90A-TGSPLIT-NEXT:    s_endpgm
3335;
3336; GFX940-NOTTGSPLIT-LABEL: flat_system_acquire_acquire_cmpxchg:
3337; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3338; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3339; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3340; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3341; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3342; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3343; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
3344; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3345; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
3346; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3347;
3348; GFX940-TGSPLIT-LABEL: flat_system_acquire_acquire_cmpxchg:
3349; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3350; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3351; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3352; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3353; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3354; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3355; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
3356; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3357; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
3358; GFX940-TGSPLIT-NEXT:    s_endpgm
3359;
3360; GFX11-WGP-LABEL: flat_system_acquire_acquire_cmpxchg:
3361; GFX11-WGP:       ; %bb.0: ; %entry
3362; GFX11-WGP-NEXT:    s_clause 0x1
3363; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3364; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3365; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3366; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3367; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3368; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3369; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3370; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3371; GFX11-WGP-NEXT:    buffer_gl0_inv
3372; GFX11-WGP-NEXT:    buffer_gl1_inv
3373; GFX11-WGP-NEXT:    s_endpgm
3374;
3375; GFX11-CU-LABEL: flat_system_acquire_acquire_cmpxchg:
3376; GFX11-CU:       ; %bb.0: ; %entry
3377; GFX11-CU-NEXT:    s_clause 0x1
3378; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3379; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3380; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3381; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3382; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3383; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3384; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3385; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3386; GFX11-CU-NEXT:    buffer_gl0_inv
3387; GFX11-CU-NEXT:    buffer_gl1_inv
3388; GFX11-CU-NEXT:    s_endpgm
3389    i32* %out, i32 %in, i32 %old) {
3390entry:
3391  %gep = getelementptr i32, i32* %out, i32 4
3392  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acquire acquire
3393  ret void
3394}
3395
3396define amdgpu_kernel void @flat_system_release_acquire_cmpxchg(
3397; GFX7-LABEL: flat_system_release_acquire_cmpxchg:
3398; GFX7:       ; %bb.0: ; %entry
3399; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3400; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3401; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3402; GFX7-NEXT:    s_add_u32 s0, s0, 16
3403; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3404; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3405; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3406; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3407; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3408; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3409; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3410; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3411; GFX7-NEXT:    buffer_wbinvl1_vol
3412; GFX7-NEXT:    s_endpgm
3413;
3414; GFX10-WGP-LABEL: flat_system_release_acquire_cmpxchg:
3415; GFX10-WGP:       ; %bb.0: ; %entry
3416; GFX10-WGP-NEXT:    s_clause 0x1
3417; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3418; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3419; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3420; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3421; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3422; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3423; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3424; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3425; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3426; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3427; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3428; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3429; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3430; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3431; GFX10-WGP-NEXT:    buffer_gl0_inv
3432; GFX10-WGP-NEXT:    buffer_gl1_inv
3433; GFX10-WGP-NEXT:    s_endpgm
3434;
3435; GFX10-CU-LABEL: flat_system_release_acquire_cmpxchg:
3436; GFX10-CU:       ; %bb.0: ; %entry
3437; GFX10-CU-NEXT:    s_clause 0x1
3438; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3439; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3440; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3441; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3442; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3443; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3444; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3445; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3446; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3447; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3448; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3449; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3450; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3451; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3452; GFX10-CU-NEXT:    buffer_gl0_inv
3453; GFX10-CU-NEXT:    buffer_gl1_inv
3454; GFX10-CU-NEXT:    s_endpgm
3455;
3456; SKIP-CACHE-INV-LABEL: flat_system_release_acquire_cmpxchg:
3457; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3458; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3459; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3460; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3461; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3462; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3463; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3464; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3465; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3466; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3467; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3468; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3469; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3470; SKIP-CACHE-INV-NEXT:    s_endpgm
3471;
3472; GFX90A-NOTTGSPLIT-LABEL: flat_system_release_acquire_cmpxchg:
3473; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3474; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3475; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3476; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3477; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3478; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3479; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
3480; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3481; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3482; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3483; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
3484; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3485; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3486;
3487; GFX90A-TGSPLIT-LABEL: flat_system_release_acquire_cmpxchg:
3488; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3489; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3490; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3491; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3492; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3493; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3494; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
3495; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3496; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3497; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3498; GFX90A-TGSPLIT-NEXT:    buffer_invl2
3499; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3500; GFX90A-TGSPLIT-NEXT:    s_endpgm
3501;
3502; GFX940-NOTTGSPLIT-LABEL: flat_system_release_acquire_cmpxchg:
3503; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3504; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3505; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3506; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3507; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3508; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3509; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
3510; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3511; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
3512; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3513; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
3514; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3515;
3516; GFX940-TGSPLIT-LABEL: flat_system_release_acquire_cmpxchg:
3517; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3518; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3519; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3520; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3521; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3522; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3523; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
3524; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3525; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
3526; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3527; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
3528; GFX940-TGSPLIT-NEXT:    s_endpgm
3529;
3530; GFX11-WGP-LABEL: flat_system_release_acquire_cmpxchg:
3531; GFX11-WGP:       ; %bb.0: ; %entry
3532; GFX11-WGP-NEXT:    s_clause 0x1
3533; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3534; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3535; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3536; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3537; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3538; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3539; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3540; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3541; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3542; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3543; GFX11-WGP-NEXT:    buffer_gl0_inv
3544; GFX11-WGP-NEXT:    buffer_gl1_inv
3545; GFX11-WGP-NEXT:    s_endpgm
3546;
3547; GFX11-CU-LABEL: flat_system_release_acquire_cmpxchg:
3548; GFX11-CU:       ; %bb.0: ; %entry
3549; GFX11-CU-NEXT:    s_clause 0x1
3550; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3551; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3552; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3553; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3554; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3555; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3556; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3557; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3558; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3559; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3560; GFX11-CU-NEXT:    buffer_gl0_inv
3561; GFX11-CU-NEXT:    buffer_gl1_inv
3562; GFX11-CU-NEXT:    s_endpgm
3563    i32* %out, i32 %in, i32 %old) {
3564entry:
3565  %gep = getelementptr i32, i32* %out, i32 4
3566  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in release acquire
3567  ret void
3568}
3569
3570define amdgpu_kernel void @flat_system_acq_rel_acquire_cmpxchg(
3571; GFX7-LABEL: flat_system_acq_rel_acquire_cmpxchg:
3572; GFX7:       ; %bb.0: ; %entry
3573; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3574; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3575; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3576; GFX7-NEXT:    s_add_u32 s0, s0, 16
3577; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3578; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3579; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3580; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3581; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3582; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3583; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3584; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3585; GFX7-NEXT:    buffer_wbinvl1_vol
3586; GFX7-NEXT:    s_endpgm
3587;
3588; GFX10-WGP-LABEL: flat_system_acq_rel_acquire_cmpxchg:
3589; GFX10-WGP:       ; %bb.0: ; %entry
3590; GFX10-WGP-NEXT:    s_clause 0x1
3591; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3592; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3593; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3594; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3595; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3596; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3597; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3598; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3599; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3600; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3601; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3602; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3603; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3604; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3605; GFX10-WGP-NEXT:    buffer_gl0_inv
3606; GFX10-WGP-NEXT:    buffer_gl1_inv
3607; GFX10-WGP-NEXT:    s_endpgm
3608;
3609; GFX10-CU-LABEL: flat_system_acq_rel_acquire_cmpxchg:
3610; GFX10-CU:       ; %bb.0: ; %entry
3611; GFX10-CU-NEXT:    s_clause 0x1
3612; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3613; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3614; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3615; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3616; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3617; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3618; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3619; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3620; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3621; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3622; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3623; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3624; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3625; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3626; GFX10-CU-NEXT:    buffer_gl0_inv
3627; GFX10-CU-NEXT:    buffer_gl1_inv
3628; GFX10-CU-NEXT:    s_endpgm
3629;
3630; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_acquire_cmpxchg:
3631; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3632; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3633; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3634; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3635; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3636; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3637; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3638; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3639; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3640; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3641; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3642; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3643; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3644; SKIP-CACHE-INV-NEXT:    s_endpgm
3645;
3646; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_acquire_cmpxchg:
3647; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3648; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3649; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3650; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3651; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3652; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3653; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
3654; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3655; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3656; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3657; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
3658; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3659; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3660;
3661; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_acquire_cmpxchg:
3662; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3663; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3664; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3665; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3666; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3667; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3668; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
3669; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3670; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3671; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3672; GFX90A-TGSPLIT-NEXT:    buffer_invl2
3673; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3674; GFX90A-TGSPLIT-NEXT:    s_endpgm
3675;
3676; GFX940-NOTTGSPLIT-LABEL: flat_system_acq_rel_acquire_cmpxchg:
3677; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3678; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3679; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3680; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3681; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3682; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3683; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
3684; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3685; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
3686; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3687; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
3688; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3689;
3690; GFX940-TGSPLIT-LABEL: flat_system_acq_rel_acquire_cmpxchg:
3691; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3692; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3693; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3694; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3695; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3696; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3697; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
3698; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3699; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
3700; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3701; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
3702; GFX940-TGSPLIT-NEXT:    s_endpgm
3703;
3704; GFX11-WGP-LABEL: flat_system_acq_rel_acquire_cmpxchg:
3705; GFX11-WGP:       ; %bb.0: ; %entry
3706; GFX11-WGP-NEXT:    s_clause 0x1
3707; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3708; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3709; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3710; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3711; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3712; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3713; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3714; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3715; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3716; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3717; GFX11-WGP-NEXT:    buffer_gl0_inv
3718; GFX11-WGP-NEXT:    buffer_gl1_inv
3719; GFX11-WGP-NEXT:    s_endpgm
3720;
3721; GFX11-CU-LABEL: flat_system_acq_rel_acquire_cmpxchg:
3722; GFX11-CU:       ; %bb.0: ; %entry
3723; GFX11-CU-NEXT:    s_clause 0x1
3724; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3725; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3726; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3727; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3728; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3729; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3730; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3731; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3732; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3733; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3734; GFX11-CU-NEXT:    buffer_gl0_inv
3735; GFX11-CU-NEXT:    buffer_gl1_inv
3736; GFX11-CU-NEXT:    s_endpgm
3737    i32* %out, i32 %in, i32 %old) {
3738entry:
3739  %gep = getelementptr i32, i32* %out, i32 4
3740  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acq_rel acquire
3741  ret void
3742}
3743
3744define amdgpu_kernel void @flat_system_seq_cst_acquire_cmpxchg(
3745; GFX7-LABEL: flat_system_seq_cst_acquire_cmpxchg:
3746; GFX7:       ; %bb.0: ; %entry
3747; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3748; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3749; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3750; GFX7-NEXT:    s_add_u32 s0, s0, 16
3751; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3752; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3753; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3754; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3755; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3756; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3757; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3758; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3759; GFX7-NEXT:    buffer_wbinvl1_vol
3760; GFX7-NEXT:    s_endpgm
3761;
3762; GFX10-WGP-LABEL: flat_system_seq_cst_acquire_cmpxchg:
3763; GFX10-WGP:       ; %bb.0: ; %entry
3764; GFX10-WGP-NEXT:    s_clause 0x1
3765; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3766; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3767; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3768; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3769; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3770; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3771; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3772; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3773; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3774; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3775; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3776; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3777; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3778; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3779; GFX10-WGP-NEXT:    buffer_gl0_inv
3780; GFX10-WGP-NEXT:    buffer_gl1_inv
3781; GFX10-WGP-NEXT:    s_endpgm
3782;
3783; GFX10-CU-LABEL: flat_system_seq_cst_acquire_cmpxchg:
3784; GFX10-CU:       ; %bb.0: ; %entry
3785; GFX10-CU-NEXT:    s_clause 0x1
3786; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3787; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3788; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3789; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3790; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3791; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3792; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3793; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3794; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3795; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3796; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3797; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3798; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3799; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3800; GFX10-CU-NEXT:    buffer_gl0_inv
3801; GFX10-CU-NEXT:    buffer_gl1_inv
3802; GFX10-CU-NEXT:    s_endpgm
3803;
3804; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_acquire_cmpxchg:
3805; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3806; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3807; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3808; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3809; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3810; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3811; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3812; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3813; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3814; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3815; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3816; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3817; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3818; SKIP-CACHE-INV-NEXT:    s_endpgm
3819;
3820; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_acquire_cmpxchg:
3821; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3822; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3823; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3824; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3825; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3826; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3827; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
3828; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3829; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3830; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3831; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
3832; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3833; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3834;
3835; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_acquire_cmpxchg:
3836; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3837; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3838; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3839; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3840; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3841; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3842; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
3843; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3844; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3845; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3846; GFX90A-TGSPLIT-NEXT:    buffer_invl2
3847; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3848; GFX90A-TGSPLIT-NEXT:    s_endpgm
3849;
3850; GFX940-NOTTGSPLIT-LABEL: flat_system_seq_cst_acquire_cmpxchg:
3851; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3852; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3853; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3854; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3855; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3856; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3857; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
3858; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3859; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
3860; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3861; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
3862; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3863;
3864; GFX940-TGSPLIT-LABEL: flat_system_seq_cst_acquire_cmpxchg:
3865; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3866; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3867; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3868; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3869; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3870; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3871; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
3872; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3873; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
3874; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3875; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
3876; GFX940-TGSPLIT-NEXT:    s_endpgm
3877;
3878; GFX11-WGP-LABEL: flat_system_seq_cst_acquire_cmpxchg:
3879; GFX11-WGP:       ; %bb.0: ; %entry
3880; GFX11-WGP-NEXT:    s_clause 0x1
3881; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3882; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3883; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3884; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3885; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3886; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3887; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3888; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3889; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3890; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3891; GFX11-WGP-NEXT:    buffer_gl0_inv
3892; GFX11-WGP-NEXT:    buffer_gl1_inv
3893; GFX11-WGP-NEXT:    s_endpgm
3894;
3895; GFX11-CU-LABEL: flat_system_seq_cst_acquire_cmpxchg:
3896; GFX11-CU:       ; %bb.0: ; %entry
3897; GFX11-CU-NEXT:    s_clause 0x1
3898; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3899; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3900; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3901; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3902; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3903; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3904; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3905; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3906; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3907; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3908; GFX11-CU-NEXT:    buffer_gl0_inv
3909; GFX11-CU-NEXT:    buffer_gl1_inv
3910; GFX11-CU-NEXT:    s_endpgm
3911    i32* %out, i32 %in, i32 %old) {
3912entry:
3913  %gep = getelementptr i32, i32* %out, i32 4
3914  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst acquire
3915  ret void
3916}
3917
3918define amdgpu_kernel void @flat_system_monotonic_seq_cst_cmpxchg(
3919; GFX7-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
3920; GFX7:       ; %bb.0: ; %entry
3921; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3922; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3923; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3924; GFX7-NEXT:    s_add_u32 s0, s0, 16
3925; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3926; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3927; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3928; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3929; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3930; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3931; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3932; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3933; GFX7-NEXT:    buffer_wbinvl1_vol
3934; GFX7-NEXT:    s_endpgm
3935;
3936; GFX10-WGP-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
3937; GFX10-WGP:       ; %bb.0: ; %entry
3938; GFX10-WGP-NEXT:    s_clause 0x1
3939; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3940; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3941; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3942; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3943; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3944; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3945; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3946; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3947; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3948; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3949; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3950; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3951; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3952; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3953; GFX10-WGP-NEXT:    buffer_gl0_inv
3954; GFX10-WGP-NEXT:    buffer_gl1_inv
3955; GFX10-WGP-NEXT:    s_endpgm
3956;
3957; GFX10-CU-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
3958; GFX10-CU:       ; %bb.0: ; %entry
3959; GFX10-CU-NEXT:    s_clause 0x1
3960; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3961; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3962; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3963; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3964; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3965; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3966; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3967; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3968; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3969; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3970; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3971; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3972; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3973; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3974; GFX10-CU-NEXT:    buffer_gl0_inv
3975; GFX10-CU-NEXT:    buffer_gl1_inv
3976; GFX10-CU-NEXT:    s_endpgm
3977;
3978; SKIP-CACHE-INV-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
3979; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3980; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3981; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3982; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3983; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3984; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3985; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3986; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3987; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3988; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3989; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3990; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3991; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3992; SKIP-CACHE-INV-NEXT:    s_endpgm
3993;
3994; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
3995; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3996; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3997; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3998; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3999; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4000; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4001; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
4002; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4003; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4004; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4005; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
4006; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4007; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4008;
4009; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
4010; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4011; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4012; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4013; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4014; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4015; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4016; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
4017; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4018; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4019; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4020; GFX90A-TGSPLIT-NEXT:    buffer_invl2
4021; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4022; GFX90A-TGSPLIT-NEXT:    s_endpgm
4023;
4024; GFX940-NOTTGSPLIT-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
4025; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4026; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4027; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4028; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4029; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4030; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4031; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
4032; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4033; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
4034; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4035; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
4036; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4037;
4038; GFX940-TGSPLIT-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
4039; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4040; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4041; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4042; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4043; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4044; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4045; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
4046; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4047; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
4048; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4049; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
4050; GFX940-TGSPLIT-NEXT:    s_endpgm
4051;
4052; GFX11-WGP-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
4053; GFX11-WGP:       ; %bb.0: ; %entry
4054; GFX11-WGP-NEXT:    s_clause 0x1
4055; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4056; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4057; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4058; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4059; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4060; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4061; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4062; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4063; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4064; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4065; GFX11-WGP-NEXT:    buffer_gl0_inv
4066; GFX11-WGP-NEXT:    buffer_gl1_inv
4067; GFX11-WGP-NEXT:    s_endpgm
4068;
4069; GFX11-CU-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
4070; GFX11-CU:       ; %bb.0: ; %entry
4071; GFX11-CU-NEXT:    s_clause 0x1
4072; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4073; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4074; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4075; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4076; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4077; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4078; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4079; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4080; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4081; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4082; GFX11-CU-NEXT:    buffer_gl0_inv
4083; GFX11-CU-NEXT:    buffer_gl1_inv
4084; GFX11-CU-NEXT:    s_endpgm
4085    i32* %out, i32 %in, i32 %old) {
4086entry:
4087  %gep = getelementptr i32, i32* %out, i32 4
4088  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in monotonic seq_cst
4089  ret void
4090}
4091
4092define amdgpu_kernel void @flat_system_acquire_seq_cst_cmpxchg(
4093; GFX7-LABEL: flat_system_acquire_seq_cst_cmpxchg:
4094; GFX7:       ; %bb.0: ; %entry
4095; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4096; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4097; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4098; GFX7-NEXT:    s_add_u32 s0, s0, 16
4099; GFX7-NEXT:    s_addc_u32 s1, s1, 0
4100; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4101; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4102; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4103; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4104; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4105; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4106; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4107; GFX7-NEXT:    buffer_wbinvl1_vol
4108; GFX7-NEXT:    s_endpgm
4109;
4110; GFX10-WGP-LABEL: flat_system_acquire_seq_cst_cmpxchg:
4111; GFX10-WGP:       ; %bb.0: ; %entry
4112; GFX10-WGP-NEXT:    s_clause 0x1
4113; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4114; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4115; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4116; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
4117; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
4118; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4119; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4120; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4121; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4122; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4123; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4124; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4125; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4126; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4127; GFX10-WGP-NEXT:    buffer_gl0_inv
4128; GFX10-WGP-NEXT:    buffer_gl1_inv
4129; GFX10-WGP-NEXT:    s_endpgm
4130;
4131; GFX10-CU-LABEL: flat_system_acquire_seq_cst_cmpxchg:
4132; GFX10-CU:       ; %bb.0: ; %entry
4133; GFX10-CU-NEXT:    s_clause 0x1
4134; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4135; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4136; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4137; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
4138; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
4139; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4140; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4141; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4142; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4143; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4144; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4145; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4146; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4147; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4148; GFX10-CU-NEXT:    buffer_gl0_inv
4149; GFX10-CU-NEXT:    buffer_gl1_inv
4150; GFX10-CU-NEXT:    s_endpgm
4151;
4152; SKIP-CACHE-INV-LABEL: flat_system_acquire_seq_cst_cmpxchg:
4153; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4154; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4155; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4156; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4157; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
4158; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
4159; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4160; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4161; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4162; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4163; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4164; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4165; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4166; SKIP-CACHE-INV-NEXT:    s_endpgm
4167;
4168; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_seq_cst_cmpxchg:
4169; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4170; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4171; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4172; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4173; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4174; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4175; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
4176; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4177; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4178; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4179; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
4180; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4181; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4182;
4183; GFX90A-TGSPLIT-LABEL: flat_system_acquire_seq_cst_cmpxchg:
4184; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4185; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4186; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4187; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4188; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4189; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4190; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
4191; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4192; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4193; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4194; GFX90A-TGSPLIT-NEXT:    buffer_invl2
4195; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4196; GFX90A-TGSPLIT-NEXT:    s_endpgm
4197;
4198; GFX940-NOTTGSPLIT-LABEL: flat_system_acquire_seq_cst_cmpxchg:
4199; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4200; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4201; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4202; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4203; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4204; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4205; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
4206; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4207; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
4208; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4209; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
4210; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4211;
4212; GFX940-TGSPLIT-LABEL: flat_system_acquire_seq_cst_cmpxchg:
4213; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4214; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4215; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4216; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4217; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4218; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4219; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
4220; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4221; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
4222; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4223; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
4224; GFX940-TGSPLIT-NEXT:    s_endpgm
4225;
4226; GFX11-WGP-LABEL: flat_system_acquire_seq_cst_cmpxchg:
4227; GFX11-WGP:       ; %bb.0: ; %entry
4228; GFX11-WGP-NEXT:    s_clause 0x1
4229; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4230; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4231; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4232; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4233; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4234; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4235; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4236; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4237; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4238; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4239; GFX11-WGP-NEXT:    buffer_gl0_inv
4240; GFX11-WGP-NEXT:    buffer_gl1_inv
4241; GFX11-WGP-NEXT:    s_endpgm
4242;
4243; GFX11-CU-LABEL: flat_system_acquire_seq_cst_cmpxchg:
4244; GFX11-CU:       ; %bb.0: ; %entry
4245; GFX11-CU-NEXT:    s_clause 0x1
4246; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4247; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4248; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4249; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4250; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4251; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4252; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4253; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4254; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4255; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4256; GFX11-CU-NEXT:    buffer_gl0_inv
4257; GFX11-CU-NEXT:    buffer_gl1_inv
4258; GFX11-CU-NEXT:    s_endpgm
4259    i32* %out, i32 %in, i32 %old) {
4260entry:
4261  %gep = getelementptr i32, i32* %out, i32 4
4262  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acquire seq_cst
4263  ret void
4264}
4265
4266define amdgpu_kernel void @flat_system_release_seq_cst_cmpxchg(
4267; GFX7-LABEL: flat_system_release_seq_cst_cmpxchg:
4268; GFX7:       ; %bb.0: ; %entry
4269; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4270; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4271; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4272; GFX7-NEXT:    s_add_u32 s0, s0, 16
4273; GFX7-NEXT:    s_addc_u32 s1, s1, 0
4274; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4275; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4276; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4277; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4278; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4279; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4280; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4281; GFX7-NEXT:    buffer_wbinvl1_vol
4282; GFX7-NEXT:    s_endpgm
4283;
4284; GFX10-WGP-LABEL: flat_system_release_seq_cst_cmpxchg:
4285; GFX10-WGP:       ; %bb.0: ; %entry
4286; GFX10-WGP-NEXT:    s_clause 0x1
4287; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4288; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4289; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4290; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
4291; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
4292; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4293; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4294; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4295; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4296; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4297; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4298; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4299; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4300; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4301; GFX10-WGP-NEXT:    buffer_gl0_inv
4302; GFX10-WGP-NEXT:    buffer_gl1_inv
4303; GFX10-WGP-NEXT:    s_endpgm
4304;
4305; GFX10-CU-LABEL: flat_system_release_seq_cst_cmpxchg:
4306; GFX10-CU:       ; %bb.0: ; %entry
4307; GFX10-CU-NEXT:    s_clause 0x1
4308; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4309; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4310; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4311; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
4312; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
4313; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4314; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4315; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4316; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4317; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4318; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4319; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4320; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4321; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4322; GFX10-CU-NEXT:    buffer_gl0_inv
4323; GFX10-CU-NEXT:    buffer_gl1_inv
4324; GFX10-CU-NEXT:    s_endpgm
4325;
4326; SKIP-CACHE-INV-LABEL: flat_system_release_seq_cst_cmpxchg:
4327; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4328; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4329; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4330; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4331; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
4332; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
4333; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4334; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4335; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4336; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4337; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4338; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4339; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4340; SKIP-CACHE-INV-NEXT:    s_endpgm
4341;
4342; GFX90A-NOTTGSPLIT-LABEL: flat_system_release_seq_cst_cmpxchg:
4343; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4344; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4345; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4346; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4347; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4348; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4349; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
4350; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4351; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4352; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4353; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
4354; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4355; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4356;
4357; GFX90A-TGSPLIT-LABEL: flat_system_release_seq_cst_cmpxchg:
4358; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4359; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4360; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4361; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4362; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4363; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4364; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
4365; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4366; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4367; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4368; GFX90A-TGSPLIT-NEXT:    buffer_invl2
4369; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4370; GFX90A-TGSPLIT-NEXT:    s_endpgm
4371;
4372; GFX940-NOTTGSPLIT-LABEL: flat_system_release_seq_cst_cmpxchg:
4373; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4374; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4375; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4376; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4377; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4378; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4379; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
4380; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4381; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
4382; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4383; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
4384; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4385;
4386; GFX940-TGSPLIT-LABEL: flat_system_release_seq_cst_cmpxchg:
4387; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4388; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4389; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4390; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4391; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4392; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4393; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
4394; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4395; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
4396; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4397; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
4398; GFX940-TGSPLIT-NEXT:    s_endpgm
4399;
4400; GFX11-WGP-LABEL: flat_system_release_seq_cst_cmpxchg:
4401; GFX11-WGP:       ; %bb.0: ; %entry
4402; GFX11-WGP-NEXT:    s_clause 0x1
4403; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4404; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4405; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4406; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4407; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4408; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4409; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4410; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4411; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4412; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4413; GFX11-WGP-NEXT:    buffer_gl0_inv
4414; GFX11-WGP-NEXT:    buffer_gl1_inv
4415; GFX11-WGP-NEXT:    s_endpgm
4416;
4417; GFX11-CU-LABEL: flat_system_release_seq_cst_cmpxchg:
4418; GFX11-CU:       ; %bb.0: ; %entry
4419; GFX11-CU-NEXT:    s_clause 0x1
4420; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4421; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4422; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4423; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4424; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4425; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4426; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4427; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4428; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4429; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4430; GFX11-CU-NEXT:    buffer_gl0_inv
4431; GFX11-CU-NEXT:    buffer_gl1_inv
4432; GFX11-CU-NEXT:    s_endpgm
4433    i32* %out, i32 %in, i32 %old) {
4434entry:
4435  %gep = getelementptr i32, i32* %out, i32 4
4436  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in release seq_cst
4437  ret void
4438}
4439
4440define amdgpu_kernel void @flat_system_acq_rel_seq_cst_cmpxchg(
4441; GFX7-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
4442; GFX7:       ; %bb.0: ; %entry
4443; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4444; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4445; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4446; GFX7-NEXT:    s_add_u32 s0, s0, 16
4447; GFX7-NEXT:    s_addc_u32 s1, s1, 0
4448; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4449; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4450; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4451; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4452; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4453; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4454; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4455; GFX7-NEXT:    buffer_wbinvl1_vol
4456; GFX7-NEXT:    s_endpgm
4457;
4458; GFX10-WGP-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
4459; GFX10-WGP:       ; %bb.0: ; %entry
4460; GFX10-WGP-NEXT:    s_clause 0x1
4461; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4462; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4463; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4464; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
4465; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
4466; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4467; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4468; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4469; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4470; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4471; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4472; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4473; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4474; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4475; GFX10-WGP-NEXT:    buffer_gl0_inv
4476; GFX10-WGP-NEXT:    buffer_gl1_inv
4477; GFX10-WGP-NEXT:    s_endpgm
4478;
4479; GFX10-CU-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
4480; GFX10-CU:       ; %bb.0: ; %entry
4481; GFX10-CU-NEXT:    s_clause 0x1
4482; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4483; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4484; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4485; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
4486; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
4487; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4488; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4489; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4490; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4491; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4492; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4493; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4494; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4495; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4496; GFX10-CU-NEXT:    buffer_gl0_inv
4497; GFX10-CU-NEXT:    buffer_gl1_inv
4498; GFX10-CU-NEXT:    s_endpgm
4499;
4500; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
4501; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4502; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4503; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4504; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4505; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
4506; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
4507; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4508; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4509; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4510; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4511; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4512; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4513; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4514; SKIP-CACHE-INV-NEXT:    s_endpgm
4515;
4516; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
4517; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4518; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4519; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4520; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4521; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4522; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4523; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
4524; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4525; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4526; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4527; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
4528; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4529; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4530;
4531; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
4532; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4533; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4534; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4535; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4536; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4537; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4538; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
4539; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4540; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4541; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4542; GFX90A-TGSPLIT-NEXT:    buffer_invl2
4543; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4544; GFX90A-TGSPLIT-NEXT:    s_endpgm
4545;
4546; GFX940-NOTTGSPLIT-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
4547; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4548; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4549; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4550; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4551; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4552; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4553; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
4554; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4555; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
4556; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4557; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
4558; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4559;
4560; GFX940-TGSPLIT-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
4561; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4562; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4563; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4564; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4565; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4566; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4567; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
4568; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4569; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
4570; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4571; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
4572; GFX940-TGSPLIT-NEXT:    s_endpgm
4573;
4574; GFX11-WGP-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
4575; GFX11-WGP:       ; %bb.0: ; %entry
4576; GFX11-WGP-NEXT:    s_clause 0x1
4577; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4578; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4579; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4580; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4581; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4582; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4583; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4584; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4585; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4586; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4587; GFX11-WGP-NEXT:    buffer_gl0_inv
4588; GFX11-WGP-NEXT:    buffer_gl1_inv
4589; GFX11-WGP-NEXT:    s_endpgm
4590;
4591; GFX11-CU-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
4592; GFX11-CU:       ; %bb.0: ; %entry
4593; GFX11-CU-NEXT:    s_clause 0x1
4594; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4595; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4596; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4597; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4598; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4599; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4600; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4601; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4602; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4603; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4604; GFX11-CU-NEXT:    buffer_gl0_inv
4605; GFX11-CU-NEXT:    buffer_gl1_inv
4606; GFX11-CU-NEXT:    s_endpgm
4607    i32* %out, i32 %in, i32 %old) {
4608entry:
4609  %gep = getelementptr i32, i32* %out, i32 4
4610  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acq_rel seq_cst
4611  ret void
4612}
4613
4614define amdgpu_kernel void @flat_system_seq_cst_seq_cst_cmpxchg(
4615; GFX7-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
4616; GFX7:       ; %bb.0: ; %entry
4617; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4618; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4619; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4620; GFX7-NEXT:    s_add_u32 s0, s0, 16
4621; GFX7-NEXT:    s_addc_u32 s1, s1, 0
4622; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4623; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4624; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4625; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4626; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4627; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4628; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4629; GFX7-NEXT:    buffer_wbinvl1_vol
4630; GFX7-NEXT:    s_endpgm
4631;
4632; GFX10-WGP-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
4633; GFX10-WGP:       ; %bb.0: ; %entry
4634; GFX10-WGP-NEXT:    s_clause 0x1
4635; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4636; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4637; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4638; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
4639; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
4640; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4641; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4642; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4643; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4644; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4645; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4646; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4647; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4648; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4649; GFX10-WGP-NEXT:    buffer_gl0_inv
4650; GFX10-WGP-NEXT:    buffer_gl1_inv
4651; GFX10-WGP-NEXT:    s_endpgm
4652;
4653; GFX10-CU-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
4654; GFX10-CU:       ; %bb.0: ; %entry
4655; GFX10-CU-NEXT:    s_clause 0x1
4656; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4657; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4658; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4659; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
4660; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
4661; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4662; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4663; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4664; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4665; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4666; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4667; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4668; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4669; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4670; GFX10-CU-NEXT:    buffer_gl0_inv
4671; GFX10-CU-NEXT:    buffer_gl1_inv
4672; GFX10-CU-NEXT:    s_endpgm
4673;
4674; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
4675; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4676; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4677; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4678; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4679; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
4680; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
4681; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4682; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4683; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4684; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4685; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4686; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4687; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4688; SKIP-CACHE-INV-NEXT:    s_endpgm
4689;
4690; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
4691; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4692; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4693; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4694; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4695; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4696; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4697; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
4698; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4699; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4700; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4701; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
4702; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4703; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4704;
4705; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
4706; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4707; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4708; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4709; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4710; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4711; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4712; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
4713; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4714; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4715; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4716; GFX90A-TGSPLIT-NEXT:    buffer_invl2
4717; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4718; GFX90A-TGSPLIT-NEXT:    s_endpgm
4719;
4720; GFX940-NOTTGSPLIT-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
4721; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4722; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4723; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4724; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4725; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4726; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4727; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
4728; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4729; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
4730; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4731; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
4732; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4733;
4734; GFX940-TGSPLIT-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
4735; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4736; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4737; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4738; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4739; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4740; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4741; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
4742; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4743; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
4744; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4745; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
4746; GFX940-TGSPLIT-NEXT:    s_endpgm
4747;
4748; GFX11-WGP-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
4749; GFX11-WGP:       ; %bb.0: ; %entry
4750; GFX11-WGP-NEXT:    s_clause 0x1
4751; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4752; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4753; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4754; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4755; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4756; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4757; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4758; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4759; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4760; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4761; GFX11-WGP-NEXT:    buffer_gl0_inv
4762; GFX11-WGP-NEXT:    buffer_gl1_inv
4763; GFX11-WGP-NEXT:    s_endpgm
4764;
4765; GFX11-CU-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
4766; GFX11-CU:       ; %bb.0: ; %entry
4767; GFX11-CU-NEXT:    s_clause 0x1
4768; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4769; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4770; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4771; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4772; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4773; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4774; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4775; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4776; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4777; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4778; GFX11-CU-NEXT:    buffer_gl0_inv
4779; GFX11-CU-NEXT:    buffer_gl1_inv
4780; GFX11-CU-NEXT:    s_endpgm
4781    i32* %out, i32 %in, i32 %old) {
4782entry:
4783  %gep = getelementptr i32, i32* %out, i32 4
4784  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst seq_cst
4785  ret void
4786}
4787
4788define amdgpu_kernel void @flat_system_monotonic_monotonic_ret_cmpxchg(
4789; GFX7-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
4790; GFX7:       ; %bb.0: ; %entry
4791; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4792; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4793; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4794; GFX7-NEXT:    s_add_u32 s4, s0, 16
4795; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4796; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4797; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4798; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4799; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4800; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4801; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4802; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4803; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4804; GFX7-NEXT:    flat_store_dword v[0:1], v2
4805; GFX7-NEXT:    s_endpgm
4806;
4807; GFX10-WGP-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
4808; GFX10-WGP:       ; %bb.0: ; %entry
4809; GFX10-WGP-NEXT:    s_clause 0x1
4810; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4811; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4812; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4813; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4814; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4815; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4816; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4817; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4818; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4819; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4820; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4821; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4822; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4823; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4824; GFX10-WGP-NEXT:    s_endpgm
4825;
4826; GFX10-CU-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
4827; GFX10-CU:       ; %bb.0: ; %entry
4828; GFX10-CU-NEXT:    s_clause 0x1
4829; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4830; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4831; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4832; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4833; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4834; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4835; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4836; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4837; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4838; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4839; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4840; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4841; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4842; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4843; GFX10-CU-NEXT:    s_endpgm
4844;
4845; SKIP-CACHE-INV-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
4846; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4847; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4848; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4849; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4850; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4851; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4852; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4853; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4854; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4855; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4856; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4857; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4858; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4859; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4860; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4861; SKIP-CACHE-INV-NEXT:    s_endpgm
4862;
4863; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
4864; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4865; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4866; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4867; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4868; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4869; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4870; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4871; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4872; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4873; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4874;
4875; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
4876; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4877; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4878; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4879; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4880; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4881; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4882; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4883; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4884; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4885; GFX90A-TGSPLIT-NEXT:    s_endpgm
4886;
4887; GFX940-NOTTGSPLIT-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
4888; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4889; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4890; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4891; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4892; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4893; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4894; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
4895; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4896; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4897; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4898;
4899; GFX940-TGSPLIT-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
4900; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4901; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4902; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4903; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4904; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4905; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4906; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
4907; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4908; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4909; GFX940-TGSPLIT-NEXT:    s_endpgm
4910;
4911; GFX11-WGP-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
4912; GFX11-WGP:       ; %bb.0: ; %entry
4913; GFX11-WGP-NEXT:    s_clause 0x1
4914; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4915; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4916; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4917; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4918; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4919; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4920; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4921; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
4922; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4923; GFX11-WGP-NEXT:    s_endpgm
4924;
4925; GFX11-CU-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
4926; GFX11-CU:       ; %bb.0: ; %entry
4927; GFX11-CU-NEXT:    s_clause 0x1
4928; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4929; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4930; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4931; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4932; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4933; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4934; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4935; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
4936; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4937; GFX11-CU-NEXT:    s_endpgm
4938    i32* %out, i32 %in, i32 %old) {
4939entry:
4940  %gep = getelementptr i32, i32* %out, i32 4
4941  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in monotonic monotonic
4942  %val0 = extractvalue { i32, i1 } %val, 0
4943  store i32 %val0, i32* %out, align 4
4944  ret void
4945}
4946
4947define amdgpu_kernel void @flat_system_acquire_monotonic_ret_cmpxchg(
4948; GFX7-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
4949; GFX7:       ; %bb.0: ; %entry
4950; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4951; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4952; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4953; GFX7-NEXT:    s_add_u32 s4, s0, 16
4954; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4955; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4956; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4957; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4958; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4959; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4960; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4961; GFX7-NEXT:    buffer_wbinvl1_vol
4962; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4963; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4964; GFX7-NEXT:    flat_store_dword v[0:1], v2
4965; GFX7-NEXT:    s_endpgm
4966;
4967; GFX10-WGP-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
4968; GFX10-WGP:       ; %bb.0: ; %entry
4969; GFX10-WGP-NEXT:    s_clause 0x1
4970; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4971; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4972; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4973; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4974; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4975; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4976; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4977; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4978; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4979; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4980; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4981; GFX10-WGP-NEXT:    buffer_gl0_inv
4982; GFX10-WGP-NEXT:    buffer_gl1_inv
4983; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4984; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4985; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4986; GFX10-WGP-NEXT:    s_endpgm
4987;
4988; GFX10-CU-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
4989; GFX10-CU:       ; %bb.0: ; %entry
4990; GFX10-CU-NEXT:    s_clause 0x1
4991; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4992; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4993; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4994; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4995; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4996; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4997; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4998; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4999; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5000; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5001; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5002; GFX10-CU-NEXT:    buffer_gl0_inv
5003; GFX10-CU-NEXT:    buffer_gl1_inv
5004; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5005; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5006; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5007; GFX10-CU-NEXT:    s_endpgm
5008;
5009; SKIP-CACHE-INV-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
5010; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5011; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5012; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5013; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5014; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5015; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5016; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5017; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5018; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5019; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5020; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5021; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5022; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5023; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5024; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5025; SKIP-CACHE-INV-NEXT:    s_endpgm
5026;
5027; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
5028; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5029; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5030; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5031; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5032; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5033; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5034; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5035; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5036; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
5037; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5038; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5039; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5040;
5041; GFX90A-TGSPLIT-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
5042; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5043; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5044; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5045; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5046; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5047; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5048; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5049; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5050; GFX90A-TGSPLIT-NEXT:    buffer_invl2
5051; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5052; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5053; GFX90A-TGSPLIT-NEXT:    s_endpgm
5054;
5055; GFX940-NOTTGSPLIT-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
5056; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5057; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5058; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5059; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5060; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5061; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5062; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
5063; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5064; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
5065; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5066; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5067;
5068; GFX940-TGSPLIT-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
5069; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5070; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5071; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5072; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5073; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5074; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5075; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
5076; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5077; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
5078; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5079; GFX940-TGSPLIT-NEXT:    s_endpgm
5080;
5081; GFX11-WGP-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
5082; GFX11-WGP:       ; %bb.0: ; %entry
5083; GFX11-WGP-NEXT:    s_clause 0x1
5084; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5085; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5086; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5087; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5088; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5089; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5090; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5091; GFX11-WGP-NEXT:    buffer_gl0_inv
5092; GFX11-WGP-NEXT:    buffer_gl1_inv
5093; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5094; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5095; GFX11-WGP-NEXT:    s_endpgm
5096;
5097; GFX11-CU-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
5098; GFX11-CU:       ; %bb.0: ; %entry
5099; GFX11-CU-NEXT:    s_clause 0x1
5100; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5101; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5102; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5103; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5104; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5105; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5106; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5107; GFX11-CU-NEXT:    buffer_gl0_inv
5108; GFX11-CU-NEXT:    buffer_gl1_inv
5109; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5110; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5111; GFX11-CU-NEXT:    s_endpgm
5112    i32* %out, i32 %in, i32 %old) {
5113entry:
5114  %gep = getelementptr i32, i32* %out, i32 4
5115  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acquire monotonic
5116  %val0 = extractvalue { i32, i1 } %val, 0
5117  store i32 %val0, i32* %out, align 4
5118  ret void
5119}
5120
5121define amdgpu_kernel void @flat_system_release_monotonic_ret_cmpxchg(
5122; GFX7-LABEL: flat_system_release_monotonic_ret_cmpxchg:
5123; GFX7:       ; %bb.0: ; %entry
5124; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5125; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5126; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5127; GFX7-NEXT:    s_add_u32 s4, s0, 16
5128; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5129; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5130; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5131; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5132; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5133; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5134; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5135; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5136; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5137; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5138; GFX7-NEXT:    flat_store_dword v[0:1], v2
5139; GFX7-NEXT:    s_endpgm
5140;
5141; GFX10-WGP-LABEL: flat_system_release_monotonic_ret_cmpxchg:
5142; GFX10-WGP:       ; %bb.0: ; %entry
5143; GFX10-WGP-NEXT:    s_clause 0x1
5144; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5145; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5146; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5147; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5148; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5149; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5150; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5151; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5152; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5153; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5154; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5155; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5156; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5157; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5158; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5159; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5160; GFX10-WGP-NEXT:    s_endpgm
5161;
5162; GFX10-CU-LABEL: flat_system_release_monotonic_ret_cmpxchg:
5163; GFX10-CU:       ; %bb.0: ; %entry
5164; GFX10-CU-NEXT:    s_clause 0x1
5165; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5166; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5167; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5168; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5169; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5170; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5171; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5172; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5173; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5174; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5175; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5176; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5177; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5178; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5179; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5180; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5181; GFX10-CU-NEXT:    s_endpgm
5182;
5183; SKIP-CACHE-INV-LABEL: flat_system_release_monotonic_ret_cmpxchg:
5184; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5185; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5186; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5187; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5188; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5189; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5190; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5191; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5192; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5193; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5194; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5195; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5196; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5197; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5198; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5199; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5200; SKIP-CACHE-INV-NEXT:    s_endpgm
5201;
5202; GFX90A-NOTTGSPLIT-LABEL: flat_system_release_monotonic_ret_cmpxchg:
5203; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5204; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5205; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5206; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5207; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5208; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5209; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
5210; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5211; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5212; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5213; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5214; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5215;
5216; GFX90A-TGSPLIT-LABEL: flat_system_release_monotonic_ret_cmpxchg:
5217; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5218; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5219; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5220; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5221; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5222; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5223; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
5224; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5225; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5226; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5227; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5228; GFX90A-TGSPLIT-NEXT:    s_endpgm
5229;
5230; GFX940-NOTTGSPLIT-LABEL: flat_system_release_monotonic_ret_cmpxchg:
5231; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5232; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5233; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5234; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5235; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5236; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5237; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
5238; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5239; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
5240; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5241; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5242; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5243;
5244; GFX940-TGSPLIT-LABEL: flat_system_release_monotonic_ret_cmpxchg:
5245; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5246; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5247; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5248; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5249; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5250; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5251; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
5252; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5253; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
5254; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5255; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5256; GFX940-TGSPLIT-NEXT:    s_endpgm
5257;
5258; GFX11-WGP-LABEL: flat_system_release_monotonic_ret_cmpxchg:
5259; GFX11-WGP:       ; %bb.0: ; %entry
5260; GFX11-WGP-NEXT:    s_clause 0x1
5261; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5262; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5263; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5264; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5265; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5266; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5267; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5268; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5269; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5270; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5271; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5272; GFX11-WGP-NEXT:    s_endpgm
5273;
5274; GFX11-CU-LABEL: flat_system_release_monotonic_ret_cmpxchg:
5275; GFX11-CU:       ; %bb.0: ; %entry
5276; GFX11-CU-NEXT:    s_clause 0x1
5277; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5278; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5279; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5280; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5281; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5282; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5283; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5284; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5285; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5286; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5287; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5288; GFX11-CU-NEXT:    s_endpgm
5289    i32* %out, i32 %in, i32 %old) {
5290entry:
5291  %gep = getelementptr i32, i32* %out, i32 4
5292  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in release monotonic
5293  %val0 = extractvalue { i32, i1 } %val, 0
5294  store i32 %val0, i32* %out, align 4
5295  ret void
5296}
5297
5298define amdgpu_kernel void @flat_system_acq_rel_monotonic_ret_cmpxchg(
5299; GFX7-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
5300; GFX7:       ; %bb.0: ; %entry
5301; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5302; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5303; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5304; GFX7-NEXT:    s_add_u32 s4, s0, 16
5305; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5306; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5307; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5308; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5309; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5310; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5311; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5312; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5313; GFX7-NEXT:    buffer_wbinvl1_vol
5314; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5315; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5316; GFX7-NEXT:    flat_store_dword v[0:1], v2
5317; GFX7-NEXT:    s_endpgm
5318;
5319; GFX10-WGP-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
5320; GFX10-WGP:       ; %bb.0: ; %entry
5321; GFX10-WGP-NEXT:    s_clause 0x1
5322; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5323; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5324; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5325; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5326; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5327; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5328; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5329; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5330; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5331; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5332; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5333; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5334; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5335; GFX10-WGP-NEXT:    buffer_gl0_inv
5336; GFX10-WGP-NEXT:    buffer_gl1_inv
5337; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5338; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5339; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5340; GFX10-WGP-NEXT:    s_endpgm
5341;
5342; GFX10-CU-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
5343; GFX10-CU:       ; %bb.0: ; %entry
5344; GFX10-CU-NEXT:    s_clause 0x1
5345; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5346; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5347; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5348; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5349; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5350; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5351; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5352; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5353; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5354; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5355; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5356; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5357; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5358; GFX10-CU-NEXT:    buffer_gl0_inv
5359; GFX10-CU-NEXT:    buffer_gl1_inv
5360; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5361; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5362; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5363; GFX10-CU-NEXT:    s_endpgm
5364;
5365; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
5366; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5367; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5368; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5369; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5370; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5371; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5372; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5373; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5374; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5375; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5376; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5377; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5378; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5379; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5380; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5381; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5382; SKIP-CACHE-INV-NEXT:    s_endpgm
5383;
5384; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
5385; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5386; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5387; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5388; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5389; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5390; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5391; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
5392; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5393; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5394; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5395; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
5396; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5397; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5398; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5399;
5400; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
5401; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5402; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5403; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5404; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5405; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5406; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5407; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
5408; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5409; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5410; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5411; GFX90A-TGSPLIT-NEXT:    buffer_invl2
5412; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5413; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5414; GFX90A-TGSPLIT-NEXT:    s_endpgm
5415;
5416; GFX940-NOTTGSPLIT-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
5417; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5418; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5419; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5420; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5421; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5422; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5423; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
5424; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5425; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
5426; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5427; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
5428; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5429; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5430;
5431; GFX940-TGSPLIT-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
5432; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5433; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5434; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5435; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5436; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5437; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5438; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
5439; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5440; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
5441; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5442; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
5443; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5444; GFX940-TGSPLIT-NEXT:    s_endpgm
5445;
5446; GFX11-WGP-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
5447; GFX11-WGP:       ; %bb.0: ; %entry
5448; GFX11-WGP-NEXT:    s_clause 0x1
5449; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5450; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5451; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5452; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5453; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5454; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5455; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5456; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5457; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5458; GFX11-WGP-NEXT:    buffer_gl0_inv
5459; GFX11-WGP-NEXT:    buffer_gl1_inv
5460; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5461; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5462; GFX11-WGP-NEXT:    s_endpgm
5463;
5464; GFX11-CU-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
5465; GFX11-CU:       ; %bb.0: ; %entry
5466; GFX11-CU-NEXT:    s_clause 0x1
5467; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5468; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5469; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5470; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5471; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5472; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5473; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5474; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5475; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5476; GFX11-CU-NEXT:    buffer_gl0_inv
5477; GFX11-CU-NEXT:    buffer_gl1_inv
5478; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5479; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5480; GFX11-CU-NEXT:    s_endpgm
5481    i32* %out, i32 %in, i32 %old) {
5482entry:
5483  %gep = getelementptr i32, i32* %out, i32 4
5484  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acq_rel monotonic
5485  %val0 = extractvalue { i32, i1 } %val, 0
5486  store i32 %val0, i32* %out, align 4
5487  ret void
5488}
5489
5490define amdgpu_kernel void @flat_system_seq_cst_monotonic_ret_cmpxchg(
5491; GFX7-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
5492; GFX7:       ; %bb.0: ; %entry
5493; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5494; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5495; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5496; GFX7-NEXT:    s_add_u32 s4, s0, 16
5497; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5498; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5499; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5500; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5501; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5502; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5503; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5504; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5505; GFX7-NEXT:    buffer_wbinvl1_vol
5506; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5507; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5508; GFX7-NEXT:    flat_store_dword v[0:1], v2
5509; GFX7-NEXT:    s_endpgm
5510;
5511; GFX10-WGP-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
5512; GFX10-WGP:       ; %bb.0: ; %entry
5513; GFX10-WGP-NEXT:    s_clause 0x1
5514; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5515; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5516; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5517; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5518; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5519; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5520; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5521; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5522; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5523; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5524; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5525; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5526; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5527; GFX10-WGP-NEXT:    buffer_gl0_inv
5528; GFX10-WGP-NEXT:    buffer_gl1_inv
5529; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5530; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5531; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5532; GFX10-WGP-NEXT:    s_endpgm
5533;
5534; GFX10-CU-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
5535; GFX10-CU:       ; %bb.0: ; %entry
5536; GFX10-CU-NEXT:    s_clause 0x1
5537; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5538; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5539; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5540; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5541; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5542; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5543; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5544; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5545; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5546; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5547; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5548; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5549; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5550; GFX10-CU-NEXT:    buffer_gl0_inv
5551; GFX10-CU-NEXT:    buffer_gl1_inv
5552; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5553; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5554; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5555; GFX10-CU-NEXT:    s_endpgm
5556;
5557; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
5558; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5559; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5560; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5561; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5562; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5563; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5564; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5565; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5566; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5567; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5568; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5569; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5570; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5571; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5572; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5573; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5574; SKIP-CACHE-INV-NEXT:    s_endpgm
5575;
5576; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
5577; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5578; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5579; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5580; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5581; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5582; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5583; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
5584; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5585; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5586; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5587; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
5588; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5589; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5590; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5591;
5592; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
5593; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5594; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5595; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5596; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5597; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5598; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5599; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
5600; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5601; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5602; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5603; GFX90A-TGSPLIT-NEXT:    buffer_invl2
5604; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5605; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5606; GFX90A-TGSPLIT-NEXT:    s_endpgm
5607;
5608; GFX940-NOTTGSPLIT-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
5609; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5610; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5611; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5612; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5613; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5614; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5615; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
5616; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5617; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
5618; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5619; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
5620; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5621; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5622;
5623; GFX940-TGSPLIT-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
5624; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5625; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5626; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5627; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5628; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5629; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5630; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
5631; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5632; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
5633; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5634; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
5635; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5636; GFX940-TGSPLIT-NEXT:    s_endpgm
5637;
5638; GFX11-WGP-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
5639; GFX11-WGP:       ; %bb.0: ; %entry
5640; GFX11-WGP-NEXT:    s_clause 0x1
5641; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5642; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5643; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5644; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5645; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5646; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5647; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5648; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5649; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5650; GFX11-WGP-NEXT:    buffer_gl0_inv
5651; GFX11-WGP-NEXT:    buffer_gl1_inv
5652; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5653; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5654; GFX11-WGP-NEXT:    s_endpgm
5655;
5656; GFX11-CU-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
5657; GFX11-CU:       ; %bb.0: ; %entry
5658; GFX11-CU-NEXT:    s_clause 0x1
5659; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5660; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5661; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5662; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5663; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5664; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5665; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5666; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5667; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5668; GFX11-CU-NEXT:    buffer_gl0_inv
5669; GFX11-CU-NEXT:    buffer_gl1_inv
5670; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5671; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5672; GFX11-CU-NEXT:    s_endpgm
5673    i32* %out, i32 %in, i32 %old) {
5674entry:
5675  %gep = getelementptr i32, i32* %out, i32 4
5676  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst monotonic
5677  %val0 = extractvalue { i32, i1 } %val, 0
5678  store i32 %val0, i32* %out, align 4
5679  ret void
5680}
5681
5682define amdgpu_kernel void @flat_system_monotonic_acquire_ret_cmpxchg(
5683; GFX7-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
5684; GFX7:       ; %bb.0: ; %entry
5685; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5686; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5687; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5688; GFX7-NEXT:    s_add_u32 s4, s0, 16
5689; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5690; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5691; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5692; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5693; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5694; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5695; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5696; GFX7-NEXT:    buffer_wbinvl1_vol
5697; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5698; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5699; GFX7-NEXT:    flat_store_dword v[0:1], v2
5700; GFX7-NEXT:    s_endpgm
5701;
5702; GFX10-WGP-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
5703; GFX10-WGP:       ; %bb.0: ; %entry
5704; GFX10-WGP-NEXT:    s_clause 0x1
5705; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5706; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5707; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5708; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5709; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5710; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5711; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5712; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5713; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5714; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5715; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5716; GFX10-WGP-NEXT:    buffer_gl0_inv
5717; GFX10-WGP-NEXT:    buffer_gl1_inv
5718; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5719; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5720; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5721; GFX10-WGP-NEXT:    s_endpgm
5722;
5723; GFX10-CU-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
5724; GFX10-CU:       ; %bb.0: ; %entry
5725; GFX10-CU-NEXT:    s_clause 0x1
5726; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5727; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5728; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5729; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5730; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5731; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5732; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5733; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5734; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5735; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5736; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5737; GFX10-CU-NEXT:    buffer_gl0_inv
5738; GFX10-CU-NEXT:    buffer_gl1_inv
5739; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5740; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5741; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5742; GFX10-CU-NEXT:    s_endpgm
5743;
5744; SKIP-CACHE-INV-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
5745; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5746; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5747; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5748; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5749; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5750; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5751; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5752; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5753; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5754; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5755; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5756; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5757; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5758; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5759; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5760; SKIP-CACHE-INV-NEXT:    s_endpgm
5761;
5762; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
5763; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5764; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5765; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5766; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5767; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5768; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5769; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5770; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5771; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
5772; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5773; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5774; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5775;
5776; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
5777; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5778; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5779; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5780; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5781; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5782; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5783; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5784; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5785; GFX90A-TGSPLIT-NEXT:    buffer_invl2
5786; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5787; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5788; GFX90A-TGSPLIT-NEXT:    s_endpgm
5789;
5790; GFX940-NOTTGSPLIT-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
5791; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5792; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5793; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5794; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5795; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5796; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5797; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
5798; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5799; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
5800; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5801; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5802;
5803; GFX940-TGSPLIT-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
5804; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5805; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5806; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5807; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5808; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5809; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5810; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
5811; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5812; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
5813; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5814; GFX940-TGSPLIT-NEXT:    s_endpgm
5815;
5816; GFX11-WGP-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
5817; GFX11-WGP:       ; %bb.0: ; %entry
5818; GFX11-WGP-NEXT:    s_clause 0x1
5819; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5820; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5821; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5822; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5823; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5824; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5825; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5826; GFX11-WGP-NEXT:    buffer_gl0_inv
5827; GFX11-WGP-NEXT:    buffer_gl1_inv
5828; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5829; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5830; GFX11-WGP-NEXT:    s_endpgm
5831;
5832; GFX11-CU-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
5833; GFX11-CU:       ; %bb.0: ; %entry
5834; GFX11-CU-NEXT:    s_clause 0x1
5835; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5836; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5837; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5838; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5839; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5840; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5841; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5842; GFX11-CU-NEXT:    buffer_gl0_inv
5843; GFX11-CU-NEXT:    buffer_gl1_inv
5844; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5845; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5846; GFX11-CU-NEXT:    s_endpgm
5847    i32* %out, i32 %in, i32 %old) {
5848entry:
5849  %gep = getelementptr i32, i32* %out, i32 4
5850  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in monotonic acquire
5851  %val0 = extractvalue { i32, i1 } %val, 0
5852  store i32 %val0, i32* %out, align 4
5853  ret void
5854}
5855
5856define amdgpu_kernel void @flat_system_acquire_acquire_ret_cmpxchg(
5857; GFX7-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
5858; GFX7:       ; %bb.0: ; %entry
5859; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5860; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5861; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5862; GFX7-NEXT:    s_add_u32 s4, s0, 16
5863; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5864; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5865; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5866; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5867; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5868; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5869; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5870; GFX7-NEXT:    buffer_wbinvl1_vol
5871; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5872; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5873; GFX7-NEXT:    flat_store_dword v[0:1], v2
5874; GFX7-NEXT:    s_endpgm
5875;
5876; GFX10-WGP-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
5877; GFX10-WGP:       ; %bb.0: ; %entry
5878; GFX10-WGP-NEXT:    s_clause 0x1
5879; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5880; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5881; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5882; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5883; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5884; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5885; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5886; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5887; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5888; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5889; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5890; GFX10-WGP-NEXT:    buffer_gl0_inv
5891; GFX10-WGP-NEXT:    buffer_gl1_inv
5892; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5893; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5894; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5895; GFX10-WGP-NEXT:    s_endpgm
5896;
5897; GFX10-CU-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
5898; GFX10-CU:       ; %bb.0: ; %entry
5899; GFX10-CU-NEXT:    s_clause 0x1
5900; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5901; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5902; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5903; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5904; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5905; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5906; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5907; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5908; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5909; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5910; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5911; GFX10-CU-NEXT:    buffer_gl0_inv
5912; GFX10-CU-NEXT:    buffer_gl1_inv
5913; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5914; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5915; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5916; GFX10-CU-NEXT:    s_endpgm
5917;
5918; SKIP-CACHE-INV-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
5919; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5920; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5921; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5922; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5923; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5924; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5925; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5926; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5927; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5928; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5929; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5930; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5931; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5932; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5933; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5934; SKIP-CACHE-INV-NEXT:    s_endpgm
5935;
5936; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
5937; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5938; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5939; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5940; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5941; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5942; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5943; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5944; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5945; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
5946; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5947; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5948; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5949;
5950; GFX90A-TGSPLIT-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
5951; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5952; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5953; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5954; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5955; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5956; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5957; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5958; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5959; GFX90A-TGSPLIT-NEXT:    buffer_invl2
5960; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5961; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5962; GFX90A-TGSPLIT-NEXT:    s_endpgm
5963;
5964; GFX940-NOTTGSPLIT-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
5965; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5966; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5967; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5968; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5969; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5970; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5971; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
5972; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5973; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
5974; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5975; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5976;
5977; GFX940-TGSPLIT-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
5978; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5979; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5980; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5981; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5982; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5983; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5984; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
5985; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5986; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
5987; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5988; GFX940-TGSPLIT-NEXT:    s_endpgm
5989;
5990; GFX11-WGP-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
5991; GFX11-WGP:       ; %bb.0: ; %entry
5992; GFX11-WGP-NEXT:    s_clause 0x1
5993; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5994; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5995; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5996; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5997; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5998; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5999; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6000; GFX11-WGP-NEXT:    buffer_gl0_inv
6001; GFX11-WGP-NEXT:    buffer_gl1_inv
6002; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6003; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6004; GFX11-WGP-NEXT:    s_endpgm
6005;
6006; GFX11-CU-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
6007; GFX11-CU:       ; %bb.0: ; %entry
6008; GFX11-CU-NEXT:    s_clause 0x1
6009; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6010; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6011; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6012; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6013; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
6014; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6015; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6016; GFX11-CU-NEXT:    buffer_gl0_inv
6017; GFX11-CU-NEXT:    buffer_gl1_inv
6018; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6019; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6020; GFX11-CU-NEXT:    s_endpgm
6021    i32* %out, i32 %in, i32 %old) {
6022entry:
6023  %gep = getelementptr i32, i32* %out, i32 4
6024  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acquire acquire
6025  %val0 = extractvalue { i32, i1 } %val, 0
6026  store i32 %val0, i32* %out, align 4
6027  ret void
6028}
6029
6030define amdgpu_kernel void @flat_system_release_acquire_ret_cmpxchg(
6031; GFX7-LABEL: flat_system_release_acquire_ret_cmpxchg:
6032; GFX7:       ; %bb.0: ; %entry
6033; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6034; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6035; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6036; GFX7-NEXT:    s_add_u32 s4, s0, 16
6037; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6038; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6039; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6040; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6041; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6042; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6043; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6044; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6045; GFX7-NEXT:    buffer_wbinvl1_vol
6046; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6047; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6048; GFX7-NEXT:    flat_store_dword v[0:1], v2
6049; GFX7-NEXT:    s_endpgm
6050;
6051; GFX10-WGP-LABEL: flat_system_release_acquire_ret_cmpxchg:
6052; GFX10-WGP:       ; %bb.0: ; %entry
6053; GFX10-WGP-NEXT:    s_clause 0x1
6054; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6055; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6056; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6057; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6058; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6059; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6060; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6061; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6062; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6063; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6064; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6065; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6066; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6067; GFX10-WGP-NEXT:    buffer_gl0_inv
6068; GFX10-WGP-NEXT:    buffer_gl1_inv
6069; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6070; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6071; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6072; GFX10-WGP-NEXT:    s_endpgm
6073;
6074; GFX10-CU-LABEL: flat_system_release_acquire_ret_cmpxchg:
6075; GFX10-CU:       ; %bb.0: ; %entry
6076; GFX10-CU-NEXT:    s_clause 0x1
6077; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6078; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6079; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6080; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6081; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6082; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6083; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6084; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6085; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6086; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6087; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6088; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6089; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6090; GFX10-CU-NEXT:    buffer_gl0_inv
6091; GFX10-CU-NEXT:    buffer_gl1_inv
6092; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6093; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6094; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6095; GFX10-CU-NEXT:    s_endpgm
6096;
6097; SKIP-CACHE-INV-LABEL: flat_system_release_acquire_ret_cmpxchg:
6098; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6099; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6100; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6101; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6102; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6103; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6104; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6105; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6106; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6107; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6108; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6109; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6110; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6111; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6112; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6113; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6114; SKIP-CACHE-INV-NEXT:    s_endpgm
6115;
6116; GFX90A-NOTTGSPLIT-LABEL: flat_system_release_acquire_ret_cmpxchg:
6117; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6118; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6119; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6120; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6121; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6122; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6123; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
6124; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6125; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6126; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6127; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
6128; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6129; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6130; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6131;
6132; GFX90A-TGSPLIT-LABEL: flat_system_release_acquire_ret_cmpxchg:
6133; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6134; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6135; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6136; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6137; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6138; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6139; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
6140; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6141; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6142; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6143; GFX90A-TGSPLIT-NEXT:    buffer_invl2
6144; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6145; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6146; GFX90A-TGSPLIT-NEXT:    s_endpgm
6147;
6148; GFX940-NOTTGSPLIT-LABEL: flat_system_release_acquire_ret_cmpxchg:
6149; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6150; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6151; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6152; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6153; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6154; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6155; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
6156; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6157; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
6158; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6159; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
6160; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6161; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6162;
6163; GFX940-TGSPLIT-LABEL: flat_system_release_acquire_ret_cmpxchg:
6164; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6165; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6166; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6167; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6168; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6169; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6170; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
6171; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6172; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
6173; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6174; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
6175; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6176; GFX940-TGSPLIT-NEXT:    s_endpgm
6177;
6178; GFX11-WGP-LABEL: flat_system_release_acquire_ret_cmpxchg:
6179; GFX11-WGP:       ; %bb.0: ; %entry
6180; GFX11-WGP-NEXT:    s_clause 0x1
6181; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6182; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6183; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6184; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6185; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
6186; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6187; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6188; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6189; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6190; GFX11-WGP-NEXT:    buffer_gl0_inv
6191; GFX11-WGP-NEXT:    buffer_gl1_inv
6192; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6193; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6194; GFX11-WGP-NEXT:    s_endpgm
6195;
6196; GFX11-CU-LABEL: flat_system_release_acquire_ret_cmpxchg:
6197; GFX11-CU:       ; %bb.0: ; %entry
6198; GFX11-CU-NEXT:    s_clause 0x1
6199; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6200; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6201; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6202; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6203; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
6204; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6205; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6206; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6207; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6208; GFX11-CU-NEXT:    buffer_gl0_inv
6209; GFX11-CU-NEXT:    buffer_gl1_inv
6210; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6211; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6212; GFX11-CU-NEXT:    s_endpgm
6213    i32* %out, i32 %in, i32 %old) {
6214entry:
6215  %gep = getelementptr i32, i32* %out, i32 4
6216  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in release acquire
6217  %val0 = extractvalue { i32, i1 } %val, 0
6218  store i32 %val0, i32* %out, align 4
6219  ret void
6220}
6221
6222define amdgpu_kernel void @flat_system_acq_rel_acquire_ret_cmpxchg(
6223; GFX7-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
6224; GFX7:       ; %bb.0: ; %entry
6225; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6226; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6227; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6228; GFX7-NEXT:    s_add_u32 s4, s0, 16
6229; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6230; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6231; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6232; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6233; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6234; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6235; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6236; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6237; GFX7-NEXT:    buffer_wbinvl1_vol
6238; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6239; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6240; GFX7-NEXT:    flat_store_dword v[0:1], v2
6241; GFX7-NEXT:    s_endpgm
6242;
6243; GFX10-WGP-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
6244; GFX10-WGP:       ; %bb.0: ; %entry
6245; GFX10-WGP-NEXT:    s_clause 0x1
6246; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6247; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6248; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6249; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6250; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6251; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6252; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6253; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6254; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6255; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6256; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6257; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6258; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6259; GFX10-WGP-NEXT:    buffer_gl0_inv
6260; GFX10-WGP-NEXT:    buffer_gl1_inv
6261; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6262; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6263; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6264; GFX10-WGP-NEXT:    s_endpgm
6265;
6266; GFX10-CU-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
6267; GFX10-CU:       ; %bb.0: ; %entry
6268; GFX10-CU-NEXT:    s_clause 0x1
6269; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6270; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6271; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6272; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6273; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6274; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6275; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6276; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6277; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6278; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6279; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6280; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6281; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6282; GFX10-CU-NEXT:    buffer_gl0_inv
6283; GFX10-CU-NEXT:    buffer_gl1_inv
6284; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6285; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6286; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6287; GFX10-CU-NEXT:    s_endpgm
6288;
6289; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
6290; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6291; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6292; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6293; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6294; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6295; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6296; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6297; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6298; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6299; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6300; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6301; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6302; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6303; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6304; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6305; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6306; SKIP-CACHE-INV-NEXT:    s_endpgm
6307;
6308; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
6309; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6310; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6311; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6312; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6313; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6314; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6315; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
6316; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6317; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6318; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6319; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
6320; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6321; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6322; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6323;
6324; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
6325; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6326; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6327; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6328; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6329; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6330; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6331; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
6332; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6333; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6334; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6335; GFX90A-TGSPLIT-NEXT:    buffer_invl2
6336; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6337; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6338; GFX90A-TGSPLIT-NEXT:    s_endpgm
6339;
6340; GFX940-NOTTGSPLIT-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
6341; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6342; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6343; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6344; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6345; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6346; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6347; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
6348; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6349; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
6350; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6351; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
6352; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6353; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6354;
6355; GFX940-TGSPLIT-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
6356; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6357; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6358; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6359; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6360; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6361; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6362; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
6363; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6364; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
6365; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6366; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
6367; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6368; GFX940-TGSPLIT-NEXT:    s_endpgm
6369;
6370; GFX11-WGP-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
6371; GFX11-WGP:       ; %bb.0: ; %entry
6372; GFX11-WGP-NEXT:    s_clause 0x1
6373; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6374; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6375; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6376; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6377; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
6378; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6379; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6380; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6381; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6382; GFX11-WGP-NEXT:    buffer_gl0_inv
6383; GFX11-WGP-NEXT:    buffer_gl1_inv
6384; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6385; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6386; GFX11-WGP-NEXT:    s_endpgm
6387;
6388; GFX11-CU-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
6389; GFX11-CU:       ; %bb.0: ; %entry
6390; GFX11-CU-NEXT:    s_clause 0x1
6391; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6392; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6393; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6394; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6395; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
6396; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6397; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6398; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6399; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6400; GFX11-CU-NEXT:    buffer_gl0_inv
6401; GFX11-CU-NEXT:    buffer_gl1_inv
6402; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6403; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6404; GFX11-CU-NEXT:    s_endpgm
6405    i32* %out, i32 %in, i32 %old) {
6406entry:
6407  %gep = getelementptr i32, i32* %out, i32 4
6408  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acq_rel acquire
6409  %val0 = extractvalue { i32, i1 } %val, 0
6410  store i32 %val0, i32* %out, align 4
6411  ret void
6412}
6413
6414define amdgpu_kernel void @flat_system_seq_cst_acquire_ret_cmpxchg(
6415; GFX7-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
6416; GFX7:       ; %bb.0: ; %entry
6417; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6418; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6419; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6420; GFX7-NEXT:    s_add_u32 s4, s0, 16
6421; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6422; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6423; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6424; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6425; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6426; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6427; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6428; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6429; GFX7-NEXT:    buffer_wbinvl1_vol
6430; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6431; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6432; GFX7-NEXT:    flat_store_dword v[0:1], v2
6433; GFX7-NEXT:    s_endpgm
6434;
6435; GFX10-WGP-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
6436; GFX10-WGP:       ; %bb.0: ; %entry
6437; GFX10-WGP-NEXT:    s_clause 0x1
6438; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6439; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6440; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6441; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6442; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6443; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6444; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6445; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6446; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6447; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6448; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6449; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6450; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6451; GFX10-WGP-NEXT:    buffer_gl0_inv
6452; GFX10-WGP-NEXT:    buffer_gl1_inv
6453; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6454; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6455; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6456; GFX10-WGP-NEXT:    s_endpgm
6457;
6458; GFX10-CU-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
6459; GFX10-CU:       ; %bb.0: ; %entry
6460; GFX10-CU-NEXT:    s_clause 0x1
6461; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6462; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6463; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6464; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6465; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6466; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6467; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6468; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6469; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6470; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6471; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6472; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6473; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6474; GFX10-CU-NEXT:    buffer_gl0_inv
6475; GFX10-CU-NEXT:    buffer_gl1_inv
6476; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6477; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6478; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6479; GFX10-CU-NEXT:    s_endpgm
6480;
6481; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
6482; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6483; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6484; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6485; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6486; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6487; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6488; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6489; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6490; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6491; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6492; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6493; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6494; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6495; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6496; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6497; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6498; SKIP-CACHE-INV-NEXT:    s_endpgm
6499;
6500; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
6501; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6502; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6503; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6504; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6505; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6506; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6507; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
6508; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6509; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6510; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6511; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
6512; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6513; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6514; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6515;
6516; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
6517; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6518; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6519; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6520; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6521; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6522; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6523; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
6524; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6525; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6526; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6527; GFX90A-TGSPLIT-NEXT:    buffer_invl2
6528; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6529; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6530; GFX90A-TGSPLIT-NEXT:    s_endpgm
6531;
6532; GFX940-NOTTGSPLIT-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
6533; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6534; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6535; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6536; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6537; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6538; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6539; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
6540; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6541; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
6542; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6543; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
6544; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6545; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6546;
6547; GFX940-TGSPLIT-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
6548; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6549; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6550; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6551; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6552; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6553; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6554; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
6555; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6556; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
6557; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6558; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
6559; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6560; GFX940-TGSPLIT-NEXT:    s_endpgm
6561;
6562; GFX11-WGP-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
6563; GFX11-WGP:       ; %bb.0: ; %entry
6564; GFX11-WGP-NEXT:    s_clause 0x1
6565; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6566; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6567; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6568; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6569; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
6570; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6571; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6572; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6573; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6574; GFX11-WGP-NEXT:    buffer_gl0_inv
6575; GFX11-WGP-NEXT:    buffer_gl1_inv
6576; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6577; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6578; GFX11-WGP-NEXT:    s_endpgm
6579;
6580; GFX11-CU-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
6581; GFX11-CU:       ; %bb.0: ; %entry
6582; GFX11-CU-NEXT:    s_clause 0x1
6583; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6584; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6585; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6586; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6587; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
6588; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6589; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6590; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6591; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6592; GFX11-CU-NEXT:    buffer_gl0_inv
6593; GFX11-CU-NEXT:    buffer_gl1_inv
6594; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6595; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6596; GFX11-CU-NEXT:    s_endpgm
6597    i32* %out, i32 %in, i32 %old) {
6598entry:
6599  %gep = getelementptr i32, i32* %out, i32 4
6600  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst acquire
6601  %val0 = extractvalue { i32, i1 } %val, 0
6602  store i32 %val0, i32* %out, align 4
6603  ret void
6604}
6605
6606define amdgpu_kernel void @flat_system_monotonic_seq_cst_ret_cmpxchg(
6607; GFX7-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
6608; GFX7:       ; %bb.0: ; %entry
6609; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6610; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6611; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6612; GFX7-NEXT:    s_add_u32 s4, s0, 16
6613; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6614; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6615; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6616; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6617; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6618; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6619; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6620; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6621; GFX7-NEXT:    buffer_wbinvl1_vol
6622; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6623; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6624; GFX7-NEXT:    flat_store_dword v[0:1], v2
6625; GFX7-NEXT:    s_endpgm
6626;
6627; GFX10-WGP-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
6628; GFX10-WGP:       ; %bb.0: ; %entry
6629; GFX10-WGP-NEXT:    s_clause 0x1
6630; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6631; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6632; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6633; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6634; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6635; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6636; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6637; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6638; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6639; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6640; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6641; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6642; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6643; GFX10-WGP-NEXT:    buffer_gl0_inv
6644; GFX10-WGP-NEXT:    buffer_gl1_inv
6645; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6646; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6647; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6648; GFX10-WGP-NEXT:    s_endpgm
6649;
6650; GFX10-CU-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
6651; GFX10-CU:       ; %bb.0: ; %entry
6652; GFX10-CU-NEXT:    s_clause 0x1
6653; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6654; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6655; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6656; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6657; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6658; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6659; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6660; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6661; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6662; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6663; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6664; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6665; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6666; GFX10-CU-NEXT:    buffer_gl0_inv
6667; GFX10-CU-NEXT:    buffer_gl1_inv
6668; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6669; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6670; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6671; GFX10-CU-NEXT:    s_endpgm
6672;
6673; SKIP-CACHE-INV-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
6674; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6675; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6676; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6677; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6678; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6679; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6680; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6681; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6682; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6683; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6684; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6685; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6686; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6687; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6688; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6689; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6690; SKIP-CACHE-INV-NEXT:    s_endpgm
6691;
6692; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
6693; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6694; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6695; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6696; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6697; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6698; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6699; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
6700; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6701; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6702; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6703; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
6704; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6705; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6706; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6707;
6708; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
6709; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6710; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6711; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6712; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6713; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6714; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6715; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
6716; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6717; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6718; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6719; GFX90A-TGSPLIT-NEXT:    buffer_invl2
6720; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6721; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6722; GFX90A-TGSPLIT-NEXT:    s_endpgm
6723;
6724; GFX940-NOTTGSPLIT-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
6725; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6726; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6727; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6728; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6729; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6730; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6731; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
6732; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6733; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
6734; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6735; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
6736; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6737; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6738;
6739; GFX940-TGSPLIT-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
6740; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6741; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6742; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6743; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6744; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6745; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6746; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
6747; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6748; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
6749; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6750; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
6751; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6752; GFX940-TGSPLIT-NEXT:    s_endpgm
6753;
6754; GFX11-WGP-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
6755; GFX11-WGP:       ; %bb.0: ; %entry
6756; GFX11-WGP-NEXT:    s_clause 0x1
6757; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6758; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6759; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6760; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6761; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
6762; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6763; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6764; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6765; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6766; GFX11-WGP-NEXT:    buffer_gl0_inv
6767; GFX11-WGP-NEXT:    buffer_gl1_inv
6768; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6769; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6770; GFX11-WGP-NEXT:    s_endpgm
6771;
6772; GFX11-CU-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
6773; GFX11-CU:       ; %bb.0: ; %entry
6774; GFX11-CU-NEXT:    s_clause 0x1
6775; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6776; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6777; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6778; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6779; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
6780; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6781; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6782; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6783; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6784; GFX11-CU-NEXT:    buffer_gl0_inv
6785; GFX11-CU-NEXT:    buffer_gl1_inv
6786; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6787; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6788; GFX11-CU-NEXT:    s_endpgm
6789    i32* %out, i32 %in, i32 %old) {
6790entry:
6791  %gep = getelementptr i32, i32* %out, i32 4
6792  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in monotonic seq_cst
6793  %val0 = extractvalue { i32, i1 } %val, 0
6794  store i32 %val0, i32* %out, align 4
6795  ret void
6796}
6797
6798define amdgpu_kernel void @flat_system_acquire_seq_cst_ret_cmpxchg(
6799; GFX7-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
6800; GFX7:       ; %bb.0: ; %entry
6801; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6802; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6803; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6804; GFX7-NEXT:    s_add_u32 s4, s0, 16
6805; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6806; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6807; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6808; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6809; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6810; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6811; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6812; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6813; GFX7-NEXT:    buffer_wbinvl1_vol
6814; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6815; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6816; GFX7-NEXT:    flat_store_dword v[0:1], v2
6817; GFX7-NEXT:    s_endpgm
6818;
6819; GFX10-WGP-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
6820; GFX10-WGP:       ; %bb.0: ; %entry
6821; GFX10-WGP-NEXT:    s_clause 0x1
6822; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6823; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6824; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6825; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6826; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6827; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6828; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6829; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6830; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6831; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6832; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6833; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6834; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6835; GFX10-WGP-NEXT:    buffer_gl0_inv
6836; GFX10-WGP-NEXT:    buffer_gl1_inv
6837; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6838; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6839; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6840; GFX10-WGP-NEXT:    s_endpgm
6841;
6842; GFX10-CU-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
6843; GFX10-CU:       ; %bb.0: ; %entry
6844; GFX10-CU-NEXT:    s_clause 0x1
6845; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6846; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6847; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6848; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6849; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6850; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6851; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6852; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6853; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6854; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6855; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6856; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6857; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6858; GFX10-CU-NEXT:    buffer_gl0_inv
6859; GFX10-CU-NEXT:    buffer_gl1_inv
6860; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6861; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6862; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6863; GFX10-CU-NEXT:    s_endpgm
6864;
6865; SKIP-CACHE-INV-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
6866; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6867; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6868; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6869; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6870; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6871; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6872; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6873; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6874; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6875; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6876; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6877; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6878; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6879; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6880; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6881; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6882; SKIP-CACHE-INV-NEXT:    s_endpgm
6883;
6884; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
6885; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6886; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6887; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6888; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6889; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6890; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6891; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
6892; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6893; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6894; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6895; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
6896; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6897; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6898; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6899;
6900; GFX90A-TGSPLIT-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
6901; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6902; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6903; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6904; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6905; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6906; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6907; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
6908; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6909; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6910; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6911; GFX90A-TGSPLIT-NEXT:    buffer_invl2
6912; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6913; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6914; GFX90A-TGSPLIT-NEXT:    s_endpgm
6915;
6916; GFX940-NOTTGSPLIT-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
6917; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6918; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6919; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6920; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6921; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6922; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6923; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
6924; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6925; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
6926; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6927; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
6928; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6929; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6930;
6931; GFX940-TGSPLIT-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
6932; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6933; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6934; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6935; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6936; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6937; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6938; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
6939; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6940; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
6941; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6942; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
6943; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6944; GFX940-TGSPLIT-NEXT:    s_endpgm
6945;
6946; GFX11-WGP-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
6947; GFX11-WGP:       ; %bb.0: ; %entry
6948; GFX11-WGP-NEXT:    s_clause 0x1
6949; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6950; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6951; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6952; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6953; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
6954; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6955; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6956; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6957; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6958; GFX11-WGP-NEXT:    buffer_gl0_inv
6959; GFX11-WGP-NEXT:    buffer_gl1_inv
6960; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6961; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6962; GFX11-WGP-NEXT:    s_endpgm
6963;
6964; GFX11-CU-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
6965; GFX11-CU:       ; %bb.0: ; %entry
6966; GFX11-CU-NEXT:    s_clause 0x1
6967; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6968; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6969; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6970; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6971; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
6972; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6973; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6974; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6975; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6976; GFX11-CU-NEXT:    buffer_gl0_inv
6977; GFX11-CU-NEXT:    buffer_gl1_inv
6978; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6979; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6980; GFX11-CU-NEXT:    s_endpgm
6981    i32* %out, i32 %in, i32 %old) {
6982entry:
6983  %gep = getelementptr i32, i32* %out, i32 4
6984  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acquire seq_cst
6985  %val0 = extractvalue { i32, i1 } %val, 0
6986  store i32 %val0, i32* %out, align 4
6987  ret void
6988}
6989
6990define amdgpu_kernel void @flat_system_release_seq_cst_ret_cmpxchg(
6991; GFX7-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
6992; GFX7:       ; %bb.0: ; %entry
6993; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6994; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6995; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6996; GFX7-NEXT:    s_add_u32 s4, s0, 16
6997; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6998; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6999; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7000; GFX7-NEXT:    v_mov_b32_e32 v1, s5
7001; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7002; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7003; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7004; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7005; GFX7-NEXT:    buffer_wbinvl1_vol
7006; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7007; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7008; GFX7-NEXT:    flat_store_dword v[0:1], v2
7009; GFX7-NEXT:    s_endpgm
7010;
7011; GFX10-WGP-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
7012; GFX10-WGP:       ; %bb.0: ; %entry
7013; GFX10-WGP-NEXT:    s_clause 0x1
7014; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7015; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7016; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7017; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
7018; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
7019; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
7020; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7021; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
7022; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7023; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7024; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7025; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7026; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7027; GFX10-WGP-NEXT:    buffer_gl0_inv
7028; GFX10-WGP-NEXT:    buffer_gl1_inv
7029; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7030; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7031; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7032; GFX10-WGP-NEXT:    s_endpgm
7033;
7034; GFX10-CU-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
7035; GFX10-CU:       ; %bb.0: ; %entry
7036; GFX10-CU-NEXT:    s_clause 0x1
7037; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7038; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7039; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7040; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
7041; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
7042; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
7043; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7044; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
7045; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7046; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7047; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7048; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7049; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7050; GFX10-CU-NEXT:    buffer_gl0_inv
7051; GFX10-CU-NEXT:    buffer_gl1_inv
7052; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7053; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7054; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7055; GFX10-CU-NEXT:    s_endpgm
7056;
7057; SKIP-CACHE-INV-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
7058; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7059; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7060; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7061; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7062; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
7063; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
7064; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7065; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7066; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7067; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7068; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7069; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7070; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7071; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7072; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7073; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7074; SKIP-CACHE-INV-NEXT:    s_endpgm
7075;
7076; GFX90A-NOTTGSPLIT-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
7077; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7078; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7079; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7080; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7081; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7082; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7083; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
7084; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7085; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7086; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7087; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
7088; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7089; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7090; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7091;
7092; GFX90A-TGSPLIT-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
7093; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7094; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7095; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7096; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7097; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7098; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7099; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
7100; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7101; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7102; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7103; GFX90A-TGSPLIT-NEXT:    buffer_invl2
7104; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7105; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7106; GFX90A-TGSPLIT-NEXT:    s_endpgm
7107;
7108; GFX940-NOTTGSPLIT-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
7109; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7110; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7111; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7112; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7113; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7114; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7115; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
7116; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7117; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
7118; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7119; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
7120; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7121; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7122;
7123; GFX940-TGSPLIT-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
7124; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7125; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7126; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7127; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7128; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7129; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7130; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
7131; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7132; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
7133; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7134; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
7135; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7136; GFX940-TGSPLIT-NEXT:    s_endpgm
7137;
7138; GFX11-WGP-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
7139; GFX11-WGP:       ; %bb.0: ; %entry
7140; GFX11-WGP-NEXT:    s_clause 0x1
7141; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7142; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7143; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7144; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7145; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
7146; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7147; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7148; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7149; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7150; GFX11-WGP-NEXT:    buffer_gl0_inv
7151; GFX11-WGP-NEXT:    buffer_gl1_inv
7152; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7153; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7154; GFX11-WGP-NEXT:    s_endpgm
7155;
7156; GFX11-CU-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
7157; GFX11-CU:       ; %bb.0: ; %entry
7158; GFX11-CU-NEXT:    s_clause 0x1
7159; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7160; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7161; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7162; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7163; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
7164; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7165; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7166; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7167; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7168; GFX11-CU-NEXT:    buffer_gl0_inv
7169; GFX11-CU-NEXT:    buffer_gl1_inv
7170; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7171; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7172; GFX11-CU-NEXT:    s_endpgm
7173    i32* %out, i32 %in, i32 %old) {
7174entry:
7175  %gep = getelementptr i32, i32* %out, i32 4
7176  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in release seq_cst
7177  %val0 = extractvalue { i32, i1 } %val, 0
7178  store i32 %val0, i32* %out, align 4
7179  ret void
7180}
7181
7182define amdgpu_kernel void @flat_system_acq_rel_seq_cst_ret_cmpxchg(
7183; GFX7-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
7184; GFX7:       ; %bb.0: ; %entry
7185; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7186; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7187; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7188; GFX7-NEXT:    s_add_u32 s4, s0, 16
7189; GFX7-NEXT:    s_addc_u32 s5, s1, 0
7190; GFX7-NEXT:    v_mov_b32_e32 v0, s4
7191; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7192; GFX7-NEXT:    v_mov_b32_e32 v1, s5
7193; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7194; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7195; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7196; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7197; GFX7-NEXT:    buffer_wbinvl1_vol
7198; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7199; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7200; GFX7-NEXT:    flat_store_dword v[0:1], v2
7201; GFX7-NEXT:    s_endpgm
7202;
7203; GFX10-WGP-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
7204; GFX10-WGP:       ; %bb.0: ; %entry
7205; GFX10-WGP-NEXT:    s_clause 0x1
7206; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7207; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7208; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7209; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
7210; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
7211; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
7212; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7213; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
7214; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7215; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7216; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7217; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7218; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7219; GFX10-WGP-NEXT:    buffer_gl0_inv
7220; GFX10-WGP-NEXT:    buffer_gl1_inv
7221; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7222; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7223; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7224; GFX10-WGP-NEXT:    s_endpgm
7225;
7226; GFX10-CU-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
7227; GFX10-CU:       ; %bb.0: ; %entry
7228; GFX10-CU-NEXT:    s_clause 0x1
7229; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7230; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7231; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7232; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
7233; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
7234; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
7235; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7236; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
7237; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7238; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7239; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7240; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7241; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7242; GFX10-CU-NEXT:    buffer_gl0_inv
7243; GFX10-CU-NEXT:    buffer_gl1_inv
7244; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7245; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7246; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7247; GFX10-CU-NEXT:    s_endpgm
7248;
7249; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
7250; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7251; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7252; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7253; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7254; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
7255; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
7256; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7257; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7258; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7259; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7260; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7261; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7262; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7263; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7264; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7265; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7266; SKIP-CACHE-INV-NEXT:    s_endpgm
7267;
7268; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
7269; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7270; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7271; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7272; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7273; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7274; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7275; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
7276; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7277; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7278; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7279; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
7280; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7281; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7282; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7283;
7284; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
7285; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7286; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7287; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7288; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7289; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7290; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7291; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
7292; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7293; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7294; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7295; GFX90A-TGSPLIT-NEXT:    buffer_invl2
7296; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7297; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7298; GFX90A-TGSPLIT-NEXT:    s_endpgm
7299;
7300; GFX940-NOTTGSPLIT-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
7301; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7302; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7303; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7304; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7305; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7306; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7307; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
7308; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7309; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
7310; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7311; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
7312; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7313; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7314;
7315; GFX940-TGSPLIT-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
7316; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7317; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7318; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7319; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7320; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7321; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7322; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
7323; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7324; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
7325; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7326; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
7327; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7328; GFX940-TGSPLIT-NEXT:    s_endpgm
7329;
7330; GFX11-WGP-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
7331; GFX11-WGP:       ; %bb.0: ; %entry
7332; GFX11-WGP-NEXT:    s_clause 0x1
7333; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7334; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7335; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7336; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7337; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
7338; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7339; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7340; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7341; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7342; GFX11-WGP-NEXT:    buffer_gl0_inv
7343; GFX11-WGP-NEXT:    buffer_gl1_inv
7344; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7345; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7346; GFX11-WGP-NEXT:    s_endpgm
7347;
7348; GFX11-CU-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
7349; GFX11-CU:       ; %bb.0: ; %entry
7350; GFX11-CU-NEXT:    s_clause 0x1
7351; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7352; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7353; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7354; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7355; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
7356; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7357; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7358; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7359; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7360; GFX11-CU-NEXT:    buffer_gl0_inv
7361; GFX11-CU-NEXT:    buffer_gl1_inv
7362; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7363; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7364; GFX11-CU-NEXT:    s_endpgm
7365    i32* %out, i32 %in, i32 %old) {
7366entry:
7367  %gep = getelementptr i32, i32* %out, i32 4
7368  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acq_rel seq_cst
7369  %val0 = extractvalue { i32, i1 } %val, 0
7370  store i32 %val0, i32* %out, align 4
7371  ret void
7372}
7373
7374define amdgpu_kernel void @flat_system_seq_cst_seq_cst_ret_cmpxchg(
7375; GFX7-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
7376; GFX7:       ; %bb.0: ; %entry
7377; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7378; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7379; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7380; GFX7-NEXT:    s_add_u32 s4, s0, 16
7381; GFX7-NEXT:    s_addc_u32 s5, s1, 0
7382; GFX7-NEXT:    v_mov_b32_e32 v0, s4
7383; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7384; GFX7-NEXT:    v_mov_b32_e32 v1, s5
7385; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7386; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7387; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7388; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7389; GFX7-NEXT:    buffer_wbinvl1_vol
7390; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7391; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7392; GFX7-NEXT:    flat_store_dword v[0:1], v2
7393; GFX7-NEXT:    s_endpgm
7394;
7395; GFX10-WGP-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
7396; GFX10-WGP:       ; %bb.0: ; %entry
7397; GFX10-WGP-NEXT:    s_clause 0x1
7398; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7399; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7400; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7401; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
7402; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
7403; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
7404; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7405; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
7406; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7407; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7408; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7409; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7410; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7411; GFX10-WGP-NEXT:    buffer_gl0_inv
7412; GFX10-WGP-NEXT:    buffer_gl1_inv
7413; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7414; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7415; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7416; GFX10-WGP-NEXT:    s_endpgm
7417;
7418; GFX10-CU-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
7419; GFX10-CU:       ; %bb.0: ; %entry
7420; GFX10-CU-NEXT:    s_clause 0x1
7421; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7422; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7423; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7424; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
7425; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
7426; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
7427; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7428; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
7429; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7430; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7431; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7432; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7433; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7434; GFX10-CU-NEXT:    buffer_gl0_inv
7435; GFX10-CU-NEXT:    buffer_gl1_inv
7436; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7437; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7438; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7439; GFX10-CU-NEXT:    s_endpgm
7440;
7441; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
7442; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7443; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7444; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7445; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7446; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
7447; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
7448; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7449; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7450; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7451; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7452; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7453; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7454; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7455; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7456; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7457; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7458; SKIP-CACHE-INV-NEXT:    s_endpgm
7459;
7460; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
7461; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7462; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7463; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7464; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7465; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7466; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7467; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
7468; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7469; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7470; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7471; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
7472; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7473; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7474; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7475;
7476; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
7477; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7478; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7479; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7480; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7481; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7482; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7483; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
7484; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7485; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7486; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7487; GFX90A-TGSPLIT-NEXT:    buffer_invl2
7488; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7489; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7490; GFX90A-TGSPLIT-NEXT:    s_endpgm
7491;
7492; GFX940-NOTTGSPLIT-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
7493; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7494; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7495; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7496; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7497; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7498; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7499; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
7500; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7501; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
7502; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7503; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
7504; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7505; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7506;
7507; GFX940-TGSPLIT-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
7508; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7509; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7510; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7511; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7512; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7513; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7514; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
7515; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7516; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
7517; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7518; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
7519; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7520; GFX940-TGSPLIT-NEXT:    s_endpgm
7521;
7522; GFX11-WGP-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
7523; GFX11-WGP:       ; %bb.0: ; %entry
7524; GFX11-WGP-NEXT:    s_clause 0x1
7525; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7526; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7527; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7528; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7529; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
7530; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7531; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7532; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7533; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7534; GFX11-WGP-NEXT:    buffer_gl0_inv
7535; GFX11-WGP-NEXT:    buffer_gl1_inv
7536; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7537; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7538; GFX11-WGP-NEXT:    s_endpgm
7539;
7540; GFX11-CU-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
7541; GFX11-CU:       ; %bb.0: ; %entry
7542; GFX11-CU-NEXT:    s_clause 0x1
7543; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7544; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7545; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7546; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7547; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
7548; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7549; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7550; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7551; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7552; GFX11-CU-NEXT:    buffer_gl0_inv
7553; GFX11-CU-NEXT:    buffer_gl1_inv
7554; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7555; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7556; GFX11-CU-NEXT:    s_endpgm
7557    i32* %out, i32 %in, i32 %old) {
7558entry:
7559  %gep = getelementptr i32, i32* %out, i32 4
7560  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst seq_cst
7561  %val0 = extractvalue { i32, i1 } %val, 0
7562  store i32 %val0, i32* %out, align 4
7563  ret void
7564}
7565
7566define amdgpu_kernel void @flat_system_one_as_unordered_load(
7567; GFX7-LABEL: flat_system_one_as_unordered_load:
7568; GFX7:       ; %bb.0: ; %entry
7569; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7570; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7571; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7572; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7573; GFX7-NEXT:    flat_load_dword v2, v[0:1]
7574; GFX7-NEXT:    v_mov_b32_e32 v0, s2
7575; GFX7-NEXT:    v_mov_b32_e32 v1, s3
7576; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7577; GFX7-NEXT:    flat_store_dword v[0:1], v2
7578; GFX7-NEXT:    s_endpgm
7579;
7580; GFX10-WGP-LABEL: flat_system_one_as_unordered_load:
7581; GFX10-WGP:       ; %bb.0: ; %entry
7582; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7583; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7584; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7585; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7586; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
7587; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
7588; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
7589; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7590; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7591; GFX10-WGP-NEXT:    s_endpgm
7592;
7593; GFX10-CU-LABEL: flat_system_one_as_unordered_load:
7594; GFX10-CU:       ; %bb.0: ; %entry
7595; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7596; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7597; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7598; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7599; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
7600; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
7601; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
7602; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7603; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7604; GFX10-CU-NEXT:    s_endpgm
7605;
7606; SKIP-CACHE-INV-LABEL: flat_system_one_as_unordered_load:
7607; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7608; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7609; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7610; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7611; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
7612; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
7613; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7614; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7615; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7616; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7617; SKIP-CACHE-INV-NEXT:    s_endpgm
7618;
7619; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_unordered_load:
7620; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7621; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7622; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7623; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7624; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7625; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
7626; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7627; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7628; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7629; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7630; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7631;
7632; GFX90A-TGSPLIT-LABEL: flat_system_one_as_unordered_load:
7633; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7634; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7635; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7636; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7637; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7638; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
7639; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7640; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7641; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7642; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7643; GFX90A-TGSPLIT-NEXT:    s_endpgm
7644;
7645; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_unordered_load:
7646; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7647; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7648; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7649; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7650; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7651; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
7652; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7653; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7654; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7655; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7656; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7657;
7658; GFX940-TGSPLIT-LABEL: flat_system_one_as_unordered_load:
7659; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7660; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7661; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7662; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7663; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7664; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
7665; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7666; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7667; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7668; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7669; GFX940-TGSPLIT-NEXT:    s_endpgm
7670;
7671; GFX11-WGP-LABEL: flat_system_one_as_unordered_load:
7672; GFX11-WGP:       ; %bb.0: ; %entry
7673; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7674; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7675; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7676; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1]
7677; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7678; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7679; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7680; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7681; GFX11-WGP-NEXT:    s_endpgm
7682;
7683; GFX11-CU-LABEL: flat_system_one_as_unordered_load:
7684; GFX11-CU:       ; %bb.0: ; %entry
7685; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7686; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7687; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7688; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
7689; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7690; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7691; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7692; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7693; GFX11-CU-NEXT:    s_endpgm
7694    i32* %in, i32* %out) {
7695entry:
7696  %val = load atomic i32, i32* %in syncscope("one-as") unordered, align 4
7697  store i32 %val, i32* %out
7698  ret void
7699}
7700
7701define amdgpu_kernel void @flat_system_one_as_monotonic_load(
7702; GFX7-LABEL: flat_system_one_as_monotonic_load:
7703; GFX7:       ; %bb.0: ; %entry
7704; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7705; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7706; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7707; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7708; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
7709; GFX7-NEXT:    v_mov_b32_e32 v0, s2
7710; GFX7-NEXT:    v_mov_b32_e32 v1, s3
7711; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7712; GFX7-NEXT:    flat_store_dword v[0:1], v2
7713; GFX7-NEXT:    s_endpgm
7714;
7715; GFX10-WGP-LABEL: flat_system_one_as_monotonic_load:
7716; GFX10-WGP:       ; %bb.0: ; %entry
7717; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7718; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7719; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7720; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7721; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
7722; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
7723; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
7724; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7725; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7726; GFX10-WGP-NEXT:    s_endpgm
7727;
7728; GFX10-CU-LABEL: flat_system_one_as_monotonic_load:
7729; GFX10-CU:       ; %bb.0: ; %entry
7730; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7731; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7732; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7733; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7734; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
7735; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
7736; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
7737; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7738; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7739; GFX10-CU-NEXT:    s_endpgm
7740;
7741; SKIP-CACHE-INV-LABEL: flat_system_one_as_monotonic_load:
7742; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7743; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7744; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7745; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7746; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
7747; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
7748; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7749; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7750; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7751; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7752; SKIP-CACHE-INV-NEXT:    s_endpgm
7753;
7754; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_load:
7755; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7756; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7757; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7758; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7759; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7760; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
7761; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7762; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7763; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7764; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7765; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7766;
7767; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_load:
7768; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7769; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7770; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7771; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7772; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7773; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
7774; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7775; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7776; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7777; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7778; GFX90A-TGSPLIT-NEXT:    s_endpgm
7779;
7780; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_load:
7781; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7782; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7783; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7784; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7785; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7786; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0 sc1
7787; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7788; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7789; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7790; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7791; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7792;
7793; GFX940-TGSPLIT-LABEL: flat_system_one_as_monotonic_load:
7794; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7795; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7796; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7797; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7798; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7799; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0 sc1
7800; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7801; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7802; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7803; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7804; GFX940-TGSPLIT-NEXT:    s_endpgm
7805;
7806; GFX11-WGP-LABEL: flat_system_one_as_monotonic_load:
7807; GFX11-WGP:       ; %bb.0: ; %entry
7808; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7809; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7810; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7811; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc
7812; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7813; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7814; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7815; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7816; GFX11-WGP-NEXT:    s_endpgm
7817;
7818; GFX11-CU-LABEL: flat_system_one_as_monotonic_load:
7819; GFX11-CU:       ; %bb.0: ; %entry
7820; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7821; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7822; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7823; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1] glc
7824; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7825; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7826; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7827; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7828; GFX11-CU-NEXT:    s_endpgm
7829    i32* %in, i32* %out) {
7830entry:
7831  %val = load atomic i32, i32* %in syncscope("one-as") monotonic, align 4
7832  store i32 %val, i32* %out
7833  ret void
7834}
7835
7836define amdgpu_kernel void @flat_system_one_as_acquire_load(
7837; GFX7-LABEL: flat_system_one_as_acquire_load:
7838; GFX7:       ; %bb.0: ; %entry
7839; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7840; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7841; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7842; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7843; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
7844; GFX7-NEXT:    s_waitcnt vmcnt(0)
7845; GFX7-NEXT:    buffer_wbinvl1_vol
7846; GFX7-NEXT:    v_mov_b32_e32 v0, s2
7847; GFX7-NEXT:    v_mov_b32_e32 v1, s3
7848; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7849; GFX7-NEXT:    flat_store_dword v[0:1], v2
7850; GFX7-NEXT:    s_endpgm
7851;
7852; GFX10-WGP-LABEL: flat_system_one_as_acquire_load:
7853; GFX10-WGP:       ; %bb.0: ; %entry
7854; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7855; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7856; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7857; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7858; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
7859; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7860; GFX10-WGP-NEXT:    buffer_gl0_inv
7861; GFX10-WGP-NEXT:    buffer_gl1_inv
7862; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
7863; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
7864; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7865; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7866; GFX10-WGP-NEXT:    s_endpgm
7867;
7868; GFX10-CU-LABEL: flat_system_one_as_acquire_load:
7869; GFX10-CU:       ; %bb.0: ; %entry
7870; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7871; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7872; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7873; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7874; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
7875; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7876; GFX10-CU-NEXT:    buffer_gl0_inv
7877; GFX10-CU-NEXT:    buffer_gl1_inv
7878; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
7879; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
7880; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7881; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7882; GFX10-CU-NEXT:    s_endpgm
7883;
7884; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_load:
7885; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7886; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7887; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7888; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7889; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
7890; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
7891; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7892; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7893; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7894; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7895; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7896; SKIP-CACHE-INV-NEXT:    s_endpgm
7897;
7898; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_load:
7899; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7900; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7901; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7902; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7903; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7904; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
7905; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7906; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
7907; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7908; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7909; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7910; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7911; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7912; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7913;
7914; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_load:
7915; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7916; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7917; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7918; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7919; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7920; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
7921; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7922; GFX90A-TGSPLIT-NEXT:    buffer_invl2
7923; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7924; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7925; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7926; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7927; GFX90A-TGSPLIT-NEXT:    s_endpgm
7928;
7929; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_load:
7930; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7931; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7932; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7933; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7934; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7935; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0 sc1
7936; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7937; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
7938; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7939; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7940; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7941; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7942; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7943;
7944; GFX940-TGSPLIT-LABEL: flat_system_one_as_acquire_load:
7945; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7946; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7947; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7948; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7949; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7950; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0 sc1
7951; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7952; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
7953; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7954; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7955; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7956; GFX940-TGSPLIT-NEXT:    s_endpgm
7957;
7958; GFX11-WGP-LABEL: flat_system_one_as_acquire_load:
7959; GFX11-WGP:       ; %bb.0: ; %entry
7960; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7961; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7962; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7963; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc
7964; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
7965; GFX11-WGP-NEXT:    buffer_gl0_inv
7966; GFX11-WGP-NEXT:    buffer_gl1_inv
7967; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7968; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7969; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7970; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7971; GFX11-WGP-NEXT:    s_endpgm
7972;
7973; GFX11-CU-LABEL: flat_system_one_as_acquire_load:
7974; GFX11-CU:       ; %bb.0: ; %entry
7975; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7976; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7977; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7978; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1] glc
7979; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
7980; GFX11-CU-NEXT:    buffer_gl0_inv
7981; GFX11-CU-NEXT:    buffer_gl1_inv
7982; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7983; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7984; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7985; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7986; GFX11-CU-NEXT:    s_endpgm
7987    i32* %in, i32* %out) {
7988entry:
7989  %val = load atomic i32, i32* %in syncscope("one-as") acquire, align 4
7990  store i32 %val, i32* %out
7991  ret void
7992}
7993
7994define amdgpu_kernel void @flat_system_one_as_seq_cst_load(
7995; GFX7-LABEL: flat_system_one_as_seq_cst_load:
7996; GFX7:       ; %bb.0: ; %entry
7997; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7998; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7999; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8000; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8001; GFX7-NEXT:    s_waitcnt vmcnt(0)
8002; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
8003; GFX7-NEXT:    s_waitcnt vmcnt(0)
8004; GFX7-NEXT:    buffer_wbinvl1_vol
8005; GFX7-NEXT:    v_mov_b32_e32 v0, s2
8006; GFX7-NEXT:    v_mov_b32_e32 v1, s3
8007; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8008; GFX7-NEXT:    flat_store_dword v[0:1], v2
8009; GFX7-NEXT:    s_endpgm
8010;
8011; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_load:
8012; GFX10-WGP:       ; %bb.0: ; %entry
8013; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8014; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8015; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8016; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8017; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8018; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8019; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
8020; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8021; GFX10-WGP-NEXT:    buffer_gl0_inv
8022; GFX10-WGP-NEXT:    buffer_gl1_inv
8023; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
8024; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
8025; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8026; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8027; GFX10-WGP-NEXT:    s_endpgm
8028;
8029; GFX10-CU-LABEL: flat_system_one_as_seq_cst_load:
8030; GFX10-CU:       ; %bb.0: ; %entry
8031; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8032; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8033; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8034; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8035; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8036; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8037; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
8038; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8039; GFX10-CU-NEXT:    buffer_gl0_inv
8040; GFX10-CU-NEXT:    buffer_gl1_inv
8041; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
8042; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
8043; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8044; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8045; GFX10-CU-NEXT:    s_endpgm
8046;
8047; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_load:
8048; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8049; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8050; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8051; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
8052; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
8053; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8054; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
8055; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8056; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8057; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8058; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8059; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8060; SKIP-CACHE-INV-NEXT:    s_endpgm
8061;
8062; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_load:
8063; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8064; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8065; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8066; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8067; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
8068; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8069; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
8070; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8071; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
8072; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
8073; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
8074; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
8075; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8076; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8077; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8078;
8079; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_load:
8080; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8081; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8082; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8083; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8084; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
8085; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8086; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
8087; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8088; GFX90A-TGSPLIT-NEXT:    buffer_invl2
8089; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8090; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
8091; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
8092; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8093; GFX90A-TGSPLIT-NEXT:    s_endpgm
8094;
8095; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_load:
8096; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8097; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8098; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8099; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8100; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
8101; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8102; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0 sc1
8103; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8104; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
8105; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
8106; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
8107; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8108; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8109; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8110;
8111; GFX940-TGSPLIT-LABEL: flat_system_one_as_seq_cst_load:
8112; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8113; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8114; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8115; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8116; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
8117; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8118; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0 sc1
8119; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8120; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
8121; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
8122; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
8123; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8124; GFX940-TGSPLIT-NEXT:    s_endpgm
8125;
8126; GFX11-WGP-LABEL: flat_system_one_as_seq_cst_load:
8127; GFX11-WGP:       ; %bb.0: ; %entry
8128; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8129; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8130; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8131; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8132; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8133; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc
8134; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8135; GFX11-WGP-NEXT:    buffer_gl0_inv
8136; GFX11-WGP-NEXT:    buffer_gl1_inv
8137; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8138; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8139; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
8140; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8141; GFX11-WGP-NEXT:    s_endpgm
8142;
8143; GFX11-CU-LABEL: flat_system_one_as_seq_cst_load:
8144; GFX11-CU:       ; %bb.0: ; %entry
8145; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8146; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8147; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8148; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
8149; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8150; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1] glc
8151; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
8152; GFX11-CU-NEXT:    buffer_gl0_inv
8153; GFX11-CU-NEXT:    buffer_gl1_inv
8154; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8155; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8156; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8157; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8158; GFX11-CU-NEXT:    s_endpgm
8159    i32* %in, i32* %out) {
8160entry:
8161  %val = load atomic i32, i32* %in syncscope("one-as") seq_cst, align 4
8162  store i32 %val, i32* %out
8163  ret void
8164}
8165
8166define amdgpu_kernel void @flat_system_one_as_unordered_store(
8167; GFX7-LABEL: flat_system_one_as_unordered_store:
8168; GFX7:       ; %bb.0: ; %entry
8169; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
8170; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
8171; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8172; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8173; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8174; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8175; GFX7-NEXT:    flat_store_dword v[0:1], v2
8176; GFX7-NEXT:    s_endpgm
8177;
8178; GFX10-WGP-LABEL: flat_system_one_as_unordered_store:
8179; GFX10-WGP:       ; %bb.0: ; %entry
8180; GFX10-WGP-NEXT:    s_clause 0x1
8181; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8182; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
8183; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8184; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8185; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8186; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8187; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8188; GFX10-WGP-NEXT:    s_endpgm
8189;
8190; GFX10-CU-LABEL: flat_system_one_as_unordered_store:
8191; GFX10-CU:       ; %bb.0: ; %entry
8192; GFX10-CU-NEXT:    s_clause 0x1
8193; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8194; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
8195; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8196; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8197; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8198; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8199; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8200; GFX10-CU-NEXT:    s_endpgm
8201;
8202; SKIP-CACHE-INV-LABEL: flat_system_one_as_unordered_store:
8203; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8204; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
8205; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
8206; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8207; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8208; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8209; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8210; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8211; SKIP-CACHE-INV-NEXT:    s_endpgm
8212;
8213; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_unordered_store:
8214; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8215; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8216; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
8217; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8218; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8219; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8220; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8221; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8222;
8223; GFX90A-TGSPLIT-LABEL: flat_system_one_as_unordered_store:
8224; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8225; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8226; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
8227; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8228; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8229; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8230; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8231; GFX90A-TGSPLIT-NEXT:    s_endpgm
8232;
8233; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_unordered_store:
8234; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8235; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8236; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
8237; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8238; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8239; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8240; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8241; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8242;
8243; GFX940-TGSPLIT-LABEL: flat_system_one_as_unordered_store:
8244; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8245; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8246; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
8247; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8248; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8249; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8250; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8251; GFX940-TGSPLIT-NEXT:    s_endpgm
8252;
8253; GFX11-WGP-LABEL: flat_system_one_as_unordered_store:
8254; GFX11-WGP:       ; %bb.0: ; %entry
8255; GFX11-WGP-NEXT:    s_clause 0x1
8256; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8257; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
8258; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8259; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8260; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8261; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
8262; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8263; GFX11-WGP-NEXT:    s_endpgm
8264;
8265; GFX11-CU-LABEL: flat_system_one_as_unordered_store:
8266; GFX11-CU:       ; %bb.0: ; %entry
8267; GFX11-CU-NEXT:    s_clause 0x1
8268; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8269; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
8270; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8271; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8272; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8273; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8274; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8275; GFX11-CU-NEXT:    s_endpgm
8276    i32 %in, i32* %out) {
8277entry:
8278  store atomic i32 %in, i32* %out syncscope("one-as") unordered, align 4
8279  ret void
8280}
8281
8282define amdgpu_kernel void @flat_system_one_as_monotonic_store(
8283; GFX7-LABEL: flat_system_one_as_monotonic_store:
8284; GFX7:       ; %bb.0: ; %entry
8285; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
8286; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
8287; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8288; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8289; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8290; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8291; GFX7-NEXT:    flat_store_dword v[0:1], v2
8292; GFX7-NEXT:    s_endpgm
8293;
8294; GFX10-WGP-LABEL: flat_system_one_as_monotonic_store:
8295; GFX10-WGP:       ; %bb.0: ; %entry
8296; GFX10-WGP-NEXT:    s_clause 0x1
8297; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8298; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
8299; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8300; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8301; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8302; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8303; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8304; GFX10-WGP-NEXT:    s_endpgm
8305;
8306; GFX10-CU-LABEL: flat_system_one_as_monotonic_store:
8307; GFX10-CU:       ; %bb.0: ; %entry
8308; GFX10-CU-NEXT:    s_clause 0x1
8309; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8310; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
8311; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8312; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8313; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8314; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8315; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8316; GFX10-CU-NEXT:    s_endpgm
8317;
8318; SKIP-CACHE-INV-LABEL: flat_system_one_as_monotonic_store:
8319; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8320; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
8321; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
8322; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8323; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8324; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8325; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8326; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8327; SKIP-CACHE-INV-NEXT:    s_endpgm
8328;
8329; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_store:
8330; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8331; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8332; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
8333; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8334; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8335; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8336; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8337; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8338;
8339; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_store:
8340; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8341; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8342; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
8343; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8344; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8345; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8346; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8347; GFX90A-TGSPLIT-NEXT:    s_endpgm
8348;
8349; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_store:
8350; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8351; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8352; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
8353; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8354; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8355; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8356; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
8357; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8358;
8359; GFX940-TGSPLIT-LABEL: flat_system_one_as_monotonic_store:
8360; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8361; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8362; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
8363; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8364; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8365; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8366; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
8367; GFX940-TGSPLIT-NEXT:    s_endpgm
8368;
8369; GFX11-WGP-LABEL: flat_system_one_as_monotonic_store:
8370; GFX11-WGP:       ; %bb.0: ; %entry
8371; GFX11-WGP-NEXT:    s_clause 0x1
8372; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8373; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
8374; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8375; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8376; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8377; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
8378; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8379; GFX11-WGP-NEXT:    s_endpgm
8380;
8381; GFX11-CU-LABEL: flat_system_one_as_monotonic_store:
8382; GFX11-CU:       ; %bb.0: ; %entry
8383; GFX11-CU-NEXT:    s_clause 0x1
8384; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8385; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
8386; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8387; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8388; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8389; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8390; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8391; GFX11-CU-NEXT:    s_endpgm
8392    i32 %in, i32* %out) {
8393entry:
8394  store atomic i32 %in, i32* %out syncscope("one-as") monotonic, align 4
8395  ret void
8396}
8397
8398define amdgpu_kernel void @flat_system_one_as_release_store(
8399; GFX7-LABEL: flat_system_one_as_release_store:
8400; GFX7:       ; %bb.0: ; %entry
8401; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
8402; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
8403; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8404; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8405; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8406; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8407; GFX7-NEXT:    s_waitcnt vmcnt(0)
8408; GFX7-NEXT:    flat_store_dword v[0:1], v2
8409; GFX7-NEXT:    s_endpgm
8410;
8411; GFX10-WGP-LABEL: flat_system_one_as_release_store:
8412; GFX10-WGP:       ; %bb.0: ; %entry
8413; GFX10-WGP-NEXT:    s_clause 0x1
8414; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8415; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
8416; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8417; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8418; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8419; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8420; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8421; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8422; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8423; GFX10-WGP-NEXT:    s_endpgm
8424;
8425; GFX10-CU-LABEL: flat_system_one_as_release_store:
8426; GFX10-CU:       ; %bb.0: ; %entry
8427; GFX10-CU-NEXT:    s_clause 0x1
8428; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8429; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
8430; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8431; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8432; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8433; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8434; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8435; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8436; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8437; GFX10-CU-NEXT:    s_endpgm
8438;
8439; SKIP-CACHE-INV-LABEL: flat_system_one_as_release_store:
8440; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8441; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
8442; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
8443; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8444; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8445; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8446; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8447; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8448; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8449; SKIP-CACHE-INV-NEXT:    s_endpgm
8450;
8451; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_release_store:
8452; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8453; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8454; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
8455; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8456; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8457; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8458; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
8459; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8460; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8461; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8462;
8463; GFX90A-TGSPLIT-LABEL: flat_system_one_as_release_store:
8464; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8465; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8466; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
8467; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8468; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8469; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8470; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
8471; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8472; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8473; GFX90A-TGSPLIT-NEXT:    s_endpgm
8474;
8475; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_release_store:
8476; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8477; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8478; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
8479; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8480; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8481; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8482; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
8483; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8484; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
8485; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8486;
8487; GFX940-TGSPLIT-LABEL: flat_system_one_as_release_store:
8488; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8489; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8490; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
8491; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8492; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8493; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8494; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
8495; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8496; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
8497; GFX940-TGSPLIT-NEXT:    s_endpgm
8498;
8499; GFX11-WGP-LABEL: flat_system_one_as_release_store:
8500; GFX11-WGP:       ; %bb.0: ; %entry
8501; GFX11-WGP-NEXT:    s_clause 0x1
8502; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8503; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
8504; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8505; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8506; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8507; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8508; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8509; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
8510; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8511; GFX11-WGP-NEXT:    s_endpgm
8512;
8513; GFX11-CU-LABEL: flat_system_one_as_release_store:
8514; GFX11-CU:       ; %bb.0: ; %entry
8515; GFX11-CU-NEXT:    s_clause 0x1
8516; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8517; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
8518; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8519; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8520; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8521; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
8522; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8523; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8524; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8525; GFX11-CU-NEXT:    s_endpgm
8526    i32 %in, i32* %out) {
8527entry:
8528  store atomic i32 %in, i32* %out syncscope("one-as") release, align 4
8529  ret void
8530}
8531
8532define amdgpu_kernel void @flat_system_one_as_seq_cst_store(
8533; GFX7-LABEL: flat_system_one_as_seq_cst_store:
8534; GFX7:       ; %bb.0: ; %entry
8535; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
8536; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
8537; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8538; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8539; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8540; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8541; GFX7-NEXT:    s_waitcnt vmcnt(0)
8542; GFX7-NEXT:    flat_store_dword v[0:1], v2
8543; GFX7-NEXT:    s_endpgm
8544;
8545; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_store:
8546; GFX10-WGP:       ; %bb.0: ; %entry
8547; GFX10-WGP-NEXT:    s_clause 0x1
8548; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8549; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
8550; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8551; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8552; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8553; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8554; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8555; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8556; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8557; GFX10-WGP-NEXT:    s_endpgm
8558;
8559; GFX10-CU-LABEL: flat_system_one_as_seq_cst_store:
8560; GFX10-CU:       ; %bb.0: ; %entry
8561; GFX10-CU-NEXT:    s_clause 0x1
8562; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8563; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
8564; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8565; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8566; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8567; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8568; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8569; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8570; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8571; GFX10-CU-NEXT:    s_endpgm
8572;
8573; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_store:
8574; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8575; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
8576; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
8577; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8578; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8579; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8580; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8581; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8582; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8583; SKIP-CACHE-INV-NEXT:    s_endpgm
8584;
8585; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_store:
8586; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8587; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8588; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
8589; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8590; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8591; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8592; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
8593; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8594; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8595; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8596;
8597; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_store:
8598; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8599; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8600; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
8601; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8602; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8603; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8604; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
8605; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8606; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8607; GFX90A-TGSPLIT-NEXT:    s_endpgm
8608;
8609; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_store:
8610; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8611; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8612; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
8613; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8614; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8615; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8616; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
8617; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8618; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
8619; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8620;
8621; GFX940-TGSPLIT-LABEL: flat_system_one_as_seq_cst_store:
8622; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8623; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8624; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
8625; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8626; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8627; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8628; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
8629; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8630; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
8631; GFX940-TGSPLIT-NEXT:    s_endpgm
8632;
8633; GFX11-WGP-LABEL: flat_system_one_as_seq_cst_store:
8634; GFX11-WGP:       ; %bb.0: ; %entry
8635; GFX11-WGP-NEXT:    s_clause 0x1
8636; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8637; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
8638; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8639; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8640; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8641; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8642; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8643; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
8644; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8645; GFX11-WGP-NEXT:    s_endpgm
8646;
8647; GFX11-CU-LABEL: flat_system_one_as_seq_cst_store:
8648; GFX11-CU:       ; %bb.0: ; %entry
8649; GFX11-CU-NEXT:    s_clause 0x1
8650; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8651; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
8652; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8653; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8654; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8655; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
8656; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8657; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8658; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8659; GFX11-CU-NEXT:    s_endpgm
8660    i32 %in, i32* %out) {
8661entry:
8662  store atomic i32 %in, i32* %out syncscope("one-as") seq_cst, align 4
8663  ret void
8664}
8665
8666define amdgpu_kernel void @flat_system_one_as_monotonic_atomicrmw(
8667; GFX7-LABEL: flat_system_one_as_monotonic_atomicrmw:
8668; GFX7:       ; %bb.0: ; %entry
8669; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8670; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
8671; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8672; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8673; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8674; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8675; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
8676; GFX7-NEXT:    s_endpgm
8677;
8678; GFX10-WGP-LABEL: flat_system_one_as_monotonic_atomicrmw:
8679; GFX10-WGP:       ; %bb.0: ; %entry
8680; GFX10-WGP-NEXT:    s_clause 0x1
8681; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8682; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
8683; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8684; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8685; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8686; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8687; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
8688; GFX10-WGP-NEXT:    s_endpgm
8689;
8690; GFX10-CU-LABEL: flat_system_one_as_monotonic_atomicrmw:
8691; GFX10-CU:       ; %bb.0: ; %entry
8692; GFX10-CU-NEXT:    s_clause 0x1
8693; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8694; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8695; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8696; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8697; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8698; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8699; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
8700; GFX10-CU-NEXT:    s_endpgm
8701;
8702; SKIP-CACHE-INV-LABEL: flat_system_one_as_monotonic_atomicrmw:
8703; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8704; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8705; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
8706; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8707; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8708; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8709; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8710; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
8711; SKIP-CACHE-INV-NEXT:    s_endpgm
8712;
8713; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_atomicrmw:
8714; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8715; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8716; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8717; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8718; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8719; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8720; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8721; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8722;
8723; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_atomicrmw:
8724; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8725; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8726; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8727; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8728; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8729; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8730; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8731; GFX90A-TGSPLIT-NEXT:    s_endpgm
8732;
8733; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_atomicrmw:
8734; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8735; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8736; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8737; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8738; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8739; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8740; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
8741; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8742;
8743; GFX940-TGSPLIT-LABEL: flat_system_one_as_monotonic_atomicrmw:
8744; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8745; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8746; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8747; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8748; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8749; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8750; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
8751; GFX940-TGSPLIT-NEXT:    s_endpgm
8752;
8753; GFX11-WGP-LABEL: flat_system_one_as_monotonic_atomicrmw:
8754; GFX11-WGP:       ; %bb.0: ; %entry
8755; GFX11-WGP-NEXT:    s_clause 0x1
8756; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8757; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
8758; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8759; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8760; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8761; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
8762; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8763; GFX11-WGP-NEXT:    s_endpgm
8764;
8765; GFX11-CU-LABEL: flat_system_one_as_monotonic_atomicrmw:
8766; GFX11-CU:       ; %bb.0: ; %entry
8767; GFX11-CU-NEXT:    s_clause 0x1
8768; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8769; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
8770; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8771; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8772; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8773; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
8774; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8775; GFX11-CU-NEXT:    s_endpgm
8776    i32* %out, i32 %in) {
8777entry:
8778  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") monotonic
8779  ret void
8780}
8781
8782define amdgpu_kernel void @flat_system_one_as_acquire_atomicrmw(
8783; GFX7-LABEL: flat_system_one_as_acquire_atomicrmw:
8784; GFX7:       ; %bb.0: ; %entry
8785; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8786; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
8787; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8788; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8789; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8790; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8791; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
8792; GFX7-NEXT:    s_waitcnt vmcnt(0)
8793; GFX7-NEXT:    buffer_wbinvl1_vol
8794; GFX7-NEXT:    s_endpgm
8795;
8796; GFX10-WGP-LABEL: flat_system_one_as_acquire_atomicrmw:
8797; GFX10-WGP:       ; %bb.0: ; %entry
8798; GFX10-WGP-NEXT:    s_clause 0x1
8799; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8800; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
8801; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8802; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8803; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8804; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8805; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
8806; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8807; GFX10-WGP-NEXT:    buffer_gl0_inv
8808; GFX10-WGP-NEXT:    buffer_gl1_inv
8809; GFX10-WGP-NEXT:    s_endpgm
8810;
8811; GFX10-CU-LABEL: flat_system_one_as_acquire_atomicrmw:
8812; GFX10-CU:       ; %bb.0: ; %entry
8813; GFX10-CU-NEXT:    s_clause 0x1
8814; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8815; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8816; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8817; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8818; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8819; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8820; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
8821; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8822; GFX10-CU-NEXT:    buffer_gl0_inv
8823; GFX10-CU-NEXT:    buffer_gl1_inv
8824; GFX10-CU-NEXT:    s_endpgm
8825;
8826; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_atomicrmw:
8827; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8828; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8829; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
8830; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8831; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8832; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8833; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8834; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
8835; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8836; SKIP-CACHE-INV-NEXT:    s_endpgm
8837;
8838; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_atomicrmw:
8839; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8840; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8841; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8842; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8843; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8844; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8845; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8846; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8847; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
8848; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
8849; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8850;
8851; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_atomicrmw:
8852; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8853; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8854; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8855; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8856; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8857; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8858; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8859; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8860; GFX90A-TGSPLIT-NEXT:    buffer_invl2
8861; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8862; GFX90A-TGSPLIT-NEXT:    s_endpgm
8863;
8864; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_atomicrmw:
8865; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8866; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8867; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8868; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8869; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8870; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8871; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
8872; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8873; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
8874; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8875;
8876; GFX940-TGSPLIT-LABEL: flat_system_one_as_acquire_atomicrmw:
8877; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8878; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8879; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8880; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8881; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8882; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8883; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
8884; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8885; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
8886; GFX940-TGSPLIT-NEXT:    s_endpgm
8887;
8888; GFX11-WGP-LABEL: flat_system_one_as_acquire_atomicrmw:
8889; GFX11-WGP:       ; %bb.0: ; %entry
8890; GFX11-WGP-NEXT:    s_clause 0x1
8891; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8892; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
8893; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8894; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8895; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8896; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
8897; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8898; GFX11-WGP-NEXT:    buffer_gl0_inv
8899; GFX11-WGP-NEXT:    buffer_gl1_inv
8900; GFX11-WGP-NEXT:    s_endpgm
8901;
8902; GFX11-CU-LABEL: flat_system_one_as_acquire_atomicrmw:
8903; GFX11-CU:       ; %bb.0: ; %entry
8904; GFX11-CU-NEXT:    s_clause 0x1
8905; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8906; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
8907; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8908; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8909; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8910; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
8911; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8912; GFX11-CU-NEXT:    buffer_gl0_inv
8913; GFX11-CU-NEXT:    buffer_gl1_inv
8914; GFX11-CU-NEXT:    s_endpgm
8915    i32* %out, i32 %in) {
8916entry:
8917  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") acquire
8918  ret void
8919}
8920
8921define amdgpu_kernel void @flat_system_one_as_release_atomicrmw(
8922; GFX7-LABEL: flat_system_one_as_release_atomicrmw:
8923; GFX7:       ; %bb.0: ; %entry
8924; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8925; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
8926; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8927; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8928; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8929; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8930; GFX7-NEXT:    s_waitcnt vmcnt(0)
8931; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
8932; GFX7-NEXT:    s_endpgm
8933;
8934; GFX10-WGP-LABEL: flat_system_one_as_release_atomicrmw:
8935; GFX10-WGP:       ; %bb.0: ; %entry
8936; GFX10-WGP-NEXT:    s_clause 0x1
8937; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8938; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
8939; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8940; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8941; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8942; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8943; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8944; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8945; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
8946; GFX10-WGP-NEXT:    s_endpgm
8947;
8948; GFX10-CU-LABEL: flat_system_one_as_release_atomicrmw:
8949; GFX10-CU:       ; %bb.0: ; %entry
8950; GFX10-CU-NEXT:    s_clause 0x1
8951; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8952; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8953; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8954; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8955; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8956; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8957; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8958; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8959; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
8960; GFX10-CU-NEXT:    s_endpgm
8961;
8962; SKIP-CACHE-INV-LABEL: flat_system_one_as_release_atomicrmw:
8963; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8964; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8965; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
8966; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8967; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8968; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8969; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8970; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8971; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
8972; SKIP-CACHE-INV-NEXT:    s_endpgm
8973;
8974; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_release_atomicrmw:
8975; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8976; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8977; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8978; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8979; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8980; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8981; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
8982; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8983; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8984; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8985;
8986; GFX90A-TGSPLIT-LABEL: flat_system_one_as_release_atomicrmw:
8987; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8988; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8989; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8990; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8991; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8992; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8993; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
8994; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8995; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8996; GFX90A-TGSPLIT-NEXT:    s_endpgm
8997;
8998; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_release_atomicrmw:
8999; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9000; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9001; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9002; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9003; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9004; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9005; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
9006; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9007; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
9008; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9009;
9010; GFX940-TGSPLIT-LABEL: flat_system_one_as_release_atomicrmw:
9011; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9012; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9013; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9014; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9015; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9016; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9017; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
9018; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9019; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
9020; GFX940-TGSPLIT-NEXT:    s_endpgm
9021;
9022; GFX11-WGP-LABEL: flat_system_one_as_release_atomicrmw:
9023; GFX11-WGP:       ; %bb.0: ; %entry
9024; GFX11-WGP-NEXT:    s_clause 0x1
9025; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9026; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
9027; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9028; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9029; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9030; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9031; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9032; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
9033; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9034; GFX11-WGP-NEXT:    s_endpgm
9035;
9036; GFX11-CU-LABEL: flat_system_one_as_release_atomicrmw:
9037; GFX11-CU:       ; %bb.0: ; %entry
9038; GFX11-CU-NEXT:    s_clause 0x1
9039; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9040; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
9041; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9042; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9043; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9044; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9045; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9046; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
9047; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9048; GFX11-CU-NEXT:    s_endpgm
9049    i32* %out, i32 %in) {
9050entry:
9051  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") release
9052  ret void
9053}
9054
9055define amdgpu_kernel void @flat_system_one_as_acq_rel_atomicrmw(
9056; GFX7-LABEL: flat_system_one_as_acq_rel_atomicrmw:
9057; GFX7:       ; %bb.0: ; %entry
9058; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9059; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
9060; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9061; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9062; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9063; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9064; GFX7-NEXT:    s_waitcnt vmcnt(0)
9065; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
9066; GFX7-NEXT:    s_waitcnt vmcnt(0)
9067; GFX7-NEXT:    buffer_wbinvl1_vol
9068; GFX7-NEXT:    s_endpgm
9069;
9070; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_atomicrmw:
9071; GFX10-WGP:       ; %bb.0: ; %entry
9072; GFX10-WGP-NEXT:    s_clause 0x1
9073; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9074; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
9075; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9076; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9077; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9078; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9079; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9080; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9081; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
9082; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9083; GFX10-WGP-NEXT:    buffer_gl0_inv
9084; GFX10-WGP-NEXT:    buffer_gl1_inv
9085; GFX10-WGP-NEXT:    s_endpgm
9086;
9087; GFX10-CU-LABEL: flat_system_one_as_acq_rel_atomicrmw:
9088; GFX10-CU:       ; %bb.0: ; %entry
9089; GFX10-CU-NEXT:    s_clause 0x1
9090; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9091; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
9092; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9093; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9094; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9095; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9096; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9097; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9098; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
9099; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9100; GFX10-CU-NEXT:    buffer_gl0_inv
9101; GFX10-CU-NEXT:    buffer_gl1_inv
9102; GFX10-CU-NEXT:    s_endpgm
9103;
9104; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_atomicrmw:
9105; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9106; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9107; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
9108; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9109; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9110; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9111; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9112; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9113; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
9114; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9115; SKIP-CACHE-INV-NEXT:    s_endpgm
9116;
9117; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_atomicrmw:
9118; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9119; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9120; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9121; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9122; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9123; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9124; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
9125; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9126; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
9127; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9128; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
9129; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9130; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9131;
9132; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_atomicrmw:
9133; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9134; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9135; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9136; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9137; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9138; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9139; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
9140; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9141; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
9142; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9143; GFX90A-TGSPLIT-NEXT:    buffer_invl2
9144; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9145; GFX90A-TGSPLIT-NEXT:    s_endpgm
9146;
9147; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_atomicrmw:
9148; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9149; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9150; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9151; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9152; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9153; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9154; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
9155; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9156; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
9157; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9158; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
9159; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9160;
9161; GFX940-TGSPLIT-LABEL: flat_system_one_as_acq_rel_atomicrmw:
9162; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9163; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9164; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9165; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9166; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9167; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9168; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
9169; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9170; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
9171; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9172; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
9173; GFX940-TGSPLIT-NEXT:    s_endpgm
9174;
9175; GFX11-WGP-LABEL: flat_system_one_as_acq_rel_atomicrmw:
9176; GFX11-WGP:       ; %bb.0: ; %entry
9177; GFX11-WGP-NEXT:    s_clause 0x1
9178; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9179; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
9180; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9181; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9182; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9183; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9184; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9185; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
9186; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9187; GFX11-WGP-NEXT:    buffer_gl0_inv
9188; GFX11-WGP-NEXT:    buffer_gl1_inv
9189; GFX11-WGP-NEXT:    s_endpgm
9190;
9191; GFX11-CU-LABEL: flat_system_one_as_acq_rel_atomicrmw:
9192; GFX11-CU:       ; %bb.0: ; %entry
9193; GFX11-CU-NEXT:    s_clause 0x1
9194; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9195; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
9196; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9197; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9198; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9199; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9200; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9201; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
9202; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9203; GFX11-CU-NEXT:    buffer_gl0_inv
9204; GFX11-CU-NEXT:    buffer_gl1_inv
9205; GFX11-CU-NEXT:    s_endpgm
9206    i32* %out, i32 %in) {
9207entry:
9208  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") acq_rel
9209  ret void
9210}
9211
9212define amdgpu_kernel void @flat_system_one_as_seq_cst_atomicrmw(
9213; GFX7-LABEL: flat_system_one_as_seq_cst_atomicrmw:
9214; GFX7:       ; %bb.0: ; %entry
9215; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9216; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
9217; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9218; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9219; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9220; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9221; GFX7-NEXT:    s_waitcnt vmcnt(0)
9222; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
9223; GFX7-NEXT:    s_waitcnt vmcnt(0)
9224; GFX7-NEXT:    buffer_wbinvl1_vol
9225; GFX7-NEXT:    s_endpgm
9226;
9227; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_atomicrmw:
9228; GFX10-WGP:       ; %bb.0: ; %entry
9229; GFX10-WGP-NEXT:    s_clause 0x1
9230; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9231; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
9232; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9233; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9234; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9235; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9236; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9237; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9238; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
9239; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9240; GFX10-WGP-NEXT:    buffer_gl0_inv
9241; GFX10-WGP-NEXT:    buffer_gl1_inv
9242; GFX10-WGP-NEXT:    s_endpgm
9243;
9244; GFX10-CU-LABEL: flat_system_one_as_seq_cst_atomicrmw:
9245; GFX10-CU:       ; %bb.0: ; %entry
9246; GFX10-CU-NEXT:    s_clause 0x1
9247; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9248; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
9249; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9250; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9251; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9252; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9253; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9254; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9255; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
9256; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9257; GFX10-CU-NEXT:    buffer_gl0_inv
9258; GFX10-CU-NEXT:    buffer_gl1_inv
9259; GFX10-CU-NEXT:    s_endpgm
9260;
9261; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_atomicrmw:
9262; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9263; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9264; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
9265; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9266; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9267; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9268; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9269; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9270; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
9271; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9272; SKIP-CACHE-INV-NEXT:    s_endpgm
9273;
9274; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_atomicrmw:
9275; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9276; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9277; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9278; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9279; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9280; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9281; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
9282; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9283; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
9284; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9285; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
9286; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9287; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9288;
9289; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_atomicrmw:
9290; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9291; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9292; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9293; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9294; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9295; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9296; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
9297; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9298; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
9299; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9300; GFX90A-TGSPLIT-NEXT:    buffer_invl2
9301; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9302; GFX90A-TGSPLIT-NEXT:    s_endpgm
9303;
9304; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_atomicrmw:
9305; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9306; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9307; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9308; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9309; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9310; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9311; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
9312; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9313; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
9314; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9315; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
9316; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9317;
9318; GFX940-TGSPLIT-LABEL: flat_system_one_as_seq_cst_atomicrmw:
9319; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9320; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9321; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9322; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9323; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9324; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9325; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
9326; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9327; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
9328; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9329; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
9330; GFX940-TGSPLIT-NEXT:    s_endpgm
9331;
9332; GFX11-WGP-LABEL: flat_system_one_as_seq_cst_atomicrmw:
9333; GFX11-WGP:       ; %bb.0: ; %entry
9334; GFX11-WGP-NEXT:    s_clause 0x1
9335; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9336; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
9337; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9338; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9339; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9340; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9341; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9342; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
9343; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9344; GFX11-WGP-NEXT:    buffer_gl0_inv
9345; GFX11-WGP-NEXT:    buffer_gl1_inv
9346; GFX11-WGP-NEXT:    s_endpgm
9347;
9348; GFX11-CU-LABEL: flat_system_one_as_seq_cst_atomicrmw:
9349; GFX11-CU:       ; %bb.0: ; %entry
9350; GFX11-CU-NEXT:    s_clause 0x1
9351; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9352; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
9353; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9354; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9355; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9356; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9357; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9358; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
9359; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9360; GFX11-CU-NEXT:    buffer_gl0_inv
9361; GFX11-CU-NEXT:    buffer_gl1_inv
9362; GFX11-CU-NEXT:    s_endpgm
9363    i32* %out, i32 %in) {
9364entry:
9365  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") seq_cst
9366  ret void
9367}
9368
9369define amdgpu_kernel void @flat_system_one_as_acquire_ret_atomicrmw(
9370; GFX7-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
9371; GFX7:       ; %bb.0: ; %entry
9372; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9373; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
9374; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9375; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9376; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9377; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9378; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9379; GFX7-NEXT:    s_waitcnt vmcnt(0)
9380; GFX7-NEXT:    buffer_wbinvl1_vol
9381; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9382; GFX7-NEXT:    flat_store_dword v[0:1], v2
9383; GFX7-NEXT:    s_endpgm
9384;
9385; GFX10-WGP-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
9386; GFX10-WGP:       ; %bb.0: ; %entry
9387; GFX10-WGP-NEXT:    s_clause 0x1
9388; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9389; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
9390; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9391; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9392; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9393; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9394; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9395; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9396; GFX10-WGP-NEXT:    buffer_gl0_inv
9397; GFX10-WGP-NEXT:    buffer_gl1_inv
9398; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9399; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
9400; GFX10-WGP-NEXT:    s_endpgm
9401;
9402; GFX10-CU-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
9403; GFX10-CU:       ; %bb.0: ; %entry
9404; GFX10-CU-NEXT:    s_clause 0x1
9405; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9406; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
9407; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9408; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9409; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9410; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9411; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9412; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9413; GFX10-CU-NEXT:    buffer_gl0_inv
9414; GFX10-CU-NEXT:    buffer_gl1_inv
9415; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9416; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
9417; GFX10-CU-NEXT:    s_endpgm
9418;
9419; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
9420; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9421; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9422; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
9423; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9424; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9425; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9426; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9427; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9428; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9429; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
9430; SKIP-CACHE-INV-NEXT:    s_endpgm
9431;
9432; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
9433; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9434; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9435; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9436; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9437; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9438; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9439; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9440; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9441; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
9442; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9443; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9444; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9445; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9446;
9447; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
9448; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9449; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9450; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9451; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9452; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9453; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9454; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9455; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9456; GFX90A-TGSPLIT-NEXT:    buffer_invl2
9457; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9458; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9459; GFX90A-TGSPLIT-NEXT:    s_endpgm
9460;
9461; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
9462; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9463; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9464; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9465; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9466; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9467; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9468; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0 sc1
9469; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9470; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
9471; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9472; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9473; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9474;
9475; GFX940-TGSPLIT-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
9476; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9477; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9478; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9479; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9480; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9481; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9482; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0 sc1
9483; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9484; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
9485; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9486; GFX940-TGSPLIT-NEXT:    s_endpgm
9487;
9488; GFX11-WGP-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
9489; GFX11-WGP:       ; %bb.0: ; %entry
9490; GFX11-WGP-NEXT:    s_clause 0x1
9491; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9492; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
9493; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9494; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9495; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9496; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
9497; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9498; GFX11-WGP-NEXT:    buffer_gl0_inv
9499; GFX11-WGP-NEXT:    buffer_gl1_inv
9500; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9501; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
9502; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9503; GFX11-WGP-NEXT:    s_endpgm
9504;
9505; GFX11-CU-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
9506; GFX11-CU:       ; %bb.0: ; %entry
9507; GFX11-CU-NEXT:    s_clause 0x1
9508; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9509; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
9510; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9511; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9512; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9513; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
9514; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9515; GFX11-CU-NEXT:    buffer_gl0_inv
9516; GFX11-CU-NEXT:    buffer_gl1_inv
9517; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9518; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
9519; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9520; GFX11-CU-NEXT:    s_endpgm
9521    i32* %out, i32 %in) {
9522entry:
9523  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") acquire
9524  store i32 %val, i32* %out, align 4
9525  ret void
9526}
9527
9528define amdgpu_kernel void @flat_system_one_as_acq_rel_ret_atomicrmw(
9529; GFX7-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
9530; GFX7:       ; %bb.0: ; %entry
9531; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9532; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
9533; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9534; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9535; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9536; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9537; GFX7-NEXT:    s_waitcnt vmcnt(0)
9538; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9539; GFX7-NEXT:    s_waitcnt vmcnt(0)
9540; GFX7-NEXT:    buffer_wbinvl1_vol
9541; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9542; GFX7-NEXT:    flat_store_dword v[0:1], v2
9543; GFX7-NEXT:    s_endpgm
9544;
9545; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
9546; GFX10-WGP:       ; %bb.0: ; %entry
9547; GFX10-WGP-NEXT:    s_clause 0x1
9548; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9549; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
9550; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9551; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9552; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9553; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9554; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9555; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9556; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9557; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9558; GFX10-WGP-NEXT:    buffer_gl0_inv
9559; GFX10-WGP-NEXT:    buffer_gl1_inv
9560; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9561; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
9562; GFX10-WGP-NEXT:    s_endpgm
9563;
9564; GFX10-CU-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
9565; GFX10-CU:       ; %bb.0: ; %entry
9566; GFX10-CU-NEXT:    s_clause 0x1
9567; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9568; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
9569; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9570; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9571; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9572; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9573; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9574; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9575; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9576; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9577; GFX10-CU-NEXT:    buffer_gl0_inv
9578; GFX10-CU-NEXT:    buffer_gl1_inv
9579; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9580; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
9581; GFX10-CU-NEXT:    s_endpgm
9582;
9583; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
9584; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9585; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9586; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
9587; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9588; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9589; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9590; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9591; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9592; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9593; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9594; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
9595; SKIP-CACHE-INV-NEXT:    s_endpgm
9596;
9597; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
9598; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9599; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9600; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9601; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9602; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9603; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9604; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
9605; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9606; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9607; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9608; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
9609; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9610; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9611; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9612; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9613;
9614; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
9615; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9616; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9617; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9618; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9619; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9620; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9621; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
9622; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9623; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9624; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9625; GFX90A-TGSPLIT-NEXT:    buffer_invl2
9626; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9627; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9628; GFX90A-TGSPLIT-NEXT:    s_endpgm
9629;
9630; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
9631; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9632; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9633; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9634; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9635; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9636; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9637; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
9638; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9639; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0 sc1
9640; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9641; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
9642; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9643; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9644; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9645;
9646; GFX940-TGSPLIT-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
9647; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9648; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9649; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9650; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9651; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9652; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9653; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
9654; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9655; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0 sc1
9656; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9657; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
9658; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9659; GFX940-TGSPLIT-NEXT:    s_endpgm
9660;
9661; GFX11-WGP-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
9662; GFX11-WGP:       ; %bb.0: ; %entry
9663; GFX11-WGP-NEXT:    s_clause 0x1
9664; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9665; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
9666; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9667; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9668; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9669; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9670; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9671; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
9672; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9673; GFX11-WGP-NEXT:    buffer_gl0_inv
9674; GFX11-WGP-NEXT:    buffer_gl1_inv
9675; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9676; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
9677; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9678; GFX11-WGP-NEXT:    s_endpgm
9679;
9680; GFX11-CU-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
9681; GFX11-CU:       ; %bb.0: ; %entry
9682; GFX11-CU-NEXT:    s_clause 0x1
9683; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9684; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
9685; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9686; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9687; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9688; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9689; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9690; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
9691; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9692; GFX11-CU-NEXT:    buffer_gl0_inv
9693; GFX11-CU-NEXT:    buffer_gl1_inv
9694; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9695; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
9696; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9697; GFX11-CU-NEXT:    s_endpgm
9698    i32* %out, i32 %in) {
9699entry:
9700  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") acq_rel
9701  store i32 %val, i32* %out, align 4
9702  ret void
9703}
9704
9705define amdgpu_kernel void @flat_system_one_as_seq_cst_ret_atomicrmw(
9706; GFX7-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
9707; GFX7:       ; %bb.0: ; %entry
9708; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9709; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
9710; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9711; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9712; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9713; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9714; GFX7-NEXT:    s_waitcnt vmcnt(0)
9715; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9716; GFX7-NEXT:    s_waitcnt vmcnt(0)
9717; GFX7-NEXT:    buffer_wbinvl1_vol
9718; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9719; GFX7-NEXT:    flat_store_dword v[0:1], v2
9720; GFX7-NEXT:    s_endpgm
9721;
9722; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
9723; GFX10-WGP:       ; %bb.0: ; %entry
9724; GFX10-WGP-NEXT:    s_clause 0x1
9725; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9726; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
9727; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9728; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9729; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9730; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9731; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9732; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9733; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9734; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9735; GFX10-WGP-NEXT:    buffer_gl0_inv
9736; GFX10-WGP-NEXT:    buffer_gl1_inv
9737; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9738; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
9739; GFX10-WGP-NEXT:    s_endpgm
9740;
9741; GFX10-CU-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
9742; GFX10-CU:       ; %bb.0: ; %entry
9743; GFX10-CU-NEXT:    s_clause 0x1
9744; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9745; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
9746; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9747; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9748; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9749; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9750; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9751; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9752; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9753; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9754; GFX10-CU-NEXT:    buffer_gl0_inv
9755; GFX10-CU-NEXT:    buffer_gl1_inv
9756; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9757; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
9758; GFX10-CU-NEXT:    s_endpgm
9759;
9760; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
9761; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9762; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9763; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
9764; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9765; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9766; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9767; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9768; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9769; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9770; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9771; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
9772; SKIP-CACHE-INV-NEXT:    s_endpgm
9773;
9774; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
9775; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9776; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9777; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9778; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9779; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9780; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9781; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
9782; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9783; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9784; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9785; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
9786; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9787; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9788; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9789; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9790;
9791; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
9792; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9793; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9794; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9795; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9796; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9797; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9798; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
9799; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9800; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9801; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9802; GFX90A-TGSPLIT-NEXT:    buffer_invl2
9803; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9804; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9805; GFX90A-TGSPLIT-NEXT:    s_endpgm
9806;
9807; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
9808; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9809; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9810; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9811; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9812; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9813; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9814; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
9815; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9816; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0 sc1
9817; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9818; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
9819; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9820; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9821; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9822;
9823; GFX940-TGSPLIT-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
9824; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9825; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9826; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9827; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9828; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9829; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9830; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
9831; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9832; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0 sc1
9833; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9834; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
9835; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9836; GFX940-TGSPLIT-NEXT:    s_endpgm
9837;
9838; GFX11-WGP-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
9839; GFX11-WGP:       ; %bb.0: ; %entry
9840; GFX11-WGP-NEXT:    s_clause 0x1
9841; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9842; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
9843; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9844; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9845; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9846; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9847; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9848; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
9849; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9850; GFX11-WGP-NEXT:    buffer_gl0_inv
9851; GFX11-WGP-NEXT:    buffer_gl1_inv
9852; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9853; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
9854; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9855; GFX11-WGP-NEXT:    s_endpgm
9856;
9857; GFX11-CU-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
9858; GFX11-CU:       ; %bb.0: ; %entry
9859; GFX11-CU-NEXT:    s_clause 0x1
9860; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9861; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
9862; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9863; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9864; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9865; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9866; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9867; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
9868; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9869; GFX11-CU-NEXT:    buffer_gl0_inv
9870; GFX11-CU-NEXT:    buffer_gl1_inv
9871; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9872; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
9873; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9874; GFX11-CU-NEXT:    s_endpgm
9875    i32* %out, i32 %in) {
9876entry:
9877  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") seq_cst
9878  store i32 %val, i32* %out, align 4
9879  ret void
9880}
9881
9882define amdgpu_kernel void @flat_system_one_as_monotonic_monotonic_cmpxchg(
9883; GFX7-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
9884; GFX7:       ; %bb.0: ; %entry
9885; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9886; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9887; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9888; GFX7-NEXT:    s_add_u32 s0, s0, 16
9889; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9890; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9891; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9892; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9893; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9894; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9895; GFX7-NEXT:    s_endpgm
9896;
9897; GFX10-WGP-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
9898; GFX10-WGP:       ; %bb.0: ; %entry
9899; GFX10-WGP-NEXT:    s_clause 0x1
9900; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9901; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9902; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9903; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9904; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9905; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9906; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9907; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9908; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9909; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9910; GFX10-WGP-NEXT:    s_endpgm
9911;
9912; GFX10-CU-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
9913; GFX10-CU:       ; %bb.0: ; %entry
9914; GFX10-CU-NEXT:    s_clause 0x1
9915; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9916; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9917; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9918; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9919; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9920; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9921; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9922; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9923; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9924; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9925; GFX10-CU-NEXT:    s_endpgm
9926;
9927; SKIP-CACHE-INV-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
9928; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9929; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9930; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9931; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9932; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9933; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9934; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9935; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9936; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9937; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9938; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9939; SKIP-CACHE-INV-NEXT:    s_endpgm
9940;
9941; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
9942; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9943; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9944; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9945; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9946; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9947; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9948; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9949; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9950;
9951; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
9952; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9953; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9954; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9955; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9956; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9957; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9958; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9959; GFX90A-TGSPLIT-NEXT:    s_endpgm
9960;
9961; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
9962; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9963; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9964; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9965; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9966; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9967; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9968; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
9969; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9970;
9971; GFX940-TGSPLIT-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
9972; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9973; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9974; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9975; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9976; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9977; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9978; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
9979; GFX940-TGSPLIT-NEXT:    s_endpgm
9980;
9981; GFX11-WGP-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
9982; GFX11-WGP:       ; %bb.0: ; %entry
9983; GFX11-WGP-NEXT:    s_clause 0x1
9984; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9985; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9986; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9987; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9988; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
9989; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9990; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9991; GFX11-WGP-NEXT:    s_endpgm
9992;
9993; GFX11-CU-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
9994; GFX11-CU:       ; %bb.0: ; %entry
9995; GFX11-CU-NEXT:    s_clause 0x1
9996; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9997; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9998; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9999; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10000; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10001; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10002; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10003; GFX11-CU-NEXT:    s_endpgm
10004    i32* %out, i32 %in, i32 %old) {
10005entry:
10006  %gep = getelementptr i32, i32* %out, i32 4
10007  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") monotonic monotonic
10008  ret void
10009}
10010
10011define amdgpu_kernel void @flat_system_one_as_acquire_monotonic_cmpxchg(
10012; GFX7-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
10013; GFX7:       ; %bb.0: ; %entry
10014; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10015; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10016; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10017; GFX7-NEXT:    s_add_u32 s0, s0, 16
10018; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10019; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10020; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10021; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10022; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10023; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10024; GFX7-NEXT:    s_waitcnt vmcnt(0)
10025; GFX7-NEXT:    buffer_wbinvl1_vol
10026; GFX7-NEXT:    s_endpgm
10027;
10028; GFX10-WGP-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
10029; GFX10-WGP:       ; %bb.0: ; %entry
10030; GFX10-WGP-NEXT:    s_clause 0x1
10031; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10032; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10033; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10034; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10035; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10036; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10037; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10038; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10039; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10040; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10041; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10042; GFX10-WGP-NEXT:    buffer_gl0_inv
10043; GFX10-WGP-NEXT:    buffer_gl1_inv
10044; GFX10-WGP-NEXT:    s_endpgm
10045;
10046; GFX10-CU-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
10047; GFX10-CU:       ; %bb.0: ; %entry
10048; GFX10-CU-NEXT:    s_clause 0x1
10049; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10050; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10051; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10052; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10053; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10054; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10055; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10056; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10057; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10058; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10059; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10060; GFX10-CU-NEXT:    buffer_gl0_inv
10061; GFX10-CU-NEXT:    buffer_gl1_inv
10062; GFX10-CU-NEXT:    s_endpgm
10063;
10064; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
10065; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10066; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10067; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10068; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10069; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10070; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10071; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10072; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10073; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10074; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10075; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10076; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10077; SKIP-CACHE-INV-NEXT:    s_endpgm
10078;
10079; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
10080; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10081; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10082; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10083; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10084; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10085; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10086; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10087; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10088; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
10089; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
10090; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10091;
10092; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
10093; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10094; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10095; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10096; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10097; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10098; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10099; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10100; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10101; GFX90A-TGSPLIT-NEXT:    buffer_invl2
10102; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10103; GFX90A-TGSPLIT-NEXT:    s_endpgm
10104;
10105; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
10106; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10107; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10108; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10109; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10110; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10111; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10112; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
10113; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10114; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
10115; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10116;
10117; GFX940-TGSPLIT-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
10118; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10119; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10120; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10121; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10122; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10123; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10124; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
10125; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10126; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
10127; GFX940-TGSPLIT-NEXT:    s_endpgm
10128;
10129; GFX11-WGP-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
10130; GFX11-WGP:       ; %bb.0: ; %entry
10131; GFX11-WGP-NEXT:    s_clause 0x1
10132; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10133; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10134; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10135; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10136; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10137; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10138; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10139; GFX11-WGP-NEXT:    buffer_gl0_inv
10140; GFX11-WGP-NEXT:    buffer_gl1_inv
10141; GFX11-WGP-NEXT:    s_endpgm
10142;
10143; GFX11-CU-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
10144; GFX11-CU:       ; %bb.0: ; %entry
10145; GFX11-CU-NEXT:    s_clause 0x1
10146; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10147; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10148; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10149; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10150; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10151; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10152; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10153; GFX11-CU-NEXT:    buffer_gl0_inv
10154; GFX11-CU-NEXT:    buffer_gl1_inv
10155; GFX11-CU-NEXT:    s_endpgm
10156    i32* %out, i32 %in, i32 %old) {
10157entry:
10158  %gep = getelementptr i32, i32* %out, i32 4
10159  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acquire monotonic
10160  ret void
10161}
10162
10163define amdgpu_kernel void @flat_system_one_as_release_monotonic_cmpxchg(
10164; GFX7-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
10165; GFX7:       ; %bb.0: ; %entry
10166; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10167; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10168; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10169; GFX7-NEXT:    s_add_u32 s0, s0, 16
10170; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10171; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10172; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10173; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10174; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10175; GFX7-NEXT:    s_waitcnt vmcnt(0)
10176; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10177; GFX7-NEXT:    s_endpgm
10178;
10179; GFX10-WGP-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
10180; GFX10-WGP:       ; %bb.0: ; %entry
10181; GFX10-WGP-NEXT:    s_clause 0x1
10182; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10183; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10184; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10185; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10186; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10187; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10188; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10189; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10190; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10191; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10192; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10193; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10194; GFX10-WGP-NEXT:    s_endpgm
10195;
10196; GFX10-CU-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
10197; GFX10-CU:       ; %bb.0: ; %entry
10198; GFX10-CU-NEXT:    s_clause 0x1
10199; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10200; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10201; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10202; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10203; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10204; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10205; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10206; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10207; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10208; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10209; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10210; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10211; GFX10-CU-NEXT:    s_endpgm
10212;
10213; SKIP-CACHE-INV-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
10214; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10215; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10216; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10217; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10218; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10219; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10220; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10221; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10222; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10223; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10224; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10225; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10226; SKIP-CACHE-INV-NEXT:    s_endpgm
10227;
10228; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
10229; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10230; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10231; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10232; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10233; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10234; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10235; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
10236; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10237; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10238; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10239;
10240; GFX90A-TGSPLIT-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
10241; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10242; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10243; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10244; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10245; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10246; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10247; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
10248; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10249; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10250; GFX90A-TGSPLIT-NEXT:    s_endpgm
10251;
10252; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
10253; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10254; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10255; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10256; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10257; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10258; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10259; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
10260; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10261; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
10262; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10263;
10264; GFX940-TGSPLIT-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
10265; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10266; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10267; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10268; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10269; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10270; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10271; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
10272; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10273; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
10274; GFX940-TGSPLIT-NEXT:    s_endpgm
10275;
10276; GFX11-WGP-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
10277; GFX11-WGP:       ; %bb.0: ; %entry
10278; GFX11-WGP-NEXT:    s_clause 0x1
10279; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10280; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10281; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10282; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10283; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10284; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
10285; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10286; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10287; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10288; GFX11-WGP-NEXT:    s_endpgm
10289;
10290; GFX11-CU-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
10291; GFX11-CU:       ; %bb.0: ; %entry
10292; GFX11-CU-NEXT:    s_clause 0x1
10293; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10294; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10295; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10296; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10297; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10298; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
10299; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10300; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10301; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10302; GFX11-CU-NEXT:    s_endpgm
10303    i32* %out, i32 %in, i32 %old) {
10304entry:
10305  %gep = getelementptr i32, i32* %out, i32 4
10306  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") release monotonic
10307  ret void
10308}
10309
10310define amdgpu_kernel void @flat_system_one_as_acq_rel_monotonic_cmpxchg(
10311; GFX7-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
10312; GFX7:       ; %bb.0: ; %entry
10313; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10314; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10315; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10316; GFX7-NEXT:    s_add_u32 s0, s0, 16
10317; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10318; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10319; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10320; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10321; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10322; GFX7-NEXT:    s_waitcnt vmcnt(0)
10323; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10324; GFX7-NEXT:    s_waitcnt vmcnt(0)
10325; GFX7-NEXT:    buffer_wbinvl1_vol
10326; GFX7-NEXT:    s_endpgm
10327;
10328; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
10329; GFX10-WGP:       ; %bb.0: ; %entry
10330; GFX10-WGP-NEXT:    s_clause 0x1
10331; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10332; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10333; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10334; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10335; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10336; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10337; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10338; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10339; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10340; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10341; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10342; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10343; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10344; GFX10-WGP-NEXT:    buffer_gl0_inv
10345; GFX10-WGP-NEXT:    buffer_gl1_inv
10346; GFX10-WGP-NEXT:    s_endpgm
10347;
10348; GFX10-CU-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
10349; GFX10-CU:       ; %bb.0: ; %entry
10350; GFX10-CU-NEXT:    s_clause 0x1
10351; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10352; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10353; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10354; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10355; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10356; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10357; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10358; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10359; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10360; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10361; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10362; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10363; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10364; GFX10-CU-NEXT:    buffer_gl0_inv
10365; GFX10-CU-NEXT:    buffer_gl1_inv
10366; GFX10-CU-NEXT:    s_endpgm
10367;
10368; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
10369; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10370; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10371; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10372; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10373; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10374; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10375; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10376; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10377; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10378; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10379; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10380; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10381; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10382; SKIP-CACHE-INV-NEXT:    s_endpgm
10383;
10384; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
10385; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10386; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10387; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10388; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10389; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10390; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10391; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
10392; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10393; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10394; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10395; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
10396; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
10397; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10398;
10399; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
10400; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10401; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10402; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10403; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10404; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10405; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10406; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
10407; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10408; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10409; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10410; GFX90A-TGSPLIT-NEXT:    buffer_invl2
10411; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10412; GFX90A-TGSPLIT-NEXT:    s_endpgm
10413;
10414; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
10415; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10416; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10417; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10418; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10419; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10420; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10421; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
10422; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10423; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
10424; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10425; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
10426; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10427;
10428; GFX940-TGSPLIT-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
10429; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10430; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10431; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10432; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10433; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10434; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10435; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
10436; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10437; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
10438; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10439; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
10440; GFX940-TGSPLIT-NEXT:    s_endpgm
10441;
10442; GFX11-WGP-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
10443; GFX11-WGP:       ; %bb.0: ; %entry
10444; GFX11-WGP-NEXT:    s_clause 0x1
10445; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10446; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10447; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10448; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10449; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10450; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
10451; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10452; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10453; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10454; GFX11-WGP-NEXT:    buffer_gl0_inv
10455; GFX11-WGP-NEXT:    buffer_gl1_inv
10456; GFX11-WGP-NEXT:    s_endpgm
10457;
10458; GFX11-CU-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
10459; GFX11-CU:       ; %bb.0: ; %entry
10460; GFX11-CU-NEXT:    s_clause 0x1
10461; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10462; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10463; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10464; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10465; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10466; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
10467; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10468; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10469; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10470; GFX11-CU-NEXT:    buffer_gl0_inv
10471; GFX11-CU-NEXT:    buffer_gl1_inv
10472; GFX11-CU-NEXT:    s_endpgm
10473    i32* %out, i32 %in, i32 %old) {
10474entry:
10475  %gep = getelementptr i32, i32* %out, i32 4
10476  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel monotonic
10477  ret void
10478}
10479
10480define amdgpu_kernel void @flat_system_one_as_seq_cst_monotonic_cmpxchg(
10481; GFX7-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
10482; GFX7:       ; %bb.0: ; %entry
10483; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10484; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10485; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10486; GFX7-NEXT:    s_add_u32 s0, s0, 16
10487; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10488; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10489; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10490; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10491; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10492; GFX7-NEXT:    s_waitcnt vmcnt(0)
10493; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10494; GFX7-NEXT:    s_waitcnt vmcnt(0)
10495; GFX7-NEXT:    buffer_wbinvl1_vol
10496; GFX7-NEXT:    s_endpgm
10497;
10498; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
10499; GFX10-WGP:       ; %bb.0: ; %entry
10500; GFX10-WGP-NEXT:    s_clause 0x1
10501; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10502; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10503; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10504; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10505; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10506; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10507; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10508; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10509; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10510; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10511; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10512; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10513; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10514; GFX10-WGP-NEXT:    buffer_gl0_inv
10515; GFX10-WGP-NEXT:    buffer_gl1_inv
10516; GFX10-WGP-NEXT:    s_endpgm
10517;
10518; GFX10-CU-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
10519; GFX10-CU:       ; %bb.0: ; %entry
10520; GFX10-CU-NEXT:    s_clause 0x1
10521; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10522; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10523; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10524; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10525; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10526; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10527; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10528; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10529; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10530; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10531; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10532; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10533; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10534; GFX10-CU-NEXT:    buffer_gl0_inv
10535; GFX10-CU-NEXT:    buffer_gl1_inv
10536; GFX10-CU-NEXT:    s_endpgm
10537;
10538; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
10539; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10540; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10541; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10542; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10543; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10544; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10545; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10546; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10547; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10548; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10549; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10550; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10551; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10552; SKIP-CACHE-INV-NEXT:    s_endpgm
10553;
10554; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
10555; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10556; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10557; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10558; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10559; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10560; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10561; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
10562; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10563; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10564; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10565; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
10566; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
10567; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10568;
10569; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
10570; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10571; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10572; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10573; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10574; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10575; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10576; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
10577; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10578; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10579; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10580; GFX90A-TGSPLIT-NEXT:    buffer_invl2
10581; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10582; GFX90A-TGSPLIT-NEXT:    s_endpgm
10583;
10584; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
10585; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10586; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10587; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10588; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10589; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10590; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10591; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
10592; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10593; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
10594; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10595; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
10596; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10597;
10598; GFX940-TGSPLIT-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
10599; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10600; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10601; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10602; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10603; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10604; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10605; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
10606; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10607; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
10608; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10609; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
10610; GFX940-TGSPLIT-NEXT:    s_endpgm
10611;
10612; GFX11-WGP-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
10613; GFX11-WGP:       ; %bb.0: ; %entry
10614; GFX11-WGP-NEXT:    s_clause 0x1
10615; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10616; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10617; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10618; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10619; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10620; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
10621; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10622; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10623; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10624; GFX11-WGP-NEXT:    buffer_gl0_inv
10625; GFX11-WGP-NEXT:    buffer_gl1_inv
10626; GFX11-WGP-NEXT:    s_endpgm
10627;
10628; GFX11-CU-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
10629; GFX11-CU:       ; %bb.0: ; %entry
10630; GFX11-CU-NEXT:    s_clause 0x1
10631; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10632; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10633; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10634; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10635; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10636; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
10637; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10638; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10639; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10640; GFX11-CU-NEXT:    buffer_gl0_inv
10641; GFX11-CU-NEXT:    buffer_gl1_inv
10642; GFX11-CU-NEXT:    s_endpgm
10643    i32* %out, i32 %in, i32 %old) {
10644entry:
10645  %gep = getelementptr i32, i32* %out, i32 4
10646  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst monotonic
10647  ret void
10648}
10649
10650define amdgpu_kernel void @flat_system_one_as_monotonic_acquire_cmpxchg(
10651; GFX7-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
10652; GFX7:       ; %bb.0: ; %entry
10653; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10654; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10655; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10656; GFX7-NEXT:    s_add_u32 s0, s0, 16
10657; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10658; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10659; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10660; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10661; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10662; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10663; GFX7-NEXT:    s_waitcnt vmcnt(0)
10664; GFX7-NEXT:    buffer_wbinvl1_vol
10665; GFX7-NEXT:    s_endpgm
10666;
10667; GFX10-WGP-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
10668; GFX10-WGP:       ; %bb.0: ; %entry
10669; GFX10-WGP-NEXT:    s_clause 0x1
10670; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10671; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10672; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10673; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10674; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10675; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10676; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10677; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10678; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10679; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10680; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10681; GFX10-WGP-NEXT:    buffer_gl0_inv
10682; GFX10-WGP-NEXT:    buffer_gl1_inv
10683; GFX10-WGP-NEXT:    s_endpgm
10684;
10685; GFX10-CU-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
10686; GFX10-CU:       ; %bb.0: ; %entry
10687; GFX10-CU-NEXT:    s_clause 0x1
10688; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10689; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10690; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10691; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10692; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10693; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10694; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10695; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10696; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10697; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10698; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10699; GFX10-CU-NEXT:    buffer_gl0_inv
10700; GFX10-CU-NEXT:    buffer_gl1_inv
10701; GFX10-CU-NEXT:    s_endpgm
10702;
10703; SKIP-CACHE-INV-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
10704; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10705; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10706; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10707; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10708; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10709; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10710; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10711; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10712; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10713; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10714; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10715; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10716; SKIP-CACHE-INV-NEXT:    s_endpgm
10717;
10718; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
10719; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10720; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10721; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10722; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10723; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10724; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10725; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10726; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10727; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
10728; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
10729; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10730;
10731; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
10732; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10733; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10734; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10735; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10736; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10737; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10738; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10739; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10740; GFX90A-TGSPLIT-NEXT:    buffer_invl2
10741; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10742; GFX90A-TGSPLIT-NEXT:    s_endpgm
10743;
10744; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
10745; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10746; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10747; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10748; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10749; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10750; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10751; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
10752; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10753; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
10754; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10755;
10756; GFX940-TGSPLIT-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
10757; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10758; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10759; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10760; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10761; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10762; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10763; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
10764; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10765; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
10766; GFX940-TGSPLIT-NEXT:    s_endpgm
10767;
10768; GFX11-WGP-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
10769; GFX11-WGP:       ; %bb.0: ; %entry
10770; GFX11-WGP-NEXT:    s_clause 0x1
10771; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10772; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10773; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10774; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10775; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10776; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10777; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10778; GFX11-WGP-NEXT:    buffer_gl0_inv
10779; GFX11-WGP-NEXT:    buffer_gl1_inv
10780; GFX11-WGP-NEXT:    s_endpgm
10781;
10782; GFX11-CU-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
10783; GFX11-CU:       ; %bb.0: ; %entry
10784; GFX11-CU-NEXT:    s_clause 0x1
10785; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10786; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10787; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10788; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10789; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10790; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10791; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10792; GFX11-CU-NEXT:    buffer_gl0_inv
10793; GFX11-CU-NEXT:    buffer_gl1_inv
10794; GFX11-CU-NEXT:    s_endpgm
10795    i32* %out, i32 %in, i32 %old) {
10796entry:
10797  %gep = getelementptr i32, i32* %out, i32 4
10798  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") monotonic acquire
10799  ret void
10800}
10801
10802define amdgpu_kernel void @flat_system_one_as_acquire_acquire_cmpxchg(
10803; GFX7-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
10804; GFX7:       ; %bb.0: ; %entry
10805; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10806; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10807; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10808; GFX7-NEXT:    s_add_u32 s0, s0, 16
10809; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10810; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10811; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10812; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10813; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10814; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10815; GFX7-NEXT:    s_waitcnt vmcnt(0)
10816; GFX7-NEXT:    buffer_wbinvl1_vol
10817; GFX7-NEXT:    s_endpgm
10818;
10819; GFX10-WGP-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
10820; GFX10-WGP:       ; %bb.0: ; %entry
10821; GFX10-WGP-NEXT:    s_clause 0x1
10822; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10823; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10824; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10825; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10826; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10827; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10828; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10829; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10830; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10831; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10832; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10833; GFX10-WGP-NEXT:    buffer_gl0_inv
10834; GFX10-WGP-NEXT:    buffer_gl1_inv
10835; GFX10-WGP-NEXT:    s_endpgm
10836;
10837; GFX10-CU-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
10838; GFX10-CU:       ; %bb.0: ; %entry
10839; GFX10-CU-NEXT:    s_clause 0x1
10840; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10841; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10842; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10843; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10844; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10845; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10846; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10847; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10848; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10849; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10850; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10851; GFX10-CU-NEXT:    buffer_gl0_inv
10852; GFX10-CU-NEXT:    buffer_gl1_inv
10853; GFX10-CU-NEXT:    s_endpgm
10854;
10855; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
10856; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10857; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10858; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10859; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10860; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10861; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10862; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10863; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10864; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10865; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10866; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10867; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10868; SKIP-CACHE-INV-NEXT:    s_endpgm
10869;
10870; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
10871; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10872; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10873; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10874; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10875; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10876; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10877; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10878; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10879; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
10880; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
10881; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10882;
10883; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
10884; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10885; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10886; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10887; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10888; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10889; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10890; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10891; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10892; GFX90A-TGSPLIT-NEXT:    buffer_invl2
10893; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10894; GFX90A-TGSPLIT-NEXT:    s_endpgm
10895;
10896; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
10897; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10898; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10899; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10900; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10901; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10902; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10903; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
10904; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10905; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
10906; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10907;
10908; GFX940-TGSPLIT-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
10909; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10910; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10911; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10912; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10913; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10914; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10915; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
10916; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10917; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
10918; GFX940-TGSPLIT-NEXT:    s_endpgm
10919;
10920; GFX11-WGP-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
10921; GFX11-WGP:       ; %bb.0: ; %entry
10922; GFX11-WGP-NEXT:    s_clause 0x1
10923; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10924; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10925; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10926; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10927; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10928; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10929; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10930; GFX11-WGP-NEXT:    buffer_gl0_inv
10931; GFX11-WGP-NEXT:    buffer_gl1_inv
10932; GFX11-WGP-NEXT:    s_endpgm
10933;
10934; GFX11-CU-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
10935; GFX11-CU:       ; %bb.0: ; %entry
10936; GFX11-CU-NEXT:    s_clause 0x1
10937; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10938; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10939; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10940; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10941; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10942; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10943; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10944; GFX11-CU-NEXT:    buffer_gl0_inv
10945; GFX11-CU-NEXT:    buffer_gl1_inv
10946; GFX11-CU-NEXT:    s_endpgm
10947    i32* %out, i32 %in, i32 %old) {
10948entry:
10949  %gep = getelementptr i32, i32* %out, i32 4
10950  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acquire acquire
10951  ret void
10952}
10953
10954define amdgpu_kernel void @flat_system_one_as_release_acquire_cmpxchg(
10955; GFX7-LABEL: flat_system_one_as_release_acquire_cmpxchg:
10956; GFX7:       ; %bb.0: ; %entry
10957; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10958; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10959; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10960; GFX7-NEXT:    s_add_u32 s0, s0, 16
10961; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10962; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10963; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10964; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10965; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10966; GFX7-NEXT:    s_waitcnt vmcnt(0)
10967; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10968; GFX7-NEXT:    s_waitcnt vmcnt(0)
10969; GFX7-NEXT:    buffer_wbinvl1_vol
10970; GFX7-NEXT:    s_endpgm
10971;
10972; GFX10-WGP-LABEL: flat_system_one_as_release_acquire_cmpxchg:
10973; GFX10-WGP:       ; %bb.0: ; %entry
10974; GFX10-WGP-NEXT:    s_clause 0x1
10975; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10976; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10977; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10978; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10979; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10980; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10981; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10982; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10983; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10984; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10985; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10986; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10987; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10988; GFX10-WGP-NEXT:    buffer_gl0_inv
10989; GFX10-WGP-NEXT:    buffer_gl1_inv
10990; GFX10-WGP-NEXT:    s_endpgm
10991;
10992; GFX10-CU-LABEL: flat_system_one_as_release_acquire_cmpxchg:
10993; GFX10-CU:       ; %bb.0: ; %entry
10994; GFX10-CU-NEXT:    s_clause 0x1
10995; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10996; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10997; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10998; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10999; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
11000; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11001; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11002; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11003; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11004; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11005; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11006; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11007; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11008; GFX10-CU-NEXT:    buffer_gl0_inv
11009; GFX10-CU-NEXT:    buffer_gl1_inv
11010; GFX10-CU-NEXT:    s_endpgm
11011;
11012; SKIP-CACHE-INV-LABEL: flat_system_one_as_release_acquire_cmpxchg:
11013; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11014; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11015; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11016; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11017; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
11018; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
11019; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11020; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11021; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11022; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11023; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11024; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11025; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11026; SKIP-CACHE-INV-NEXT:    s_endpgm
11027;
11028; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_release_acquire_cmpxchg:
11029; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11030; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11031; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11032; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11033; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11034; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11035; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
11036; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11037; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11038; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11039; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
11040; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
11041; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11042;
11043; GFX90A-TGSPLIT-LABEL: flat_system_one_as_release_acquire_cmpxchg:
11044; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11045; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11046; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11047; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11048; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11049; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11050; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
11051; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11052; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11053; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11054; GFX90A-TGSPLIT-NEXT:    buffer_invl2
11055; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11056; GFX90A-TGSPLIT-NEXT:    s_endpgm
11057;
11058; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_release_acquire_cmpxchg:
11059; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11060; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11061; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11062; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11063; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11064; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11065; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
11066; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11067; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
11068; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11069; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
11070; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11071;
11072; GFX940-TGSPLIT-LABEL: flat_system_one_as_release_acquire_cmpxchg:
11073; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11074; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11075; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11076; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11077; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11078; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11079; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
11080; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11081; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
11082; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11083; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
11084; GFX940-TGSPLIT-NEXT:    s_endpgm
11085;
11086; GFX11-WGP-LABEL: flat_system_one_as_release_acquire_cmpxchg:
11087; GFX11-WGP:       ; %bb.0: ; %entry
11088; GFX11-WGP-NEXT:    s_clause 0x1
11089; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11090; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11091; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11092; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11093; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11094; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11095; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11096; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11097; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11098; GFX11-WGP-NEXT:    buffer_gl0_inv
11099; GFX11-WGP-NEXT:    buffer_gl1_inv
11100; GFX11-WGP-NEXT:    s_endpgm
11101;
11102; GFX11-CU-LABEL: flat_system_one_as_release_acquire_cmpxchg:
11103; GFX11-CU:       ; %bb.0: ; %entry
11104; GFX11-CU-NEXT:    s_clause 0x1
11105; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11106; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11107; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11108; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11109; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11110; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11111; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11112; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11113; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11114; GFX11-CU-NEXT:    buffer_gl0_inv
11115; GFX11-CU-NEXT:    buffer_gl1_inv
11116; GFX11-CU-NEXT:    s_endpgm
11117    i32* %out, i32 %in, i32 %old) {
11118entry:
11119  %gep = getelementptr i32, i32* %out, i32 4
11120  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") release acquire
11121  ret void
11122}
11123
11124define amdgpu_kernel void @flat_system_one_as_acq_rel_acquire_cmpxchg(
11125; GFX7-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
11126; GFX7:       ; %bb.0: ; %entry
11127; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11128; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11129; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11130; GFX7-NEXT:    s_add_u32 s0, s0, 16
11131; GFX7-NEXT:    s_addc_u32 s1, s1, 0
11132; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11133; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11134; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11135; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11136; GFX7-NEXT:    s_waitcnt vmcnt(0)
11137; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11138; GFX7-NEXT:    s_waitcnt vmcnt(0)
11139; GFX7-NEXT:    buffer_wbinvl1_vol
11140; GFX7-NEXT:    s_endpgm
11141;
11142; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
11143; GFX10-WGP:       ; %bb.0: ; %entry
11144; GFX10-WGP-NEXT:    s_clause 0x1
11145; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11146; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11147; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11148; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
11149; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
11150; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11151; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11152; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11153; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11154; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11155; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11156; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11157; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11158; GFX10-WGP-NEXT:    buffer_gl0_inv
11159; GFX10-WGP-NEXT:    buffer_gl1_inv
11160; GFX10-WGP-NEXT:    s_endpgm
11161;
11162; GFX10-CU-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
11163; GFX10-CU:       ; %bb.0: ; %entry
11164; GFX10-CU-NEXT:    s_clause 0x1
11165; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11166; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11167; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11168; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
11169; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
11170; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11171; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11172; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11173; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11174; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11175; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11176; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11177; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11178; GFX10-CU-NEXT:    buffer_gl0_inv
11179; GFX10-CU-NEXT:    buffer_gl1_inv
11180; GFX10-CU-NEXT:    s_endpgm
11181;
11182; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
11183; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11184; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11185; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11186; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11187; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
11188; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
11189; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11190; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11191; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11192; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11193; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11194; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11195; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11196; SKIP-CACHE-INV-NEXT:    s_endpgm
11197;
11198; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
11199; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11200; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11201; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11202; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11203; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11204; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11205; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
11206; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11207; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11208; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11209; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
11210; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
11211; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11212;
11213; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
11214; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11215; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11216; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11217; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11218; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11219; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11220; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
11221; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11222; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11223; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11224; GFX90A-TGSPLIT-NEXT:    buffer_invl2
11225; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11226; GFX90A-TGSPLIT-NEXT:    s_endpgm
11227;
11228; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
11229; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11230; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11231; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11232; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11233; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11234; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11235; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
11236; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11237; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
11238; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11239; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
11240; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11241;
11242; GFX940-TGSPLIT-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
11243; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11244; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11245; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11246; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11247; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11248; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11249; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
11250; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11251; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
11252; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11253; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
11254; GFX940-TGSPLIT-NEXT:    s_endpgm
11255;
11256; GFX11-WGP-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
11257; GFX11-WGP:       ; %bb.0: ; %entry
11258; GFX11-WGP-NEXT:    s_clause 0x1
11259; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11260; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11261; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11262; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11263; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11264; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11265; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11266; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11267; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11268; GFX11-WGP-NEXT:    buffer_gl0_inv
11269; GFX11-WGP-NEXT:    buffer_gl1_inv
11270; GFX11-WGP-NEXT:    s_endpgm
11271;
11272; GFX11-CU-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
11273; GFX11-CU:       ; %bb.0: ; %entry
11274; GFX11-CU-NEXT:    s_clause 0x1
11275; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11276; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11277; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11278; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11279; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11280; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11281; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11282; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11283; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11284; GFX11-CU-NEXT:    buffer_gl0_inv
11285; GFX11-CU-NEXT:    buffer_gl1_inv
11286; GFX11-CU-NEXT:    s_endpgm
11287    i32* %out, i32 %in, i32 %old) {
11288entry:
11289  %gep = getelementptr i32, i32* %out, i32 4
11290  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel acquire
11291  ret void
11292}
11293
11294define amdgpu_kernel void @flat_system_one_as_seq_cst_acquire_cmpxchg(
11295; GFX7-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
11296; GFX7:       ; %bb.0: ; %entry
11297; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11298; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11299; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11300; GFX7-NEXT:    s_add_u32 s0, s0, 16
11301; GFX7-NEXT:    s_addc_u32 s1, s1, 0
11302; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11303; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11304; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11305; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11306; GFX7-NEXT:    s_waitcnt vmcnt(0)
11307; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11308; GFX7-NEXT:    s_waitcnt vmcnt(0)
11309; GFX7-NEXT:    buffer_wbinvl1_vol
11310; GFX7-NEXT:    s_endpgm
11311;
11312; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
11313; GFX10-WGP:       ; %bb.0: ; %entry
11314; GFX10-WGP-NEXT:    s_clause 0x1
11315; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11316; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11317; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11318; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
11319; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
11320; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11321; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11322; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11323; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11324; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11325; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11326; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11327; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11328; GFX10-WGP-NEXT:    buffer_gl0_inv
11329; GFX10-WGP-NEXT:    buffer_gl1_inv
11330; GFX10-WGP-NEXT:    s_endpgm
11331;
11332; GFX10-CU-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
11333; GFX10-CU:       ; %bb.0: ; %entry
11334; GFX10-CU-NEXT:    s_clause 0x1
11335; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11336; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11337; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11338; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
11339; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
11340; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11341; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11342; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11343; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11344; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11345; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11346; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11347; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11348; GFX10-CU-NEXT:    buffer_gl0_inv
11349; GFX10-CU-NEXT:    buffer_gl1_inv
11350; GFX10-CU-NEXT:    s_endpgm
11351;
11352; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
11353; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11354; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11355; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11356; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11357; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
11358; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
11359; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11360; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11361; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11362; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11363; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11364; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11365; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11366; SKIP-CACHE-INV-NEXT:    s_endpgm
11367;
11368; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
11369; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11370; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11371; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11372; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11373; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11374; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11375; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
11376; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11377; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11378; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11379; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
11380; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
11381; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11382;
11383; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
11384; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11385; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11386; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11387; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11388; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11389; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11390; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
11391; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11392; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11393; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11394; GFX90A-TGSPLIT-NEXT:    buffer_invl2
11395; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11396; GFX90A-TGSPLIT-NEXT:    s_endpgm
11397;
11398; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
11399; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11400; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11401; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11402; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11403; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11404; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11405; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
11406; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11407; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
11408; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11409; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
11410; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11411;
11412; GFX940-TGSPLIT-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
11413; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11414; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11415; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11416; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11417; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11418; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11419; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
11420; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11421; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
11422; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11423; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
11424; GFX940-TGSPLIT-NEXT:    s_endpgm
11425;
11426; GFX11-WGP-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
11427; GFX11-WGP:       ; %bb.0: ; %entry
11428; GFX11-WGP-NEXT:    s_clause 0x1
11429; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11430; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11431; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11432; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11433; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11434; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11435; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11436; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11437; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11438; GFX11-WGP-NEXT:    buffer_gl0_inv
11439; GFX11-WGP-NEXT:    buffer_gl1_inv
11440; GFX11-WGP-NEXT:    s_endpgm
11441;
11442; GFX11-CU-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
11443; GFX11-CU:       ; %bb.0: ; %entry
11444; GFX11-CU-NEXT:    s_clause 0x1
11445; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11446; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11447; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11448; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11449; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11450; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11451; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11452; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11453; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11454; GFX11-CU-NEXT:    buffer_gl0_inv
11455; GFX11-CU-NEXT:    buffer_gl1_inv
11456; GFX11-CU-NEXT:    s_endpgm
11457    i32* %out, i32 %in, i32 %old) {
11458entry:
11459  %gep = getelementptr i32, i32* %out, i32 4
11460  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst acquire
11461  ret void
11462}
11463
11464define amdgpu_kernel void @flat_system_one_as_monotonic_seq_cst_cmpxchg(
11465; GFX7-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
11466; GFX7:       ; %bb.0: ; %entry
11467; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11468; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11469; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11470; GFX7-NEXT:    s_add_u32 s0, s0, 16
11471; GFX7-NEXT:    s_addc_u32 s1, s1, 0
11472; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11473; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11474; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11475; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11476; GFX7-NEXT:    s_waitcnt vmcnt(0)
11477; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11478; GFX7-NEXT:    s_waitcnt vmcnt(0)
11479; GFX7-NEXT:    buffer_wbinvl1_vol
11480; GFX7-NEXT:    s_endpgm
11481;
11482; GFX10-WGP-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
11483; GFX10-WGP:       ; %bb.0: ; %entry
11484; GFX10-WGP-NEXT:    s_clause 0x1
11485; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11486; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11487; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11488; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
11489; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
11490; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11491; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11492; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11493; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11494; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11495; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11496; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11497; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11498; GFX10-WGP-NEXT:    buffer_gl0_inv
11499; GFX10-WGP-NEXT:    buffer_gl1_inv
11500; GFX10-WGP-NEXT:    s_endpgm
11501;
11502; GFX10-CU-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
11503; GFX10-CU:       ; %bb.0: ; %entry
11504; GFX10-CU-NEXT:    s_clause 0x1
11505; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11506; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11507; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11508; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
11509; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
11510; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11511; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11512; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11513; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11514; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11515; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11516; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11517; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11518; GFX10-CU-NEXT:    buffer_gl0_inv
11519; GFX10-CU-NEXT:    buffer_gl1_inv
11520; GFX10-CU-NEXT:    s_endpgm
11521;
11522; SKIP-CACHE-INV-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
11523; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11524; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11525; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11526; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11527; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
11528; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
11529; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11530; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11531; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11532; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11533; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11534; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11535; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11536; SKIP-CACHE-INV-NEXT:    s_endpgm
11537;
11538; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
11539; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11540; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11541; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11542; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11543; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11544; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11545; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
11546; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11547; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11548; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11549; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
11550; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
11551; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11552;
11553; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
11554; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11555; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11556; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11557; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11558; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11559; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11560; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
11561; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11562; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11563; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11564; GFX90A-TGSPLIT-NEXT:    buffer_invl2
11565; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11566; GFX90A-TGSPLIT-NEXT:    s_endpgm
11567;
11568; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
11569; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11570; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11571; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11572; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11573; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11574; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11575; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
11576; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11577; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
11578; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11579; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
11580; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11581;
11582; GFX940-TGSPLIT-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
11583; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11584; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11585; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11586; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11587; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11588; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11589; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
11590; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11591; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
11592; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11593; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
11594; GFX940-TGSPLIT-NEXT:    s_endpgm
11595;
11596; GFX11-WGP-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
11597; GFX11-WGP:       ; %bb.0: ; %entry
11598; GFX11-WGP-NEXT:    s_clause 0x1
11599; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11600; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11601; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11602; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11603; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11604; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11605; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11606; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11607; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11608; GFX11-WGP-NEXT:    buffer_gl0_inv
11609; GFX11-WGP-NEXT:    buffer_gl1_inv
11610; GFX11-WGP-NEXT:    s_endpgm
11611;
11612; GFX11-CU-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
11613; GFX11-CU:       ; %bb.0: ; %entry
11614; GFX11-CU-NEXT:    s_clause 0x1
11615; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11616; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11617; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11618; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11619; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11620; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11621; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11622; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11623; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11624; GFX11-CU-NEXT:    buffer_gl0_inv
11625; GFX11-CU-NEXT:    buffer_gl1_inv
11626; GFX11-CU-NEXT:    s_endpgm
11627    i32* %out, i32 %in, i32 %old) {
11628entry:
11629  %gep = getelementptr i32, i32* %out, i32 4
11630  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") monotonic seq_cst
11631  ret void
11632}
11633
11634define amdgpu_kernel void @flat_system_one_as_acquire_seq_cst_cmpxchg(
11635; GFX7-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
11636; GFX7:       ; %bb.0: ; %entry
11637; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11638; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11639; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11640; GFX7-NEXT:    s_add_u32 s0, s0, 16
11641; GFX7-NEXT:    s_addc_u32 s1, s1, 0
11642; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11643; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11644; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11645; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11646; GFX7-NEXT:    s_waitcnt vmcnt(0)
11647; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11648; GFX7-NEXT:    s_waitcnt vmcnt(0)
11649; GFX7-NEXT:    buffer_wbinvl1_vol
11650; GFX7-NEXT:    s_endpgm
11651;
11652; GFX10-WGP-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
11653; GFX10-WGP:       ; %bb.0: ; %entry
11654; GFX10-WGP-NEXT:    s_clause 0x1
11655; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11656; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11657; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11658; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
11659; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
11660; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11661; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11662; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11663; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11664; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11665; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11666; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11667; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11668; GFX10-WGP-NEXT:    buffer_gl0_inv
11669; GFX10-WGP-NEXT:    buffer_gl1_inv
11670; GFX10-WGP-NEXT:    s_endpgm
11671;
11672; GFX10-CU-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
11673; GFX10-CU:       ; %bb.0: ; %entry
11674; GFX10-CU-NEXT:    s_clause 0x1
11675; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11676; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11677; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11678; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
11679; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
11680; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11681; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11682; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11683; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11684; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11685; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11686; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11687; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11688; GFX10-CU-NEXT:    buffer_gl0_inv
11689; GFX10-CU-NEXT:    buffer_gl1_inv
11690; GFX10-CU-NEXT:    s_endpgm
11691;
11692; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
11693; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11694; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11695; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11696; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11697; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
11698; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
11699; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11700; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11701; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11702; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11703; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11704; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11705; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11706; SKIP-CACHE-INV-NEXT:    s_endpgm
11707;
11708; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
11709; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11710; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11711; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11712; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11713; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11714; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11715; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
11716; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11717; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11718; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11719; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
11720; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
11721; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11722;
11723; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
11724; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11725; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11726; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11727; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11728; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11729; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11730; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
11731; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11732; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11733; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11734; GFX90A-TGSPLIT-NEXT:    buffer_invl2
11735; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11736; GFX90A-TGSPLIT-NEXT:    s_endpgm
11737;
11738; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
11739; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11740; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11741; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11742; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11743; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11744; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11745; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
11746; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11747; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
11748; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11749; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
11750; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11751;
11752; GFX940-TGSPLIT-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
11753; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11754; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11755; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11756; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11757; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11758; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11759; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
11760; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11761; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
11762; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11763; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
11764; GFX940-TGSPLIT-NEXT:    s_endpgm
11765;
11766; GFX11-WGP-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
11767; GFX11-WGP:       ; %bb.0: ; %entry
11768; GFX11-WGP-NEXT:    s_clause 0x1
11769; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11770; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11771; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11772; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11773; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11774; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11775; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11776; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11777; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11778; GFX11-WGP-NEXT:    buffer_gl0_inv
11779; GFX11-WGP-NEXT:    buffer_gl1_inv
11780; GFX11-WGP-NEXT:    s_endpgm
11781;
11782; GFX11-CU-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
11783; GFX11-CU:       ; %bb.0: ; %entry
11784; GFX11-CU-NEXT:    s_clause 0x1
11785; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11786; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11787; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11788; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11789; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11790; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11791; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11792; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11793; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11794; GFX11-CU-NEXT:    buffer_gl0_inv
11795; GFX11-CU-NEXT:    buffer_gl1_inv
11796; GFX11-CU-NEXT:    s_endpgm
11797    i32* %out, i32 %in, i32 %old) {
11798entry:
11799  %gep = getelementptr i32, i32* %out, i32 4
11800  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acquire seq_cst
11801  ret void
11802}
11803
11804define amdgpu_kernel void @flat_system_one_as_release_seq_cst_cmpxchg(
11805; GFX7-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
11806; GFX7:       ; %bb.0: ; %entry
11807; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11808; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11809; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11810; GFX7-NEXT:    s_add_u32 s0, s0, 16
11811; GFX7-NEXT:    s_addc_u32 s1, s1, 0
11812; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11813; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11814; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11815; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11816; GFX7-NEXT:    s_waitcnt vmcnt(0)
11817; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11818; GFX7-NEXT:    s_waitcnt vmcnt(0)
11819; GFX7-NEXT:    buffer_wbinvl1_vol
11820; GFX7-NEXT:    s_endpgm
11821;
11822; GFX10-WGP-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
11823; GFX10-WGP:       ; %bb.0: ; %entry
11824; GFX10-WGP-NEXT:    s_clause 0x1
11825; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11826; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11827; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11828; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
11829; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
11830; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11831; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11832; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11833; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11834; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11835; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11836; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11837; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11838; GFX10-WGP-NEXT:    buffer_gl0_inv
11839; GFX10-WGP-NEXT:    buffer_gl1_inv
11840; GFX10-WGP-NEXT:    s_endpgm
11841;
11842; GFX10-CU-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
11843; GFX10-CU:       ; %bb.0: ; %entry
11844; GFX10-CU-NEXT:    s_clause 0x1
11845; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11846; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11847; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11848; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
11849; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
11850; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11851; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11852; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11853; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11854; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11855; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11856; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11857; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11858; GFX10-CU-NEXT:    buffer_gl0_inv
11859; GFX10-CU-NEXT:    buffer_gl1_inv
11860; GFX10-CU-NEXT:    s_endpgm
11861;
11862; SKIP-CACHE-INV-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
11863; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11864; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11865; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11866; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11867; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
11868; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
11869; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11870; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11871; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11872; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11873; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11874; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11875; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11876; SKIP-CACHE-INV-NEXT:    s_endpgm
11877;
11878; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
11879; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11880; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11881; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11882; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11883; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11884; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11885; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
11886; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11887; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11888; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11889; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
11890; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
11891; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11892;
11893; GFX90A-TGSPLIT-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
11894; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11895; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11896; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11897; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11898; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11899; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11900; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
11901; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11902; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11903; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11904; GFX90A-TGSPLIT-NEXT:    buffer_invl2
11905; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11906; GFX90A-TGSPLIT-NEXT:    s_endpgm
11907;
11908; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
11909; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11910; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11911; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11912; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11913; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11914; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11915; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
11916; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11917; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
11918; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11919; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
11920; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11921;
11922; GFX940-TGSPLIT-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
11923; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11924; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11925; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11926; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11927; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11928; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11929; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
11930; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11931; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
11932; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11933; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
11934; GFX940-TGSPLIT-NEXT:    s_endpgm
11935;
11936; GFX11-WGP-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
11937; GFX11-WGP:       ; %bb.0: ; %entry
11938; GFX11-WGP-NEXT:    s_clause 0x1
11939; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11940; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11941; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11942; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11943; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11944; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11945; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11946; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11947; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11948; GFX11-WGP-NEXT:    buffer_gl0_inv
11949; GFX11-WGP-NEXT:    buffer_gl1_inv
11950; GFX11-WGP-NEXT:    s_endpgm
11951;
11952; GFX11-CU-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
11953; GFX11-CU:       ; %bb.0: ; %entry
11954; GFX11-CU-NEXT:    s_clause 0x1
11955; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11956; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11957; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11958; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11959; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11960; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11961; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11962; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11963; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11964; GFX11-CU-NEXT:    buffer_gl0_inv
11965; GFX11-CU-NEXT:    buffer_gl1_inv
11966; GFX11-CU-NEXT:    s_endpgm
11967    i32* %out, i32 %in, i32 %old) {
11968entry:
11969  %gep = getelementptr i32, i32* %out, i32 4
11970  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") release seq_cst
11971  ret void
11972}
11973
11974define amdgpu_kernel void @flat_system_one_as_acq_rel_seq_cst_cmpxchg(
11975; GFX7-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
11976; GFX7:       ; %bb.0: ; %entry
11977; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11978; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11979; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11980; GFX7-NEXT:    s_add_u32 s0, s0, 16
11981; GFX7-NEXT:    s_addc_u32 s1, s1, 0
11982; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11983; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11984; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11985; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11986; GFX7-NEXT:    s_waitcnt vmcnt(0)
11987; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11988; GFX7-NEXT:    s_waitcnt vmcnt(0)
11989; GFX7-NEXT:    buffer_wbinvl1_vol
11990; GFX7-NEXT:    s_endpgm
11991;
11992; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
11993; GFX10-WGP:       ; %bb.0: ; %entry
11994; GFX10-WGP-NEXT:    s_clause 0x1
11995; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11996; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11997; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11998; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
11999; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
12000; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12001; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12002; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12003; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12004; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12005; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12006; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
12007; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12008; GFX10-WGP-NEXT:    buffer_gl0_inv
12009; GFX10-WGP-NEXT:    buffer_gl1_inv
12010; GFX10-WGP-NEXT:    s_endpgm
12011;
12012; GFX10-CU-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
12013; GFX10-CU:       ; %bb.0: ; %entry
12014; GFX10-CU-NEXT:    s_clause 0x1
12015; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12016; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12017; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12018; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
12019; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
12020; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12021; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12022; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12023; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12024; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12025; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12026; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
12027; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12028; GFX10-CU-NEXT:    buffer_gl0_inv
12029; GFX10-CU-NEXT:    buffer_gl1_inv
12030; GFX10-CU-NEXT:    s_endpgm
12031;
12032; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
12033; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12034; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12035; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12036; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12037; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
12038; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
12039; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12040; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12041; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12042; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12043; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12044; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
12045; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12046; SKIP-CACHE-INV-NEXT:    s_endpgm
12047;
12048; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
12049; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12050; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12051; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12052; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12053; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12054; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12055; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
12056; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12057; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
12058; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12059; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
12060; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
12061; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12062;
12063; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
12064; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12065; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12066; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12067; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12068; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12069; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12070; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
12071; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12072; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
12073; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12074; GFX90A-TGSPLIT-NEXT:    buffer_invl2
12075; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
12076; GFX90A-TGSPLIT-NEXT:    s_endpgm
12077;
12078; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
12079; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12080; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12081; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12082; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12083; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12084; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12085; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
12086; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12087; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
12088; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12089; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
12090; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12091;
12092; GFX940-TGSPLIT-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
12093; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12094; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12095; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12096; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12097; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12098; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12099; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
12100; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12101; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
12102; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12103; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
12104; GFX940-TGSPLIT-NEXT:    s_endpgm
12105;
12106; GFX11-WGP-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
12107; GFX11-WGP:       ; %bb.0: ; %entry
12108; GFX11-WGP-NEXT:    s_clause 0x1
12109; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12110; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12111; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12112; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12113; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12114; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12115; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12116; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
12117; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12118; GFX11-WGP-NEXT:    buffer_gl0_inv
12119; GFX11-WGP-NEXT:    buffer_gl1_inv
12120; GFX11-WGP-NEXT:    s_endpgm
12121;
12122; GFX11-CU-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
12123; GFX11-CU:       ; %bb.0: ; %entry
12124; GFX11-CU-NEXT:    s_clause 0x1
12125; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12126; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12127; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12128; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12129; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12130; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
12131; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12132; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
12133; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12134; GFX11-CU-NEXT:    buffer_gl0_inv
12135; GFX11-CU-NEXT:    buffer_gl1_inv
12136; GFX11-CU-NEXT:    s_endpgm
12137    i32* %out, i32 %in, i32 %old) {
12138entry:
12139  %gep = getelementptr i32, i32* %out, i32 4
12140  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel seq_cst
12141  ret void
12142}
12143
12144define amdgpu_kernel void @flat_system_one_as_seq_cst_seq_cst_cmpxchg(
12145; GFX7-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
12146; GFX7:       ; %bb.0: ; %entry
12147; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12148; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12149; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12150; GFX7-NEXT:    s_add_u32 s0, s0, 16
12151; GFX7-NEXT:    s_addc_u32 s1, s1, 0
12152; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12153; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12154; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12155; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12156; GFX7-NEXT:    s_waitcnt vmcnt(0)
12157; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
12158; GFX7-NEXT:    s_waitcnt vmcnt(0)
12159; GFX7-NEXT:    buffer_wbinvl1_vol
12160; GFX7-NEXT:    s_endpgm
12161;
12162; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
12163; GFX10-WGP:       ; %bb.0: ; %entry
12164; GFX10-WGP-NEXT:    s_clause 0x1
12165; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12166; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12167; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12168; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
12169; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
12170; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12171; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12172; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12173; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12174; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12175; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12176; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
12177; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12178; GFX10-WGP-NEXT:    buffer_gl0_inv
12179; GFX10-WGP-NEXT:    buffer_gl1_inv
12180; GFX10-WGP-NEXT:    s_endpgm
12181;
12182; GFX10-CU-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
12183; GFX10-CU:       ; %bb.0: ; %entry
12184; GFX10-CU-NEXT:    s_clause 0x1
12185; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12186; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12187; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12188; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
12189; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
12190; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12191; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12192; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12193; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12194; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12195; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12196; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
12197; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12198; GFX10-CU-NEXT:    buffer_gl0_inv
12199; GFX10-CU-NEXT:    buffer_gl1_inv
12200; GFX10-CU-NEXT:    s_endpgm
12201;
12202; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
12203; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12204; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12205; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12206; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12207; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
12208; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
12209; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12210; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12211; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12212; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12213; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12214; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
12215; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12216; SKIP-CACHE-INV-NEXT:    s_endpgm
12217;
12218; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
12219; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12220; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12221; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12222; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12223; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12224; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12225; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
12226; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12227; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
12228; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12229; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
12230; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
12231; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12232;
12233; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
12234; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12235; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12236; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12237; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12238; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12239; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12240; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
12241; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12242; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
12243; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12244; GFX90A-TGSPLIT-NEXT:    buffer_invl2
12245; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
12246; GFX90A-TGSPLIT-NEXT:    s_endpgm
12247;
12248; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
12249; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12250; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12251; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12252; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12253; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12254; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12255; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
12256; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12257; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
12258; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12259; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
12260; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12261;
12262; GFX940-TGSPLIT-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
12263; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12264; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12265; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12266; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12267; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12268; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12269; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
12270; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12271; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
12272; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12273; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
12274; GFX940-TGSPLIT-NEXT:    s_endpgm
12275;
12276; GFX11-WGP-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
12277; GFX11-WGP:       ; %bb.0: ; %entry
12278; GFX11-WGP-NEXT:    s_clause 0x1
12279; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12280; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12281; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12282; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12283; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12284; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12285; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12286; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
12287; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12288; GFX11-WGP-NEXT:    buffer_gl0_inv
12289; GFX11-WGP-NEXT:    buffer_gl1_inv
12290; GFX11-WGP-NEXT:    s_endpgm
12291;
12292; GFX11-CU-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
12293; GFX11-CU:       ; %bb.0: ; %entry
12294; GFX11-CU-NEXT:    s_clause 0x1
12295; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12296; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12297; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12298; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12299; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12300; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
12301; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12302; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
12303; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12304; GFX11-CU-NEXT:    buffer_gl0_inv
12305; GFX11-CU-NEXT:    buffer_gl1_inv
12306; GFX11-CU-NEXT:    s_endpgm
12307    i32* %out, i32 %in, i32 %old) {
12308entry:
12309  %gep = getelementptr i32, i32* %out, i32 4
12310  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst seq_cst
12311  ret void
12312}
12313
12314define amdgpu_kernel void @flat_system_one_as_monotonic_monotonic_ret_cmpxchg(
12315; GFX7-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
12316; GFX7:       ; %bb.0: ; %entry
12317; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12318; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12319; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12320; GFX7-NEXT:    s_add_u32 s4, s0, 16
12321; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12322; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12323; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12324; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12325; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12326; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12327; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12328; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12329; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12330; GFX7-NEXT:    flat_store_dword v[0:1], v2
12331; GFX7-NEXT:    s_endpgm
12332;
12333; GFX10-WGP-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
12334; GFX10-WGP:       ; %bb.0: ; %entry
12335; GFX10-WGP-NEXT:    s_clause 0x1
12336; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12337; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12338; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12339; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12340; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12341; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12342; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12343; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12344; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12345; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12346; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12347; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12348; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12349; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12350; GFX10-WGP-NEXT:    s_endpgm
12351;
12352; GFX10-CU-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
12353; GFX10-CU:       ; %bb.0: ; %entry
12354; GFX10-CU-NEXT:    s_clause 0x1
12355; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12356; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12357; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12358; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12359; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12360; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12361; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12362; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12363; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12364; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12365; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12366; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12367; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12368; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12369; GFX10-CU-NEXT:    s_endpgm
12370;
12371; SKIP-CACHE-INV-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
12372; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12373; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12374; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12375; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12376; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12377; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12378; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12379; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12380; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12381; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12382; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12383; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12384; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12385; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12386; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12387; SKIP-CACHE-INV-NEXT:    s_endpgm
12388;
12389; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
12390; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12391; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12392; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12393; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12394; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12395; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12396; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12397; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12398; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12399; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12400;
12401; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
12402; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12403; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12404; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12405; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12406; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12407; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12408; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12409; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12410; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12411; GFX90A-TGSPLIT-NEXT:    s_endpgm
12412;
12413; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
12414; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12415; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12416; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12417; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12418; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12419; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12420; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
12421; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12422; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12423; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12424;
12425; GFX940-TGSPLIT-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
12426; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12427; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12428; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12429; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12430; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12431; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12432; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
12433; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12434; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12435; GFX940-TGSPLIT-NEXT:    s_endpgm
12436;
12437; GFX11-WGP-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
12438; GFX11-WGP:       ; %bb.0: ; %entry
12439; GFX11-WGP-NEXT:    s_clause 0x1
12440; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12441; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12442; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12443; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12444; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12445; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12446; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12447; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12448; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12449; GFX11-WGP-NEXT:    s_endpgm
12450;
12451; GFX11-CU-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
12452; GFX11-CU:       ; %bb.0: ; %entry
12453; GFX11-CU-NEXT:    s_clause 0x1
12454; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12455; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12456; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12457; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12458; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12459; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12460; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12461; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12462; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12463; GFX11-CU-NEXT:    s_endpgm
12464    i32* %out, i32 %in, i32 %old) {
12465entry:
12466  %gep = getelementptr i32, i32* %out, i32 4
12467  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") monotonic monotonic
12468  %val0 = extractvalue { i32, i1 } %val, 0
12469  store i32 %val0, i32* %out, align 4
12470  ret void
12471}
12472
12473define amdgpu_kernel void @flat_system_one_as_acquire_monotonic_ret_cmpxchg(
12474; GFX7-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
12475; GFX7:       ; %bb.0: ; %entry
12476; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12477; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12478; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12479; GFX7-NEXT:    s_add_u32 s4, s0, 16
12480; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12481; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12482; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12483; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12484; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12485; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12486; GFX7-NEXT:    s_waitcnt vmcnt(0)
12487; GFX7-NEXT:    buffer_wbinvl1_vol
12488; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12489; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12490; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12491; GFX7-NEXT:    flat_store_dword v[0:1], v2
12492; GFX7-NEXT:    s_endpgm
12493;
12494; GFX10-WGP-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
12495; GFX10-WGP:       ; %bb.0: ; %entry
12496; GFX10-WGP-NEXT:    s_clause 0x1
12497; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12498; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12499; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12500; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12501; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12502; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12503; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12504; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12505; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12506; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12507; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12508; GFX10-WGP-NEXT:    buffer_gl0_inv
12509; GFX10-WGP-NEXT:    buffer_gl1_inv
12510; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12511; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12512; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12513; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12514; GFX10-WGP-NEXT:    s_endpgm
12515;
12516; GFX10-CU-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
12517; GFX10-CU:       ; %bb.0: ; %entry
12518; GFX10-CU-NEXT:    s_clause 0x1
12519; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12520; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12521; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12522; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12523; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12524; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12525; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12526; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12527; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12528; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12529; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12530; GFX10-CU-NEXT:    buffer_gl0_inv
12531; GFX10-CU-NEXT:    buffer_gl1_inv
12532; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12533; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12534; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12535; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12536; GFX10-CU-NEXT:    s_endpgm
12537;
12538; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
12539; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12540; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12541; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12542; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12543; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12544; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12545; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12546; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12547; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12548; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12549; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12550; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12551; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12552; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12553; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12554; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12555; SKIP-CACHE-INV-NEXT:    s_endpgm
12556;
12557; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
12558; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12559; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12560; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12561; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12562; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12563; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12564; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12565; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12566; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
12567; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
12568; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12569; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12570; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12571;
12572; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
12573; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12574; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12575; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12576; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12577; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12578; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12579; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12580; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12581; GFX90A-TGSPLIT-NEXT:    buffer_invl2
12582; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
12583; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12584; GFX90A-TGSPLIT-NEXT:    s_endpgm
12585;
12586; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
12587; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12588; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12589; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12590; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12591; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12592; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12593; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
12594; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12595; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
12596; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12597; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12598; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12599;
12600; GFX940-TGSPLIT-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
12601; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12602; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12603; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12604; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12605; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12606; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12607; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
12608; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12609; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
12610; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12611; GFX940-TGSPLIT-NEXT:    s_endpgm
12612;
12613; GFX11-WGP-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
12614; GFX11-WGP:       ; %bb.0: ; %entry
12615; GFX11-WGP-NEXT:    s_clause 0x1
12616; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12617; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12618; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12619; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12620; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12621; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12622; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12623; GFX11-WGP-NEXT:    buffer_gl0_inv
12624; GFX11-WGP-NEXT:    buffer_gl1_inv
12625; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12626; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12627; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12628; GFX11-WGP-NEXT:    s_endpgm
12629;
12630; GFX11-CU-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
12631; GFX11-CU:       ; %bb.0: ; %entry
12632; GFX11-CU-NEXT:    s_clause 0x1
12633; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12634; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12635; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12636; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12637; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12638; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12639; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
12640; GFX11-CU-NEXT:    buffer_gl0_inv
12641; GFX11-CU-NEXT:    buffer_gl1_inv
12642; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12643; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12644; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12645; GFX11-CU-NEXT:    s_endpgm
12646    i32* %out, i32 %in, i32 %old) {
12647entry:
12648  %gep = getelementptr i32, i32* %out, i32 4
12649  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acquire monotonic
12650  %val0 = extractvalue { i32, i1 } %val, 0
12651  store i32 %val0, i32* %out, align 4
12652  ret void
12653}
12654
12655define amdgpu_kernel void @flat_system_one_as_release_monotonic_ret_cmpxchg(
12656; GFX7-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
12657; GFX7:       ; %bb.0: ; %entry
12658; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12659; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12660; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12661; GFX7-NEXT:    s_add_u32 s4, s0, 16
12662; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12663; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12664; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12665; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12666; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12667; GFX7-NEXT:    s_waitcnt vmcnt(0)
12668; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12669; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12670; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12671; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12672; GFX7-NEXT:    flat_store_dword v[0:1], v2
12673; GFX7-NEXT:    s_endpgm
12674;
12675; GFX10-WGP-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
12676; GFX10-WGP:       ; %bb.0: ; %entry
12677; GFX10-WGP-NEXT:    s_clause 0x1
12678; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12679; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12680; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12681; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12682; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12683; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12684; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12685; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12686; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12687; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12688; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12689; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12690; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12691; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12692; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12693; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12694; GFX10-WGP-NEXT:    s_endpgm
12695;
12696; GFX10-CU-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
12697; GFX10-CU:       ; %bb.0: ; %entry
12698; GFX10-CU-NEXT:    s_clause 0x1
12699; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12700; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12701; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12702; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12703; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12704; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12705; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12706; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12707; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12708; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12709; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12710; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12711; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12712; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12713; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12714; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12715; GFX10-CU-NEXT:    s_endpgm
12716;
12717; SKIP-CACHE-INV-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
12718; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12719; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12720; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12721; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12722; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12723; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12724; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12725; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12726; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12727; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12728; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12729; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12730; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12731; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12732; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12733; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12734; SKIP-CACHE-INV-NEXT:    s_endpgm
12735;
12736; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
12737; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12738; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12739; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12740; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12741; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12742; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12743; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
12744; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12745; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12746; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12747; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12748; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12749;
12750; GFX90A-TGSPLIT-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
12751; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12752; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12753; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12754; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12755; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12756; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12757; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
12758; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12759; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12760; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12761; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12762; GFX90A-TGSPLIT-NEXT:    s_endpgm
12763;
12764; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
12765; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12766; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12767; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12768; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12769; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12770; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12771; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
12772; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12773; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
12774; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12775; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12776; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12777;
12778; GFX940-TGSPLIT-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
12779; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12780; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12781; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12782; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12783; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12784; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12785; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
12786; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12787; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
12788; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12789; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12790; GFX940-TGSPLIT-NEXT:    s_endpgm
12791;
12792; GFX11-WGP-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
12793; GFX11-WGP:       ; %bb.0: ; %entry
12794; GFX11-WGP-NEXT:    s_clause 0x1
12795; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12796; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12797; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12798; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12799; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12800; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12801; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12802; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12803; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12804; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12805; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12806; GFX11-WGP-NEXT:    s_endpgm
12807;
12808; GFX11-CU-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
12809; GFX11-CU:       ; %bb.0: ; %entry
12810; GFX11-CU-NEXT:    s_clause 0x1
12811; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12812; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12813; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12814; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12815; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12816; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
12817; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12818; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12819; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12820; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12821; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12822; GFX11-CU-NEXT:    s_endpgm
12823    i32* %out, i32 %in, i32 %old) {
12824entry:
12825  %gep = getelementptr i32, i32* %out, i32 4
12826  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") release monotonic
12827  %val0 = extractvalue { i32, i1 } %val, 0
12828  store i32 %val0, i32* %out, align 4
12829  ret void
12830}
12831
12832define amdgpu_kernel void @flat_system_one_as_acq_rel_monotonic_ret_cmpxchg(
12833; GFX7-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
12834; GFX7:       ; %bb.0: ; %entry
12835; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12836; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12837; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12838; GFX7-NEXT:    s_add_u32 s4, s0, 16
12839; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12840; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12841; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12842; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12843; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12844; GFX7-NEXT:    s_waitcnt vmcnt(0)
12845; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12846; GFX7-NEXT:    s_waitcnt vmcnt(0)
12847; GFX7-NEXT:    buffer_wbinvl1_vol
12848; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12849; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12850; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12851; GFX7-NEXT:    flat_store_dword v[0:1], v2
12852; GFX7-NEXT:    s_endpgm
12853;
12854; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
12855; GFX10-WGP:       ; %bb.0: ; %entry
12856; GFX10-WGP-NEXT:    s_clause 0x1
12857; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12858; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12859; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12860; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12861; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12862; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12863; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12864; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12865; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12866; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12867; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12868; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12869; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12870; GFX10-WGP-NEXT:    buffer_gl0_inv
12871; GFX10-WGP-NEXT:    buffer_gl1_inv
12872; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12873; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12874; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12875; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12876; GFX10-WGP-NEXT:    s_endpgm
12877;
12878; GFX10-CU-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
12879; GFX10-CU:       ; %bb.0: ; %entry
12880; GFX10-CU-NEXT:    s_clause 0x1
12881; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12882; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12883; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12884; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12885; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12886; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12887; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12888; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12889; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12890; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12891; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12892; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12893; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12894; GFX10-CU-NEXT:    buffer_gl0_inv
12895; GFX10-CU-NEXT:    buffer_gl1_inv
12896; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12897; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12898; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12899; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12900; GFX10-CU-NEXT:    s_endpgm
12901;
12902; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
12903; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12904; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12905; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12906; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12907; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12908; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12909; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12910; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12911; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12912; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12913; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12914; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12915; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12916; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12917; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12918; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12919; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12920; SKIP-CACHE-INV-NEXT:    s_endpgm
12921;
12922; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
12923; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12924; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12925; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12926; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12927; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12928; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12929; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
12930; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12931; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12932; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12933; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
12934; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
12935; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12936; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12937; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12938;
12939; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
12940; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12941; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12942; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12943; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12944; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12945; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12946; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
12947; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12948; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12949; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12950; GFX90A-TGSPLIT-NEXT:    buffer_invl2
12951; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
12952; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12953; GFX90A-TGSPLIT-NEXT:    s_endpgm
12954;
12955; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
12956; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12957; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12958; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12959; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12960; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12961; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12962; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
12963; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12964; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
12965; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12966; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
12967; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12968; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12969; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12970;
12971; GFX940-TGSPLIT-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
12972; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12973; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12974; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12975; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12976; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12977; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12978; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
12979; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12980; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
12981; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12982; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
12983; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12984; GFX940-TGSPLIT-NEXT:    s_endpgm
12985;
12986; GFX11-WGP-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
12987; GFX11-WGP:       ; %bb.0: ; %entry
12988; GFX11-WGP-NEXT:    s_clause 0x1
12989; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12990; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12991; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12992; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12993; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12994; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12995; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12996; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12997; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12998; GFX11-WGP-NEXT:    buffer_gl0_inv
12999; GFX11-WGP-NEXT:    buffer_gl1_inv
13000; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13001; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
13002; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
13003; GFX11-WGP-NEXT:    s_endpgm
13004;
13005; GFX11-CU-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
13006; GFX11-CU:       ; %bb.0: ; %entry
13007; GFX11-CU-NEXT:    s_clause 0x1
13008; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13009; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13010; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13011; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
13012; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
13013; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13014; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
13015; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13016; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13017; GFX11-CU-NEXT:    buffer_gl0_inv
13018; GFX11-CU-NEXT:    buffer_gl1_inv
13019; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13020; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
13021; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
13022; GFX11-CU-NEXT:    s_endpgm
13023    i32* %out, i32 %in, i32 %old) {
13024entry:
13025  %gep = getelementptr i32, i32* %out, i32 4
13026  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel monotonic
13027  %val0 = extractvalue { i32, i1 } %val, 0
13028  store i32 %val0, i32* %out, align 4
13029  ret void
13030}
13031
13032define amdgpu_kernel void @flat_system_one_as_seq_cst_monotonic_ret_cmpxchg(
13033; GFX7-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
13034; GFX7:       ; %bb.0: ; %entry
13035; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13036; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
13037; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13038; GFX7-NEXT:    s_add_u32 s4, s0, 16
13039; GFX7-NEXT:    s_addc_u32 s5, s1, 0
13040; GFX7-NEXT:    v_mov_b32_e32 v0, s4
13041; GFX7-NEXT:    v_mov_b32_e32 v2, s2
13042; GFX7-NEXT:    v_mov_b32_e32 v1, s5
13043; GFX7-NEXT:    v_mov_b32_e32 v3, s3
13044; GFX7-NEXT:    s_waitcnt vmcnt(0)
13045; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13046; GFX7-NEXT:    s_waitcnt vmcnt(0)
13047; GFX7-NEXT:    buffer_wbinvl1_vol
13048; GFX7-NEXT:    v_mov_b32_e32 v0, s0
13049; GFX7-NEXT:    v_mov_b32_e32 v1, s1
13050; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13051; GFX7-NEXT:    flat_store_dword v[0:1], v2
13052; GFX7-NEXT:    s_endpgm
13053;
13054; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
13055; GFX10-WGP:       ; %bb.0: ; %entry
13056; GFX10-WGP-NEXT:    s_clause 0x1
13057; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13058; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13059; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13060; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
13061; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
13062; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
13063; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
13064; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
13065; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
13066; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13067; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
13068; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13069; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13070; GFX10-WGP-NEXT:    buffer_gl0_inv
13071; GFX10-WGP-NEXT:    buffer_gl1_inv
13072; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
13073; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
13074; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13075; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
13076; GFX10-WGP-NEXT:    s_endpgm
13077;
13078; GFX10-CU-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
13079; GFX10-CU:       ; %bb.0: ; %entry
13080; GFX10-CU-NEXT:    s_clause 0x1
13081; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13082; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13083; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13084; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
13085; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
13086; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
13087; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
13088; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
13089; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
13090; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13091; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
13092; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13093; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13094; GFX10-CU-NEXT:    buffer_gl0_inv
13095; GFX10-CU-NEXT:    buffer_gl1_inv
13096; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
13097; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
13098; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13099; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
13100; GFX10-CU-NEXT:    s_endpgm
13101;
13102; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
13103; SKIP-CACHE-INV:       ; %bb.0: ; %entry
13104; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13105; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
13106; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13107; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
13108; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
13109; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
13110; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
13111; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
13112; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
13113; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
13114; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13115; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
13116; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
13117; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
13118; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13119; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
13120; SKIP-CACHE-INV-NEXT:    s_endpgm
13121;
13122; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
13123; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
13124; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13125; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13126; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13127; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13128; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13129; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
13130; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13131; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13132; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13133; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
13134; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
13135; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13136; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13137; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
13138;
13139; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
13140; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
13141; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13142; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13143; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13144; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13145; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13146; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
13147; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13148; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13149; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13150; GFX90A-TGSPLIT-NEXT:    buffer_invl2
13151; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
13152; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13153; GFX90A-TGSPLIT-NEXT:    s_endpgm
13154;
13155; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
13156; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
13157; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13158; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13159; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13160; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13161; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13162; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
13163; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13164; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
13165; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13166; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
13167; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13168; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13169; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
13170;
13171; GFX940-TGSPLIT-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
13172; GFX940-TGSPLIT:       ; %bb.0: ; %entry
13173; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13174; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13175; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13176; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13177; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13178; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
13179; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13180; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
13181; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13182; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
13183; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13184; GFX940-TGSPLIT-NEXT:    s_endpgm
13185;
13186; GFX11-WGP-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
13187; GFX11-WGP:       ; %bb.0: ; %entry
13188; GFX11-WGP-NEXT:    s_clause 0x1
13189; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13190; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13191; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13192; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
13193; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
13194; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13195; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
13196; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13197; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13198; GFX11-WGP-NEXT:    buffer_gl0_inv
13199; GFX11-WGP-NEXT:    buffer_gl1_inv
13200; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13201; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
13202; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
13203; GFX11-WGP-NEXT:    s_endpgm
13204;
13205; GFX11-CU-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
13206; GFX11-CU:       ; %bb.0: ; %entry
13207; GFX11-CU-NEXT:    s_clause 0x1
13208; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13209; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13210; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13211; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
13212; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
13213; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13214; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
13215; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13216; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13217; GFX11-CU-NEXT:    buffer_gl0_inv
13218; GFX11-CU-NEXT:    buffer_gl1_inv
13219; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13220; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
13221; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
13222; GFX11-CU-NEXT:    s_endpgm
13223    i32* %out, i32 %in, i32 %old) {
13224entry:
13225  %gep = getelementptr i32, i32* %out, i32 4
13226  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst monotonic
13227  %val0 = extractvalue { i32, i1 } %val, 0
13228  store i32 %val0, i32* %out, align 4
13229  ret void
13230}
13231
13232define amdgpu_kernel void @flat_system_one_as_monotonic_acquire_ret_cmpxchg(
13233; GFX7-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
13234; GFX7:       ; %bb.0: ; %entry
13235; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13236; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
13237; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13238; GFX7-NEXT:    s_add_u32 s4, s0, 16
13239; GFX7-NEXT:    s_addc_u32 s5, s1, 0
13240; GFX7-NEXT:    v_mov_b32_e32 v0, s4
13241; GFX7-NEXT:    v_mov_b32_e32 v2, s2
13242; GFX7-NEXT:    v_mov_b32_e32 v1, s5
13243; GFX7-NEXT:    v_mov_b32_e32 v3, s3
13244; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13245; GFX7-NEXT:    s_waitcnt vmcnt(0)
13246; GFX7-NEXT:    buffer_wbinvl1_vol
13247; GFX7-NEXT:    v_mov_b32_e32 v0, s0
13248; GFX7-NEXT:    v_mov_b32_e32 v1, s1
13249; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13250; GFX7-NEXT:    flat_store_dword v[0:1], v2
13251; GFX7-NEXT:    s_endpgm
13252;
13253; GFX10-WGP-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
13254; GFX10-WGP:       ; %bb.0: ; %entry
13255; GFX10-WGP-NEXT:    s_clause 0x1
13256; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13257; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13258; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13259; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
13260; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
13261; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
13262; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
13263; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
13264; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
13265; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13266; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13267; GFX10-WGP-NEXT:    buffer_gl0_inv
13268; GFX10-WGP-NEXT:    buffer_gl1_inv
13269; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
13270; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
13271; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13272; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
13273; GFX10-WGP-NEXT:    s_endpgm
13274;
13275; GFX10-CU-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
13276; GFX10-CU:       ; %bb.0: ; %entry
13277; GFX10-CU-NEXT:    s_clause 0x1
13278; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13279; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13280; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13281; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
13282; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
13283; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
13284; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
13285; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
13286; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
13287; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13288; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13289; GFX10-CU-NEXT:    buffer_gl0_inv
13290; GFX10-CU-NEXT:    buffer_gl1_inv
13291; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
13292; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
13293; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13294; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
13295; GFX10-CU-NEXT:    s_endpgm
13296;
13297; SKIP-CACHE-INV-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
13298; SKIP-CACHE-INV:       ; %bb.0: ; %entry
13299; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13300; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
13301; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13302; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
13303; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
13304; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
13305; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
13306; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
13307; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
13308; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13309; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
13310; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
13311; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
13312; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13313; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
13314; SKIP-CACHE-INV-NEXT:    s_endpgm
13315;
13316; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
13317; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
13318; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13319; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13320; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13321; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13322; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13323; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13324; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13325; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
13326; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
13327; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13328; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13329; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
13330;
13331; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
13332; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
13333; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13334; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13335; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13336; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13337; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13338; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13339; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13340; GFX90A-TGSPLIT-NEXT:    buffer_invl2
13341; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
13342; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13343; GFX90A-TGSPLIT-NEXT:    s_endpgm
13344;
13345; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
13346; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
13347; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13348; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13349; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13350; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13351; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13352; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
13353; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13354; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
13355; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13356; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13357; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
13358;
13359; GFX940-TGSPLIT-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
13360; GFX940-TGSPLIT:       ; %bb.0: ; %entry
13361; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13362; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13363; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13364; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13365; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13366; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
13367; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13368; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
13369; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13370; GFX940-TGSPLIT-NEXT:    s_endpgm
13371;
13372; GFX11-WGP-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
13373; GFX11-WGP:       ; %bb.0: ; %entry
13374; GFX11-WGP-NEXT:    s_clause 0x1
13375; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13376; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13377; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13378; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
13379; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
13380; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13381; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13382; GFX11-WGP-NEXT:    buffer_gl0_inv
13383; GFX11-WGP-NEXT:    buffer_gl1_inv
13384; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13385; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
13386; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
13387; GFX11-WGP-NEXT:    s_endpgm
13388;
13389; GFX11-CU-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
13390; GFX11-CU:       ; %bb.0: ; %entry
13391; GFX11-CU-NEXT:    s_clause 0x1
13392; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13393; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13394; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13395; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
13396; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
13397; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13398; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13399; GFX11-CU-NEXT:    buffer_gl0_inv
13400; GFX11-CU-NEXT:    buffer_gl1_inv
13401; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13402; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
13403; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
13404; GFX11-CU-NEXT:    s_endpgm
13405    i32* %out, i32 %in, i32 %old) {
13406entry:
13407  %gep = getelementptr i32, i32* %out, i32 4
13408  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") monotonic acquire
13409  %val0 = extractvalue { i32, i1 } %val, 0
13410  store i32 %val0, i32* %out, align 4
13411  ret void
13412}
13413
13414define amdgpu_kernel void @flat_system_one_as_acquire_acquire_ret_cmpxchg(
13415; GFX7-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
13416; GFX7:       ; %bb.0: ; %entry
13417; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13418; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
13419; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13420; GFX7-NEXT:    s_add_u32 s4, s0, 16
13421; GFX7-NEXT:    s_addc_u32 s5, s1, 0
13422; GFX7-NEXT:    v_mov_b32_e32 v0, s4
13423; GFX7-NEXT:    v_mov_b32_e32 v2, s2
13424; GFX7-NEXT:    v_mov_b32_e32 v1, s5
13425; GFX7-NEXT:    v_mov_b32_e32 v3, s3
13426; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13427; GFX7-NEXT:    s_waitcnt vmcnt(0)
13428; GFX7-NEXT:    buffer_wbinvl1_vol
13429; GFX7-NEXT:    v_mov_b32_e32 v0, s0
13430; GFX7-NEXT:    v_mov_b32_e32 v1, s1
13431; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13432; GFX7-NEXT:    flat_store_dword v[0:1], v2
13433; GFX7-NEXT:    s_endpgm
13434;
13435; GFX10-WGP-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
13436; GFX10-WGP:       ; %bb.0: ; %entry
13437; GFX10-WGP-NEXT:    s_clause 0x1
13438; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13439; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13440; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13441; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
13442; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
13443; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
13444; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
13445; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
13446; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
13447; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13448; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13449; GFX10-WGP-NEXT:    buffer_gl0_inv
13450; GFX10-WGP-NEXT:    buffer_gl1_inv
13451; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
13452; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
13453; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13454; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
13455; GFX10-WGP-NEXT:    s_endpgm
13456;
13457; GFX10-CU-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
13458; GFX10-CU:       ; %bb.0: ; %entry
13459; GFX10-CU-NEXT:    s_clause 0x1
13460; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13461; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13462; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13463; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
13464; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
13465; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
13466; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
13467; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
13468; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
13469; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13470; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13471; GFX10-CU-NEXT:    buffer_gl0_inv
13472; GFX10-CU-NEXT:    buffer_gl1_inv
13473; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
13474; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
13475; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13476; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
13477; GFX10-CU-NEXT:    s_endpgm
13478;
13479; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
13480; SKIP-CACHE-INV:       ; %bb.0: ; %entry
13481; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13482; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
13483; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13484; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
13485; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
13486; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
13487; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
13488; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
13489; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
13490; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13491; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
13492; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
13493; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
13494; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13495; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
13496; SKIP-CACHE-INV-NEXT:    s_endpgm
13497;
13498; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
13499; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
13500; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13501; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13502; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13503; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13504; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13505; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13506; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13507; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
13508; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
13509; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13510; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13511; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
13512;
13513; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
13514; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
13515; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13516; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13517; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13518; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13519; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13520; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13521; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13522; GFX90A-TGSPLIT-NEXT:    buffer_invl2
13523; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
13524; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13525; GFX90A-TGSPLIT-NEXT:    s_endpgm
13526;
13527; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
13528; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
13529; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13530; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13531; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13532; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13533; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13534; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
13535; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13536; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
13537; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13538; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13539; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
13540;
13541; GFX940-TGSPLIT-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
13542; GFX940-TGSPLIT:       ; %bb.0: ; %entry
13543; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13544; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13545; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13546; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13547; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13548; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
13549; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13550; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
13551; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13552; GFX940-TGSPLIT-NEXT:    s_endpgm
13553;
13554; GFX11-WGP-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
13555; GFX11-WGP:       ; %bb.0: ; %entry
13556; GFX11-WGP-NEXT:    s_clause 0x1
13557; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13558; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13559; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13560; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
13561; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
13562; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13563; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13564; GFX11-WGP-NEXT:    buffer_gl0_inv
13565; GFX11-WGP-NEXT:    buffer_gl1_inv
13566; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13567; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
13568; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
13569; GFX11-WGP-NEXT:    s_endpgm
13570;
13571; GFX11-CU-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
13572; GFX11-CU:       ; %bb.0: ; %entry
13573; GFX11-CU-NEXT:    s_clause 0x1
13574; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13575; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13576; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13577; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
13578; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
13579; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13580; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13581; GFX11-CU-NEXT:    buffer_gl0_inv
13582; GFX11-CU-NEXT:    buffer_gl1_inv
13583; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13584; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
13585; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
13586; GFX11-CU-NEXT:    s_endpgm
13587    i32* %out, i32 %in, i32 %old) {
13588entry:
13589  %gep = getelementptr i32, i32* %out, i32 4
13590  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acquire acquire
13591  %val0 = extractvalue { i32, i1 } %val, 0
13592  store i32 %val0, i32* %out, align 4
13593  ret void
13594}
13595
13596define amdgpu_kernel void @flat_system_one_as_release_acquire_ret_cmpxchg(
13597; GFX7-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
13598; GFX7:       ; %bb.0: ; %entry
13599; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13600; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
13601; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13602; GFX7-NEXT:    s_add_u32 s4, s0, 16
13603; GFX7-NEXT:    s_addc_u32 s5, s1, 0
13604; GFX7-NEXT:    v_mov_b32_e32 v0, s4
13605; GFX7-NEXT:    v_mov_b32_e32 v2, s2
13606; GFX7-NEXT:    v_mov_b32_e32 v1, s5
13607; GFX7-NEXT:    v_mov_b32_e32 v3, s3
13608; GFX7-NEXT:    s_waitcnt vmcnt(0)
13609; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13610; GFX7-NEXT:    s_waitcnt vmcnt(0)
13611; GFX7-NEXT:    buffer_wbinvl1_vol
13612; GFX7-NEXT:    v_mov_b32_e32 v0, s0
13613; GFX7-NEXT:    v_mov_b32_e32 v1, s1
13614; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13615; GFX7-NEXT:    flat_store_dword v[0:1], v2
13616; GFX7-NEXT:    s_endpgm
13617;
13618; GFX10-WGP-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
13619; GFX10-WGP:       ; %bb.0: ; %entry
13620; GFX10-WGP-NEXT:    s_clause 0x1
13621; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13622; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13623; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13624; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
13625; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
13626; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
13627; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
13628; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
13629; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
13630; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13631; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
13632; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13633; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13634; GFX10-WGP-NEXT:    buffer_gl0_inv
13635; GFX10-WGP-NEXT:    buffer_gl1_inv
13636; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
13637; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
13638; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13639; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
13640; GFX10-WGP-NEXT:    s_endpgm
13641;
13642; GFX10-CU-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
13643; GFX10-CU:       ; %bb.0: ; %entry
13644; GFX10-CU-NEXT:    s_clause 0x1
13645; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13646; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13647; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13648; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
13649; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
13650; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
13651; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
13652; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
13653; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
13654; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13655; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
13656; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13657; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13658; GFX10-CU-NEXT:    buffer_gl0_inv
13659; GFX10-CU-NEXT:    buffer_gl1_inv
13660; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
13661; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
13662; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13663; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
13664; GFX10-CU-NEXT:    s_endpgm
13665;
13666; SKIP-CACHE-INV-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
13667; SKIP-CACHE-INV:       ; %bb.0: ; %entry
13668; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13669; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
13670; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13671; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
13672; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
13673; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
13674; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
13675; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
13676; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
13677; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
13678; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13679; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
13680; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
13681; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
13682; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13683; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
13684; SKIP-CACHE-INV-NEXT:    s_endpgm
13685;
13686; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
13687; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
13688; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13689; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13690; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13691; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13692; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13693; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
13694; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13695; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13696; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13697; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
13698; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
13699; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13700; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13701; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
13702;
13703; GFX90A-TGSPLIT-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
13704; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
13705; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13706; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13707; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13708; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13709; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13710; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
13711; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13712; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13713; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13714; GFX90A-TGSPLIT-NEXT:    buffer_invl2
13715; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
13716; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13717; GFX90A-TGSPLIT-NEXT:    s_endpgm
13718;
13719; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
13720; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
13721; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13722; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13723; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13724; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13725; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13726; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
13727; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13728; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
13729; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13730; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
13731; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13732; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13733; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
13734;
13735; GFX940-TGSPLIT-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
13736; GFX940-TGSPLIT:       ; %bb.0: ; %entry
13737; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13738; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13739; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13740; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13741; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13742; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
13743; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13744; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
13745; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13746; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
13747; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13748; GFX940-TGSPLIT-NEXT:    s_endpgm
13749;
13750; GFX11-WGP-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
13751; GFX11-WGP:       ; %bb.0: ; %entry
13752; GFX11-WGP-NEXT:    s_clause 0x1
13753; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13754; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13755; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13756; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
13757; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
13758; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13759; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
13760; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13761; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13762; GFX11-WGP-NEXT:    buffer_gl0_inv
13763; GFX11-WGP-NEXT:    buffer_gl1_inv
13764; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13765; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
13766; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
13767; GFX11-WGP-NEXT:    s_endpgm
13768;
13769; GFX11-CU-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
13770; GFX11-CU:       ; %bb.0: ; %entry
13771; GFX11-CU-NEXT:    s_clause 0x1
13772; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13773; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13774; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13775; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
13776; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
13777; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13778; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
13779; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13780; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13781; GFX11-CU-NEXT:    buffer_gl0_inv
13782; GFX11-CU-NEXT:    buffer_gl1_inv
13783; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13784; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
13785; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
13786; GFX11-CU-NEXT:    s_endpgm
13787    i32* %out, i32 %in, i32 %old) {
13788entry:
13789  %gep = getelementptr i32, i32* %out, i32 4
13790  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") release acquire
13791  %val0 = extractvalue { i32, i1 } %val, 0
13792  store i32 %val0, i32* %out, align 4
13793  ret void
13794}
13795
13796define amdgpu_kernel void @flat_system_one_as_acq_rel_acquire_ret_cmpxchg(
13797; GFX7-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
13798; GFX7:       ; %bb.0: ; %entry
13799; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13800; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
13801; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13802; GFX7-NEXT:    s_add_u32 s4, s0, 16
13803; GFX7-NEXT:    s_addc_u32 s5, s1, 0
13804; GFX7-NEXT:    v_mov_b32_e32 v0, s4
13805; GFX7-NEXT:    v_mov_b32_e32 v2, s2
13806; GFX7-NEXT:    v_mov_b32_e32 v1, s5
13807; GFX7-NEXT:    v_mov_b32_e32 v3, s3
13808; GFX7-NEXT:    s_waitcnt vmcnt(0)
13809; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13810; GFX7-NEXT:    s_waitcnt vmcnt(0)
13811; GFX7-NEXT:    buffer_wbinvl1_vol
13812; GFX7-NEXT:    v_mov_b32_e32 v0, s0
13813; GFX7-NEXT:    v_mov_b32_e32 v1, s1
13814; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13815; GFX7-NEXT:    flat_store_dword v[0:1], v2
13816; GFX7-NEXT:    s_endpgm
13817;
13818; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
13819; GFX10-WGP:       ; %bb.0: ; %entry
13820; GFX10-WGP-NEXT:    s_clause 0x1
13821; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13822; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13823; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13824; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
13825; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
13826; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
13827; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
13828; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
13829; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
13830; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13831; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
13832; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13833; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13834; GFX10-WGP-NEXT:    buffer_gl0_inv
13835; GFX10-WGP-NEXT:    buffer_gl1_inv
13836; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
13837; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
13838; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13839; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
13840; GFX10-WGP-NEXT:    s_endpgm
13841;
13842; GFX10-CU-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
13843; GFX10-CU:       ; %bb.0: ; %entry
13844; GFX10-CU-NEXT:    s_clause 0x1
13845; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13846; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13847; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13848; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
13849; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
13850; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
13851; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
13852; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
13853; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
13854; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13855; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
13856; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13857; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13858; GFX10-CU-NEXT:    buffer_gl0_inv
13859; GFX10-CU-NEXT:    buffer_gl1_inv
13860; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
13861; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
13862; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13863; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
13864; GFX10-CU-NEXT:    s_endpgm
13865;
13866; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
13867; SKIP-CACHE-INV:       ; %bb.0: ; %entry
13868; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13869; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
13870; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13871; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
13872; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
13873; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
13874; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
13875; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
13876; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
13877; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
13878; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13879; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
13880; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
13881; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
13882; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13883; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
13884; SKIP-CACHE-INV-NEXT:    s_endpgm
13885;
13886; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
13887; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
13888; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13889; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13890; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13891; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13892; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13893; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
13894; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13895; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13896; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13897; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
13898; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
13899; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13900; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13901; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
13902;
13903; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
13904; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
13905; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13906; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13907; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13908; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13909; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13910; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
13911; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13912; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13913; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13914; GFX90A-TGSPLIT-NEXT:    buffer_invl2
13915; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
13916; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13917; GFX90A-TGSPLIT-NEXT:    s_endpgm
13918;
13919; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
13920; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
13921; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13922; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13923; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13924; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13925; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13926; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
13927; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13928; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
13929; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13930; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
13931; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13932; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13933; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
13934;
13935; GFX940-TGSPLIT-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
13936; GFX940-TGSPLIT:       ; %bb.0: ; %entry
13937; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13938; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13939; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13940; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13941; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13942; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
13943; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13944; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
13945; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13946; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
13947; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13948; GFX940-TGSPLIT-NEXT:    s_endpgm
13949;
13950; GFX11-WGP-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
13951; GFX11-WGP:       ; %bb.0: ; %entry
13952; GFX11-WGP-NEXT:    s_clause 0x1
13953; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13954; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13955; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13956; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
13957; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
13958; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13959; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
13960; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13961; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13962; GFX11-WGP-NEXT:    buffer_gl0_inv
13963; GFX11-WGP-NEXT:    buffer_gl1_inv
13964; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13965; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
13966; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
13967; GFX11-WGP-NEXT:    s_endpgm
13968;
13969; GFX11-CU-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
13970; GFX11-CU:       ; %bb.0: ; %entry
13971; GFX11-CU-NEXT:    s_clause 0x1
13972; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13973; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13974; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13975; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
13976; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
13977; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13978; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
13979; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13980; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13981; GFX11-CU-NEXT:    buffer_gl0_inv
13982; GFX11-CU-NEXT:    buffer_gl1_inv
13983; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13984; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
13985; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
13986; GFX11-CU-NEXT:    s_endpgm
13987    i32* %out, i32 %in, i32 %old) {
13988entry:
13989  %gep = getelementptr i32, i32* %out, i32 4
13990  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel acquire
13991  %val0 = extractvalue { i32, i1 } %val, 0
13992  store i32 %val0, i32* %out, align 4
13993  ret void
13994}
13995
13996define amdgpu_kernel void @flat_system_one_as_seq_cst_acquire_ret_cmpxchg(
13997; GFX7-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
13998; GFX7:       ; %bb.0: ; %entry
13999; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14000; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
14001; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14002; GFX7-NEXT:    s_add_u32 s4, s0, 16
14003; GFX7-NEXT:    s_addc_u32 s5, s1, 0
14004; GFX7-NEXT:    v_mov_b32_e32 v0, s4
14005; GFX7-NEXT:    v_mov_b32_e32 v2, s2
14006; GFX7-NEXT:    v_mov_b32_e32 v1, s5
14007; GFX7-NEXT:    v_mov_b32_e32 v3, s3
14008; GFX7-NEXT:    s_waitcnt vmcnt(0)
14009; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14010; GFX7-NEXT:    s_waitcnt vmcnt(0)
14011; GFX7-NEXT:    buffer_wbinvl1_vol
14012; GFX7-NEXT:    v_mov_b32_e32 v0, s0
14013; GFX7-NEXT:    v_mov_b32_e32 v1, s1
14014; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14015; GFX7-NEXT:    flat_store_dword v[0:1], v2
14016; GFX7-NEXT:    s_endpgm
14017;
14018; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
14019; GFX10-WGP:       ; %bb.0: ; %entry
14020; GFX10-WGP-NEXT:    s_clause 0x1
14021; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14022; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14023; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14024; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
14025; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
14026; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
14027; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
14028; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
14029; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
14030; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14031; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14032; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14033; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14034; GFX10-WGP-NEXT:    buffer_gl0_inv
14035; GFX10-WGP-NEXT:    buffer_gl1_inv
14036; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
14037; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
14038; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14039; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
14040; GFX10-WGP-NEXT:    s_endpgm
14041;
14042; GFX10-CU-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
14043; GFX10-CU:       ; %bb.0: ; %entry
14044; GFX10-CU-NEXT:    s_clause 0x1
14045; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14046; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14047; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14048; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
14049; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
14050; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
14051; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
14052; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
14053; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
14054; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14055; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14056; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14057; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14058; GFX10-CU-NEXT:    buffer_gl0_inv
14059; GFX10-CU-NEXT:    buffer_gl1_inv
14060; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
14061; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
14062; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14063; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
14064; GFX10-CU-NEXT:    s_endpgm
14065;
14066; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
14067; SKIP-CACHE-INV:       ; %bb.0: ; %entry
14068; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14069; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
14070; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14071; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
14072; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
14073; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
14074; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
14075; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
14076; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
14077; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14078; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14079; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14080; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
14081; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
14082; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14083; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
14084; SKIP-CACHE-INV-NEXT:    s_endpgm
14085;
14086; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
14087; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
14088; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14089; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14090; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14091; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14092; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14093; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
14094; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14095; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14096; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14097; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
14098; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
14099; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14100; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14101; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
14102;
14103; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
14104; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
14105; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14106; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14107; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14108; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14109; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14110; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
14111; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14112; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14113; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14114; GFX90A-TGSPLIT-NEXT:    buffer_invl2
14115; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
14116; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14117; GFX90A-TGSPLIT-NEXT:    s_endpgm
14118;
14119; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
14120; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
14121; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14122; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14123; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14124; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14125; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14126; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
14127; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14128; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
14129; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14130; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
14131; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14132; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14133; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
14134;
14135; GFX940-TGSPLIT-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
14136; GFX940-TGSPLIT:       ; %bb.0: ; %entry
14137; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14138; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14139; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14140; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14141; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14142; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
14143; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14144; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
14145; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14146; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
14147; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14148; GFX940-TGSPLIT-NEXT:    s_endpgm
14149;
14150; GFX11-WGP-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
14151; GFX11-WGP:       ; %bb.0: ; %entry
14152; GFX11-WGP-NEXT:    s_clause 0x1
14153; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14154; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14155; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14156; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
14157; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
14158; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14159; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14160; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14161; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14162; GFX11-WGP-NEXT:    buffer_gl0_inv
14163; GFX11-WGP-NEXT:    buffer_gl1_inv
14164; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14165; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
14166; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
14167; GFX11-WGP-NEXT:    s_endpgm
14168;
14169; GFX11-CU-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
14170; GFX11-CU:       ; %bb.0: ; %entry
14171; GFX11-CU-NEXT:    s_clause 0x1
14172; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14173; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14174; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14175; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
14176; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
14177; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14178; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14179; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14180; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14181; GFX11-CU-NEXT:    buffer_gl0_inv
14182; GFX11-CU-NEXT:    buffer_gl1_inv
14183; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14184; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
14185; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
14186; GFX11-CU-NEXT:    s_endpgm
14187    i32* %out, i32 %in, i32 %old) {
14188entry:
14189  %gep = getelementptr i32, i32* %out, i32 4
14190  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst acquire
14191  %val0 = extractvalue { i32, i1 } %val, 0
14192  store i32 %val0, i32* %out, align 4
14193  ret void
14194}
14195
14196define amdgpu_kernel void @flat_system_one_as_monotonic_seq_cst_ret_cmpxchg(
14197; GFX7-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
14198; GFX7:       ; %bb.0: ; %entry
14199; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14200; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
14201; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14202; GFX7-NEXT:    s_add_u32 s4, s0, 16
14203; GFX7-NEXT:    s_addc_u32 s5, s1, 0
14204; GFX7-NEXT:    v_mov_b32_e32 v0, s4
14205; GFX7-NEXT:    v_mov_b32_e32 v2, s2
14206; GFX7-NEXT:    v_mov_b32_e32 v1, s5
14207; GFX7-NEXT:    v_mov_b32_e32 v3, s3
14208; GFX7-NEXT:    s_waitcnt vmcnt(0)
14209; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14210; GFX7-NEXT:    s_waitcnt vmcnt(0)
14211; GFX7-NEXT:    buffer_wbinvl1_vol
14212; GFX7-NEXT:    v_mov_b32_e32 v0, s0
14213; GFX7-NEXT:    v_mov_b32_e32 v1, s1
14214; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14215; GFX7-NEXT:    flat_store_dword v[0:1], v2
14216; GFX7-NEXT:    s_endpgm
14217;
14218; GFX10-WGP-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
14219; GFX10-WGP:       ; %bb.0: ; %entry
14220; GFX10-WGP-NEXT:    s_clause 0x1
14221; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14222; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14223; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14224; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
14225; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
14226; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
14227; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
14228; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
14229; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
14230; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14231; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14232; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14233; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14234; GFX10-WGP-NEXT:    buffer_gl0_inv
14235; GFX10-WGP-NEXT:    buffer_gl1_inv
14236; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
14237; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
14238; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14239; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
14240; GFX10-WGP-NEXT:    s_endpgm
14241;
14242; GFX10-CU-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
14243; GFX10-CU:       ; %bb.0: ; %entry
14244; GFX10-CU-NEXT:    s_clause 0x1
14245; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14246; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14247; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14248; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
14249; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
14250; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
14251; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
14252; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
14253; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
14254; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14255; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14256; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14257; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14258; GFX10-CU-NEXT:    buffer_gl0_inv
14259; GFX10-CU-NEXT:    buffer_gl1_inv
14260; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
14261; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
14262; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14263; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
14264; GFX10-CU-NEXT:    s_endpgm
14265;
14266; SKIP-CACHE-INV-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
14267; SKIP-CACHE-INV:       ; %bb.0: ; %entry
14268; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14269; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
14270; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14271; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
14272; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
14273; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
14274; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
14275; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
14276; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
14277; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14278; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14279; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14280; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
14281; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
14282; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14283; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
14284; SKIP-CACHE-INV-NEXT:    s_endpgm
14285;
14286; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
14287; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
14288; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14289; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14290; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14291; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14292; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14293; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
14294; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14295; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14296; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14297; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
14298; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
14299; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14300; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14301; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
14302;
14303; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
14304; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
14305; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14306; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14307; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14308; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14309; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14310; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
14311; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14312; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14313; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14314; GFX90A-TGSPLIT-NEXT:    buffer_invl2
14315; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
14316; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14317; GFX90A-TGSPLIT-NEXT:    s_endpgm
14318;
14319; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
14320; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
14321; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14322; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14323; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14324; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14325; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14326; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
14327; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14328; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
14329; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14330; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
14331; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14332; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14333; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
14334;
14335; GFX940-TGSPLIT-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
14336; GFX940-TGSPLIT:       ; %bb.0: ; %entry
14337; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14338; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14339; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14340; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14341; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14342; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
14343; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14344; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
14345; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14346; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
14347; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14348; GFX940-TGSPLIT-NEXT:    s_endpgm
14349;
14350; GFX11-WGP-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
14351; GFX11-WGP:       ; %bb.0: ; %entry
14352; GFX11-WGP-NEXT:    s_clause 0x1
14353; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14354; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14355; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14356; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
14357; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
14358; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14359; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14360; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14361; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14362; GFX11-WGP-NEXT:    buffer_gl0_inv
14363; GFX11-WGP-NEXT:    buffer_gl1_inv
14364; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14365; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
14366; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
14367; GFX11-WGP-NEXT:    s_endpgm
14368;
14369; GFX11-CU-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
14370; GFX11-CU:       ; %bb.0: ; %entry
14371; GFX11-CU-NEXT:    s_clause 0x1
14372; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14373; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14374; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14375; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
14376; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
14377; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14378; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14379; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14380; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14381; GFX11-CU-NEXT:    buffer_gl0_inv
14382; GFX11-CU-NEXT:    buffer_gl1_inv
14383; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14384; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
14385; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
14386; GFX11-CU-NEXT:    s_endpgm
14387    i32* %out, i32 %in, i32 %old) {
14388entry:
14389  %gep = getelementptr i32, i32* %out, i32 4
14390  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") monotonic seq_cst
14391  %val0 = extractvalue { i32, i1 } %val, 0
14392  store i32 %val0, i32* %out, align 4
14393  ret void
14394}
14395
14396define amdgpu_kernel void @flat_system_one_as_acquire_seq_cst_ret_cmpxchg(
14397; GFX7-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
14398; GFX7:       ; %bb.0: ; %entry
14399; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14400; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
14401; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14402; GFX7-NEXT:    s_add_u32 s4, s0, 16
14403; GFX7-NEXT:    s_addc_u32 s5, s1, 0
14404; GFX7-NEXT:    v_mov_b32_e32 v0, s4
14405; GFX7-NEXT:    v_mov_b32_e32 v2, s2
14406; GFX7-NEXT:    v_mov_b32_e32 v1, s5
14407; GFX7-NEXT:    v_mov_b32_e32 v3, s3
14408; GFX7-NEXT:    s_waitcnt vmcnt(0)
14409; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14410; GFX7-NEXT:    s_waitcnt vmcnt(0)
14411; GFX7-NEXT:    buffer_wbinvl1_vol
14412; GFX7-NEXT:    v_mov_b32_e32 v0, s0
14413; GFX7-NEXT:    v_mov_b32_e32 v1, s1
14414; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14415; GFX7-NEXT:    flat_store_dword v[0:1], v2
14416; GFX7-NEXT:    s_endpgm
14417;
14418; GFX10-WGP-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
14419; GFX10-WGP:       ; %bb.0: ; %entry
14420; GFX10-WGP-NEXT:    s_clause 0x1
14421; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14422; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14423; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14424; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
14425; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
14426; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
14427; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
14428; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
14429; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
14430; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14431; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14432; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14433; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14434; GFX10-WGP-NEXT:    buffer_gl0_inv
14435; GFX10-WGP-NEXT:    buffer_gl1_inv
14436; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
14437; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
14438; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14439; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
14440; GFX10-WGP-NEXT:    s_endpgm
14441;
14442; GFX10-CU-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
14443; GFX10-CU:       ; %bb.0: ; %entry
14444; GFX10-CU-NEXT:    s_clause 0x1
14445; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14446; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14447; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14448; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
14449; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
14450; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
14451; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
14452; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
14453; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
14454; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14455; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14456; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14457; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14458; GFX10-CU-NEXT:    buffer_gl0_inv
14459; GFX10-CU-NEXT:    buffer_gl1_inv
14460; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
14461; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
14462; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14463; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
14464; GFX10-CU-NEXT:    s_endpgm
14465;
14466; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
14467; SKIP-CACHE-INV:       ; %bb.0: ; %entry
14468; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14469; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
14470; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14471; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
14472; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
14473; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
14474; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
14475; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
14476; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
14477; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14478; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14479; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14480; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
14481; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
14482; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14483; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
14484; SKIP-CACHE-INV-NEXT:    s_endpgm
14485;
14486; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
14487; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
14488; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14489; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14490; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14491; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14492; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14493; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
14494; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14495; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14496; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14497; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
14498; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
14499; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14500; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14501; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
14502;
14503; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
14504; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
14505; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14506; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14507; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14508; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14509; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14510; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
14511; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14512; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14513; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14514; GFX90A-TGSPLIT-NEXT:    buffer_invl2
14515; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
14516; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14517; GFX90A-TGSPLIT-NEXT:    s_endpgm
14518;
14519; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
14520; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
14521; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14522; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14523; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14524; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14525; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14526; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
14527; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14528; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
14529; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14530; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
14531; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14532; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14533; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
14534;
14535; GFX940-TGSPLIT-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
14536; GFX940-TGSPLIT:       ; %bb.0: ; %entry
14537; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14538; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14539; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14540; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14541; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14542; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
14543; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14544; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
14545; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14546; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
14547; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14548; GFX940-TGSPLIT-NEXT:    s_endpgm
14549;
14550; GFX11-WGP-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
14551; GFX11-WGP:       ; %bb.0: ; %entry
14552; GFX11-WGP-NEXT:    s_clause 0x1
14553; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14554; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14555; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14556; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
14557; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
14558; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14559; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14560; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14561; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14562; GFX11-WGP-NEXT:    buffer_gl0_inv
14563; GFX11-WGP-NEXT:    buffer_gl1_inv
14564; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14565; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
14566; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
14567; GFX11-WGP-NEXT:    s_endpgm
14568;
14569; GFX11-CU-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
14570; GFX11-CU:       ; %bb.0: ; %entry
14571; GFX11-CU-NEXT:    s_clause 0x1
14572; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14573; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14574; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14575; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
14576; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
14577; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14578; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14579; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14580; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14581; GFX11-CU-NEXT:    buffer_gl0_inv
14582; GFX11-CU-NEXT:    buffer_gl1_inv
14583; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14584; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
14585; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
14586; GFX11-CU-NEXT:    s_endpgm
14587    i32* %out, i32 %in, i32 %old) {
14588entry:
14589  %gep = getelementptr i32, i32* %out, i32 4
14590  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acquire seq_cst
14591  %val0 = extractvalue { i32, i1 } %val, 0
14592  store i32 %val0, i32* %out, align 4
14593  ret void
14594}
14595
14596define amdgpu_kernel void @flat_system_one_as_release_seq_cst_ret_cmpxchg(
14597; GFX7-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
14598; GFX7:       ; %bb.0: ; %entry
14599; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14600; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
14601; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14602; GFX7-NEXT:    s_add_u32 s4, s0, 16
14603; GFX7-NEXT:    s_addc_u32 s5, s1, 0
14604; GFX7-NEXT:    v_mov_b32_e32 v0, s4
14605; GFX7-NEXT:    v_mov_b32_e32 v2, s2
14606; GFX7-NEXT:    v_mov_b32_e32 v1, s5
14607; GFX7-NEXT:    v_mov_b32_e32 v3, s3
14608; GFX7-NEXT:    s_waitcnt vmcnt(0)
14609; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14610; GFX7-NEXT:    s_waitcnt vmcnt(0)
14611; GFX7-NEXT:    buffer_wbinvl1_vol
14612; GFX7-NEXT:    v_mov_b32_e32 v0, s0
14613; GFX7-NEXT:    v_mov_b32_e32 v1, s1
14614; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14615; GFX7-NEXT:    flat_store_dword v[0:1], v2
14616; GFX7-NEXT:    s_endpgm
14617;
14618; GFX10-WGP-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
14619; GFX10-WGP:       ; %bb.0: ; %entry
14620; GFX10-WGP-NEXT:    s_clause 0x1
14621; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14622; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14623; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14624; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
14625; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
14626; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
14627; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
14628; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
14629; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
14630; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14631; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14632; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14633; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14634; GFX10-WGP-NEXT:    buffer_gl0_inv
14635; GFX10-WGP-NEXT:    buffer_gl1_inv
14636; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
14637; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
14638; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14639; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
14640; GFX10-WGP-NEXT:    s_endpgm
14641;
14642; GFX10-CU-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
14643; GFX10-CU:       ; %bb.0: ; %entry
14644; GFX10-CU-NEXT:    s_clause 0x1
14645; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14646; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14647; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14648; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
14649; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
14650; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
14651; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
14652; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
14653; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
14654; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14655; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14656; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14657; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14658; GFX10-CU-NEXT:    buffer_gl0_inv
14659; GFX10-CU-NEXT:    buffer_gl1_inv
14660; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
14661; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
14662; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14663; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
14664; GFX10-CU-NEXT:    s_endpgm
14665;
14666; SKIP-CACHE-INV-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
14667; SKIP-CACHE-INV:       ; %bb.0: ; %entry
14668; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14669; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
14670; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14671; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
14672; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
14673; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
14674; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
14675; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
14676; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
14677; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14678; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14679; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14680; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
14681; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
14682; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14683; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
14684; SKIP-CACHE-INV-NEXT:    s_endpgm
14685;
14686; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
14687; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
14688; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14689; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14690; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14691; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14692; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14693; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
14694; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14695; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14696; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14697; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
14698; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
14699; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14700; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14701; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
14702;
14703; GFX90A-TGSPLIT-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
14704; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
14705; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14706; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14707; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14708; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14709; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14710; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
14711; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14712; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14713; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14714; GFX90A-TGSPLIT-NEXT:    buffer_invl2
14715; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
14716; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14717; GFX90A-TGSPLIT-NEXT:    s_endpgm
14718;
14719; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
14720; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
14721; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14722; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14723; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14724; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14725; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14726; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
14727; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14728; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
14729; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14730; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
14731; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14732; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14733; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
14734;
14735; GFX940-TGSPLIT-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
14736; GFX940-TGSPLIT:       ; %bb.0: ; %entry
14737; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14738; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14739; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14740; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14741; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14742; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
14743; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14744; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
14745; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14746; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
14747; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14748; GFX940-TGSPLIT-NEXT:    s_endpgm
14749;
14750; GFX11-WGP-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
14751; GFX11-WGP:       ; %bb.0: ; %entry
14752; GFX11-WGP-NEXT:    s_clause 0x1
14753; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14754; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14755; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14756; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
14757; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
14758; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14759; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14760; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14761; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14762; GFX11-WGP-NEXT:    buffer_gl0_inv
14763; GFX11-WGP-NEXT:    buffer_gl1_inv
14764; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14765; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
14766; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
14767; GFX11-WGP-NEXT:    s_endpgm
14768;
14769; GFX11-CU-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
14770; GFX11-CU:       ; %bb.0: ; %entry
14771; GFX11-CU-NEXT:    s_clause 0x1
14772; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14773; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14774; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14775; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
14776; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
14777; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14778; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14779; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14780; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14781; GFX11-CU-NEXT:    buffer_gl0_inv
14782; GFX11-CU-NEXT:    buffer_gl1_inv
14783; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14784; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
14785; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
14786; GFX11-CU-NEXT:    s_endpgm
14787    i32* %out, i32 %in, i32 %old) {
14788entry:
14789  %gep = getelementptr i32, i32* %out, i32 4
14790  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") release seq_cst
14791  %val0 = extractvalue { i32, i1 } %val, 0
14792  store i32 %val0, i32* %out, align 4
14793  ret void
14794}
14795
14796define amdgpu_kernel void @flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg(
14797; GFX7-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
14798; GFX7:       ; %bb.0: ; %entry
14799; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14800; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
14801; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14802; GFX7-NEXT:    s_add_u32 s4, s0, 16
14803; GFX7-NEXT:    s_addc_u32 s5, s1, 0
14804; GFX7-NEXT:    v_mov_b32_e32 v0, s4
14805; GFX7-NEXT:    v_mov_b32_e32 v2, s2
14806; GFX7-NEXT:    v_mov_b32_e32 v1, s5
14807; GFX7-NEXT:    v_mov_b32_e32 v3, s3
14808; GFX7-NEXT:    s_waitcnt vmcnt(0)
14809; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14810; GFX7-NEXT:    s_waitcnt vmcnt(0)
14811; GFX7-NEXT:    buffer_wbinvl1_vol
14812; GFX7-NEXT:    v_mov_b32_e32 v0, s0
14813; GFX7-NEXT:    v_mov_b32_e32 v1, s1
14814; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14815; GFX7-NEXT:    flat_store_dword v[0:1], v2
14816; GFX7-NEXT:    s_endpgm
14817;
14818; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
14819; GFX10-WGP:       ; %bb.0: ; %entry
14820; GFX10-WGP-NEXT:    s_clause 0x1
14821; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14822; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14823; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14824; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
14825; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
14826; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
14827; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
14828; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
14829; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
14830; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14831; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14832; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14833; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14834; GFX10-WGP-NEXT:    buffer_gl0_inv
14835; GFX10-WGP-NEXT:    buffer_gl1_inv
14836; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
14837; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
14838; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14839; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
14840; GFX10-WGP-NEXT:    s_endpgm
14841;
14842; GFX10-CU-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
14843; GFX10-CU:       ; %bb.0: ; %entry
14844; GFX10-CU-NEXT:    s_clause 0x1
14845; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14846; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14847; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14848; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
14849; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
14850; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
14851; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
14852; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
14853; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
14854; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14855; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14856; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14857; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14858; GFX10-CU-NEXT:    buffer_gl0_inv
14859; GFX10-CU-NEXT:    buffer_gl1_inv
14860; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
14861; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
14862; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14863; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
14864; GFX10-CU-NEXT:    s_endpgm
14865;
14866; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
14867; SKIP-CACHE-INV:       ; %bb.0: ; %entry
14868; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14869; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
14870; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14871; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
14872; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
14873; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
14874; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
14875; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
14876; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
14877; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14878; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14879; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14880; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
14881; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
14882; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14883; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
14884; SKIP-CACHE-INV-NEXT:    s_endpgm
14885;
14886; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
14887; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
14888; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14889; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14890; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14891; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14892; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14893; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
14894; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14895; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14896; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14897; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
14898; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
14899; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14900; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14901; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
14902;
14903; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
14904; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
14905; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14906; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14907; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14908; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14909; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14910; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
14911; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14912; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14913; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14914; GFX90A-TGSPLIT-NEXT:    buffer_invl2
14915; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
14916; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14917; GFX90A-TGSPLIT-NEXT:    s_endpgm
14918;
14919; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
14920; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
14921; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14922; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14923; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14924; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14925; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14926; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
14927; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14928; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
14929; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14930; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
14931; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14932; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14933; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
14934;
14935; GFX940-TGSPLIT-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
14936; GFX940-TGSPLIT:       ; %bb.0: ; %entry
14937; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14938; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14939; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14940; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14941; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14942; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
14943; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14944; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
14945; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14946; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
14947; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14948; GFX940-TGSPLIT-NEXT:    s_endpgm
14949;
14950; GFX11-WGP-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
14951; GFX11-WGP:       ; %bb.0: ; %entry
14952; GFX11-WGP-NEXT:    s_clause 0x1
14953; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14954; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14955; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14956; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
14957; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
14958; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14959; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14960; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14961; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14962; GFX11-WGP-NEXT:    buffer_gl0_inv
14963; GFX11-WGP-NEXT:    buffer_gl1_inv
14964; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14965; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
14966; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
14967; GFX11-WGP-NEXT:    s_endpgm
14968;
14969; GFX11-CU-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
14970; GFX11-CU:       ; %bb.0: ; %entry
14971; GFX11-CU-NEXT:    s_clause 0x1
14972; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14973; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14974; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14975; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
14976; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
14977; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14978; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14979; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14980; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14981; GFX11-CU-NEXT:    buffer_gl0_inv
14982; GFX11-CU-NEXT:    buffer_gl1_inv
14983; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14984; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
14985; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
14986; GFX11-CU-NEXT:    s_endpgm
14987    i32* %out, i32 %in, i32 %old) {
14988entry:
14989  %gep = getelementptr i32, i32* %out, i32 4
14990  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel seq_cst
14991  %val0 = extractvalue { i32, i1 } %val, 0
14992  store i32 %val0, i32* %out, align 4
14993  ret void
14994}
14995
14996define amdgpu_kernel void @flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg(
14997; GFX7-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
14998; GFX7:       ; %bb.0: ; %entry
14999; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
15000; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
15001; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
15002; GFX7-NEXT:    s_add_u32 s4, s0, 16
15003; GFX7-NEXT:    s_addc_u32 s5, s1, 0
15004; GFX7-NEXT:    v_mov_b32_e32 v0, s4
15005; GFX7-NEXT:    v_mov_b32_e32 v2, s2
15006; GFX7-NEXT:    v_mov_b32_e32 v1, s5
15007; GFX7-NEXT:    v_mov_b32_e32 v3, s3
15008; GFX7-NEXT:    s_waitcnt vmcnt(0)
15009; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
15010; GFX7-NEXT:    s_waitcnt vmcnt(0)
15011; GFX7-NEXT:    buffer_wbinvl1_vol
15012; GFX7-NEXT:    v_mov_b32_e32 v0, s0
15013; GFX7-NEXT:    v_mov_b32_e32 v1, s1
15014; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
15015; GFX7-NEXT:    flat_store_dword v[0:1], v2
15016; GFX7-NEXT:    s_endpgm
15017;
15018; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
15019; GFX10-WGP:       ; %bb.0: ; %entry
15020; GFX10-WGP-NEXT:    s_clause 0x1
15021; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
15022; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
15023; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
15024; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
15025; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
15026; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
15027; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
15028; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
15029; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
15030; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
15031; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
15032; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
15033; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
15034; GFX10-WGP-NEXT:    buffer_gl0_inv
15035; GFX10-WGP-NEXT:    buffer_gl1_inv
15036; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
15037; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
15038; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
15039; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
15040; GFX10-WGP-NEXT:    s_endpgm
15041;
15042; GFX10-CU-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
15043; GFX10-CU:       ; %bb.0: ; %entry
15044; GFX10-CU-NEXT:    s_clause 0x1
15045; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
15046; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
15047; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
15048; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
15049; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
15050; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
15051; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
15052; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
15053; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
15054; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
15055; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
15056; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
15057; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
15058; GFX10-CU-NEXT:    buffer_gl0_inv
15059; GFX10-CU-NEXT:    buffer_gl1_inv
15060; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
15061; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
15062; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
15063; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
15064; GFX10-CU-NEXT:    s_endpgm
15065;
15066; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
15067; SKIP-CACHE-INV:       ; %bb.0: ; %entry
15068; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
15069; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
15070; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
15071; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
15072; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
15073; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
15074; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
15075; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
15076; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
15077; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
15078; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
15079; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
15080; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
15081; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
15082; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
15083; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
15084; SKIP-CACHE-INV-NEXT:    s_endpgm
15085;
15086; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
15087; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
15088; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
15089; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
15090; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
15091; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
15092; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
15093; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
15094; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
15095; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
15096; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
15097; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
15098; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
15099; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
15100; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
15101; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
15102;
15103; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
15104; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
15105; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
15106; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
15107; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
15108; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
15109; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
15110; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
15111; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
15112; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
15113; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
15114; GFX90A-TGSPLIT-NEXT:    buffer_invl2
15115; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
15116; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
15117; GFX90A-TGSPLIT-NEXT:    s_endpgm
15118;
15119; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
15120; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
15121; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
15122; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
15123; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
15124; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
15125; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
15126; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
15127; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
15128; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
15129; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
15130; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
15131; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
15132; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
15133; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
15134;
15135; GFX940-TGSPLIT-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
15136; GFX940-TGSPLIT:       ; %bb.0: ; %entry
15137; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
15138; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
15139; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
15140; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
15141; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
15142; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
15143; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
15144; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
15145; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
15146; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
15147; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
15148; GFX940-TGSPLIT-NEXT:    s_endpgm
15149;
15150; GFX11-WGP-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
15151; GFX11-WGP:       ; %bb.0: ; %entry
15152; GFX11-WGP-NEXT:    s_clause 0x1
15153; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
15154; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
15155; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
15156; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
15157; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
15158; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
15159; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
15160; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
15161; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
15162; GFX11-WGP-NEXT:    buffer_gl0_inv
15163; GFX11-WGP-NEXT:    buffer_gl1_inv
15164; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
15165; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
15166; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
15167; GFX11-WGP-NEXT:    s_endpgm
15168;
15169; GFX11-CU-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
15170; GFX11-CU:       ; %bb.0: ; %entry
15171; GFX11-CU-NEXT:    s_clause 0x1
15172; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
15173; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
15174; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
15175; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
15176; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
15177; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
15178; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
15179; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
15180; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
15181; GFX11-CU-NEXT:    buffer_gl0_inv
15182; GFX11-CU-NEXT:    buffer_gl1_inv
15183; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
15184; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
15185; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
15186; GFX11-CU-NEXT:    s_endpgm
15187    i32* %out, i32 %in, i32 %old) {
15188entry:
15189  %gep = getelementptr i32, i32* %out, i32 4
15190  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst seq_cst
15191  %val0 = extractvalue { i32, i1 } %val, 0
15192  store i32 %val0, i32* %out, align 4
15193  ret void
15194}
15195
15196