1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
3; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
4; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
5; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
6; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-NOTTGSPLIT %s
7; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-TGSPLIT %s
8; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-NOTTGSPLIT %s
9; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
10; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11-WGP %s
11; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11-CU %s
12
13define amdgpu_kernel void @flat_system_unordered_load(
14; GFX7-LABEL: flat_system_unordered_load:
15; GFX7:       ; %bb.0: ; %entry
16; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
17; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
18; GFX7-NEXT:    v_mov_b32_e32 v0, s0
19; GFX7-NEXT:    v_mov_b32_e32 v1, s1
20; GFX7-NEXT:    flat_load_dword v2, v[0:1]
21; GFX7-NEXT:    v_mov_b32_e32 v0, s2
22; GFX7-NEXT:    v_mov_b32_e32 v1, s3
23; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
24; GFX7-NEXT:    flat_store_dword v[0:1], v2
25; GFX7-NEXT:    s_endpgm
26;
27; GFX10-WGP-LABEL: flat_system_unordered_load:
28; GFX10-WGP:       ; %bb.0: ; %entry
29; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
30; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
31; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
32; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
33; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
34; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
35; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
36; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
37; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
38; GFX10-WGP-NEXT:    s_endpgm
39;
40; GFX10-CU-LABEL: flat_system_unordered_load:
41; GFX10-CU:       ; %bb.0: ; %entry
42; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
43; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
44; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
45; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
46; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
47; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
48; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
49; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
50; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
51; GFX10-CU-NEXT:    s_endpgm
52;
53; SKIP-CACHE-INV-LABEL: flat_system_unordered_load:
54; SKIP-CACHE-INV:       ; %bb.0: ; %entry
55; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
56; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
57; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
58; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
59; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
60; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
61; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
62; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
63; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
64; SKIP-CACHE-INV-NEXT:    s_endpgm
65;
66; GFX90A-NOTTGSPLIT-LABEL: flat_system_unordered_load:
67; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
68; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
69; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
70; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
71; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
72; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
73; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
74; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
75; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
76; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
77; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
78;
79; GFX90A-TGSPLIT-LABEL: flat_system_unordered_load:
80; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
81; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
82; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
83; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
84; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
85; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
86; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
87; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
88; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
89; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
90; GFX90A-TGSPLIT-NEXT:    s_endpgm
91;
92; GFX940-NOTTGSPLIT-LABEL: flat_system_unordered_load:
93; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
94; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
95; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
96; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
97; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
98; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
99; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
100; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
101; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
102; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
103; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
104;
105; GFX940-TGSPLIT-LABEL: flat_system_unordered_load:
106; GFX940-TGSPLIT:       ; %bb.0: ; %entry
107; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
108; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
109; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
110; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
111; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
112; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
113; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
114; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
115; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
116; GFX940-TGSPLIT-NEXT:    s_endpgm
117;
118; GFX11-WGP-LABEL: flat_system_unordered_load:
119; GFX11-WGP:       ; %bb.0: ; %entry
120; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
121; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
122; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
123; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
124; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1]
125; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
126; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
127; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
128; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
129; GFX11-WGP-NEXT:    s_endpgm
130;
131; GFX11-CU-LABEL: flat_system_unordered_load:
132; GFX11-CU:       ; %bb.0: ; %entry
133; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
134; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
135; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
136; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
137; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
138; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
139; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
140; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
141; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
142; GFX11-CU-NEXT:    s_endpgm
143    i32* %in, i32* %out) {
144entry:
145  %val = load atomic i32, i32* %in unordered, align 4
146  store i32 %val, i32* %out
147  ret void
148}
149
150define amdgpu_kernel void @flat_system_monotonic_load(
151; GFX7-LABEL: flat_system_monotonic_load:
152; GFX7:       ; %bb.0: ; %entry
153; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
154; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
155; GFX7-NEXT:    v_mov_b32_e32 v0, s0
156; GFX7-NEXT:    v_mov_b32_e32 v1, s1
157; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
158; GFX7-NEXT:    v_mov_b32_e32 v0, s2
159; GFX7-NEXT:    v_mov_b32_e32 v1, s3
160; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
161; GFX7-NEXT:    flat_store_dword v[0:1], v2
162; GFX7-NEXT:    s_endpgm
163;
164; GFX10-WGP-LABEL: flat_system_monotonic_load:
165; GFX10-WGP:       ; %bb.0: ; %entry
166; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
167; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
168; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
169; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
170; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
171; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
172; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
173; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
174; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
175; GFX10-WGP-NEXT:    s_endpgm
176;
177; GFX10-CU-LABEL: flat_system_monotonic_load:
178; GFX10-CU:       ; %bb.0: ; %entry
179; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
180; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
181; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
182; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
183; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
184; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
185; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
186; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
187; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
188; GFX10-CU-NEXT:    s_endpgm
189;
190; SKIP-CACHE-INV-LABEL: flat_system_monotonic_load:
191; SKIP-CACHE-INV:       ; %bb.0: ; %entry
192; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
193; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
194; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
195; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
196; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
197; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
198; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
199; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
200; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
201; SKIP-CACHE-INV-NEXT:    s_endpgm
202;
203; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_load:
204; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
205; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
206; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
207; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
208; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
209; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
210; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
211; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
212; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
213; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
214; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
215;
216; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_load:
217; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
218; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
219; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
220; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
221; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
222; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
223; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
224; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
225; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
226; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
227; GFX90A-TGSPLIT-NEXT:    s_endpgm
228;
229; GFX940-NOTTGSPLIT-LABEL: flat_system_monotonic_load:
230; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
231; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
232; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
233; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
234; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
235; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0 sc1
236; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
237; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
238; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
239; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
240; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
241;
242; GFX940-TGSPLIT-LABEL: flat_system_monotonic_load:
243; GFX940-TGSPLIT:       ; %bb.0: ; %entry
244; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
245; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
246; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
247; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
248; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0 sc1
249; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
250; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
251; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
252; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
253; GFX940-TGSPLIT-NEXT:    s_endpgm
254;
255; GFX11-WGP-LABEL: flat_system_monotonic_load:
256; GFX11-WGP:       ; %bb.0: ; %entry
257; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
258; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
259; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
260; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
261; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc
262; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
263; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
264; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
265; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
266; GFX11-WGP-NEXT:    s_endpgm
267;
268; GFX11-CU-LABEL: flat_system_monotonic_load:
269; GFX11-CU:       ; %bb.0: ; %entry
270; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
271; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
272; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
273; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
274; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1] glc
275; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
276; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
277; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
278; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
279; GFX11-CU-NEXT:    s_endpgm
280    i32* %in, i32* %out) {
281entry:
282  %val = load atomic i32, i32* %in monotonic, align 4
283  store i32 %val, i32* %out
284  ret void
285}
286
287define amdgpu_kernel void @flat_system_acquire_load(
288; GFX7-LABEL: flat_system_acquire_load:
289; GFX7:       ; %bb.0: ; %entry
290; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
291; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
292; GFX7-NEXT:    v_mov_b32_e32 v0, s0
293; GFX7-NEXT:    v_mov_b32_e32 v1, s1
294; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
295; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
296; GFX7-NEXT:    buffer_wbinvl1_vol
297; GFX7-NEXT:    v_mov_b32_e32 v0, s2
298; GFX7-NEXT:    v_mov_b32_e32 v1, s3
299; GFX7-NEXT:    flat_store_dword v[0:1], v2
300; GFX7-NEXT:    s_endpgm
301;
302; GFX10-WGP-LABEL: flat_system_acquire_load:
303; GFX10-WGP:       ; %bb.0: ; %entry
304; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
305; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
306; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
307; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
308; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
309; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
310; GFX10-WGP-NEXT:    buffer_gl0_inv
311; GFX10-WGP-NEXT:    buffer_gl1_inv
312; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
313; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
314; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
315; GFX10-WGP-NEXT:    s_endpgm
316;
317; GFX10-CU-LABEL: flat_system_acquire_load:
318; GFX10-CU:       ; %bb.0: ; %entry
319; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
320; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
321; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
322; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
323; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
324; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
325; GFX10-CU-NEXT:    buffer_gl0_inv
326; GFX10-CU-NEXT:    buffer_gl1_inv
327; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
328; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
329; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
330; GFX10-CU-NEXT:    s_endpgm
331;
332; SKIP-CACHE-INV-LABEL: flat_system_acquire_load:
333; SKIP-CACHE-INV:       ; %bb.0: ; %entry
334; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
335; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
336; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
337; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
338; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
339; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
340; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
341; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
342; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
343; SKIP-CACHE-INV-NEXT:    s_endpgm
344;
345; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_load:
346; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
347; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
348; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
349; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
350; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
351; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
352; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
353; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
354; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
355; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
356; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
357; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
358; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
359;
360; GFX90A-TGSPLIT-LABEL: flat_system_acquire_load:
361; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
362; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
363; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
364; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
365; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
366; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
367; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
368; GFX90A-TGSPLIT-NEXT:    buffer_invl2
369; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
370; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
371; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
372; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
373; GFX90A-TGSPLIT-NEXT:    s_endpgm
374;
375; GFX940-NOTTGSPLIT-LABEL: flat_system_acquire_load:
376; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
377; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
378; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
379; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
380; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
381; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0 sc1
382; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
383; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
384; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
385; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
386; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
387; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
388;
389; GFX940-TGSPLIT-LABEL: flat_system_acquire_load:
390; GFX940-TGSPLIT:       ; %bb.0: ; %entry
391; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
392; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
393; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
394; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
395; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0 sc1
396; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
397; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
398; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
399; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
400; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
401; GFX940-TGSPLIT-NEXT:    s_endpgm
402;
403; GFX11-WGP-LABEL: flat_system_acquire_load:
404; GFX11-WGP:       ; %bb.0: ; %entry
405; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
406; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
407; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
408; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
409; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc
410; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
411; GFX11-WGP-NEXT:    buffer_gl0_inv
412; GFX11-WGP-NEXT:    buffer_gl1_inv
413; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
414; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
415; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
416; GFX11-WGP-NEXT:    s_endpgm
417;
418; GFX11-CU-LABEL: flat_system_acquire_load:
419; GFX11-CU:       ; %bb.0: ; %entry
420; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
421; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
422; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
423; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
424; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1] glc
425; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
426; GFX11-CU-NEXT:    buffer_gl0_inv
427; GFX11-CU-NEXT:    buffer_gl1_inv
428; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
429; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
430; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
431; GFX11-CU-NEXT:    s_endpgm
432    i32* %in, i32* %out) {
433entry:
434  %val = load atomic i32, i32* %in acquire, align 4
435  store i32 %val, i32* %out
436  ret void
437}
438
439define amdgpu_kernel void @flat_system_seq_cst_load(
440; GFX7-LABEL: flat_system_seq_cst_load:
441; GFX7:       ; %bb.0: ; %entry
442; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
443; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
444; GFX7-NEXT:    v_mov_b32_e32 v0, s0
445; GFX7-NEXT:    v_mov_b32_e32 v1, s1
446; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
447; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
448; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
449; GFX7-NEXT:    buffer_wbinvl1_vol
450; GFX7-NEXT:    v_mov_b32_e32 v0, s2
451; GFX7-NEXT:    v_mov_b32_e32 v1, s3
452; GFX7-NEXT:    flat_store_dword v[0:1], v2
453; GFX7-NEXT:    s_endpgm
454;
455; GFX10-WGP-LABEL: flat_system_seq_cst_load:
456; GFX10-WGP:       ; %bb.0: ; %entry
457; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
458; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
459; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
460; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
461; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
462; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
463; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
464; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
465; GFX10-WGP-NEXT:    buffer_gl0_inv
466; GFX10-WGP-NEXT:    buffer_gl1_inv
467; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
468; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
469; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
470; GFX10-WGP-NEXT:    s_endpgm
471;
472; GFX10-CU-LABEL: flat_system_seq_cst_load:
473; GFX10-CU:       ; %bb.0: ; %entry
474; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
475; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
476; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
477; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
478; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
479; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
480; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
481; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
482; GFX10-CU-NEXT:    buffer_gl0_inv
483; GFX10-CU-NEXT:    buffer_gl1_inv
484; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
485; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
486; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
487; GFX10-CU-NEXT:    s_endpgm
488;
489; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_load:
490; SKIP-CACHE-INV:       ; %bb.0: ; %entry
491; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
492; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
493; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
494; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
495; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
496; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
497; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
498; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
499; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
500; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
501; SKIP-CACHE-INV-NEXT:    s_endpgm
502;
503; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_load:
504; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
505; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
506; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
507; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
508; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
509; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
510; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
511; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
512; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
513; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
514; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
515; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
516; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
517; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
518;
519; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_load:
520; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
521; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
522; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
523; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
524; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
525; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
526; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
527; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
528; GFX90A-TGSPLIT-NEXT:    buffer_invl2
529; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
530; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
531; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
532; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
533; GFX90A-TGSPLIT-NEXT:    s_endpgm
534;
535; GFX940-NOTTGSPLIT-LABEL: flat_system_seq_cst_load:
536; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
537; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
538; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
539; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
540; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
541; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
542; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0 sc1
543; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
544; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
545; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
546; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
547; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
548; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
549;
550; GFX940-TGSPLIT-LABEL: flat_system_seq_cst_load:
551; GFX940-TGSPLIT:       ; %bb.0: ; %entry
552; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
553; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
554; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
555; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
556; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
557; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0 sc1
558; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
559; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
560; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
561; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
562; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
563; GFX940-TGSPLIT-NEXT:    s_endpgm
564;
565; GFX11-WGP-LABEL: flat_system_seq_cst_load:
566; GFX11-WGP:       ; %bb.0: ; %entry
567; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
568; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
569; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
570; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
571; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
572; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
573; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc
574; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
575; GFX11-WGP-NEXT:    buffer_gl0_inv
576; GFX11-WGP-NEXT:    buffer_gl1_inv
577; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
578; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
579; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
580; GFX11-WGP-NEXT:    s_endpgm
581;
582; GFX11-CU-LABEL: flat_system_seq_cst_load:
583; GFX11-CU:       ; %bb.0: ; %entry
584; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
585; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
586; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
587; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
588; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
589; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
590; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1] glc
591; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
592; GFX11-CU-NEXT:    buffer_gl0_inv
593; GFX11-CU-NEXT:    buffer_gl1_inv
594; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
595; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
596; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
597; GFX11-CU-NEXT:    s_endpgm
598    i32* %in, i32* %out) {
599entry:
600  %val = load atomic i32, i32* %in seq_cst, align 4
601  store i32 %val, i32* %out
602  ret void
603}
604
605define amdgpu_kernel void @flat_system_unordered_store(
606; GFX7-LABEL: flat_system_unordered_store:
607; GFX7:       ; %bb.0: ; %entry
608; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
609; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
610; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
611; GFX7-NEXT:    v_mov_b32_e32 v0, s0
612; GFX7-NEXT:    v_mov_b32_e32 v1, s1
613; GFX7-NEXT:    v_mov_b32_e32 v2, s2
614; GFX7-NEXT:    flat_store_dword v[0:1], v2
615; GFX7-NEXT:    s_endpgm
616;
617; GFX10-WGP-LABEL: flat_system_unordered_store:
618; GFX10-WGP:       ; %bb.0: ; %entry
619; GFX10-WGP-NEXT:    s_clause 0x1
620; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
621; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
622; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
623; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
624; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
625; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
626; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
627; GFX10-WGP-NEXT:    s_endpgm
628;
629; GFX10-CU-LABEL: flat_system_unordered_store:
630; GFX10-CU:       ; %bb.0: ; %entry
631; GFX10-CU-NEXT:    s_clause 0x1
632; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
633; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
634; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
635; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
636; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
637; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
638; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
639; GFX10-CU-NEXT:    s_endpgm
640;
641; SKIP-CACHE-INV-LABEL: flat_system_unordered_store:
642; SKIP-CACHE-INV:       ; %bb.0: ; %entry
643; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
644; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
645; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
646; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
647; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
648; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
649; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
650; SKIP-CACHE-INV-NEXT:    s_endpgm
651;
652; GFX90A-NOTTGSPLIT-LABEL: flat_system_unordered_store:
653; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
654; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
655; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
656; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
657; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
658; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
659; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
660; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
661;
662; GFX90A-TGSPLIT-LABEL: flat_system_unordered_store:
663; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
664; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
665; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
666; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
667; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
668; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
669; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
670; GFX90A-TGSPLIT-NEXT:    s_endpgm
671;
672; GFX940-NOTTGSPLIT-LABEL: flat_system_unordered_store:
673; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
674; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
675; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
676; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
677; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
678; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
679; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
680; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
681;
682; GFX940-TGSPLIT-LABEL: flat_system_unordered_store:
683; GFX940-TGSPLIT:       ; %bb.0: ; %entry
684; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
685; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
686; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
687; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
688; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
689; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
690; GFX940-TGSPLIT-NEXT:    s_endpgm
691;
692; GFX11-WGP-LABEL: flat_system_unordered_store:
693; GFX11-WGP:       ; %bb.0: ; %entry
694; GFX11-WGP-NEXT:    s_clause 0x1
695; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
696; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
697; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
698; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
699; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
700; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
701; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
702; GFX11-WGP-NEXT:    s_endpgm
703;
704; GFX11-CU-LABEL: flat_system_unordered_store:
705; GFX11-CU:       ; %bb.0: ; %entry
706; GFX11-CU-NEXT:    s_clause 0x1
707; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
708; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
709; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
710; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
711; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
712; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
713; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
714; GFX11-CU-NEXT:    s_endpgm
715    i32 %in, i32* %out) {
716entry:
717  store atomic i32 %in, i32* %out unordered, align 4
718  ret void
719}
720
721define amdgpu_kernel void @flat_system_monotonic_store(
722; GFX7-LABEL: flat_system_monotonic_store:
723; GFX7:       ; %bb.0: ; %entry
724; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
725; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
726; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
727; GFX7-NEXT:    v_mov_b32_e32 v0, s0
728; GFX7-NEXT:    v_mov_b32_e32 v1, s1
729; GFX7-NEXT:    v_mov_b32_e32 v2, s2
730; GFX7-NEXT:    flat_store_dword v[0:1], v2
731; GFX7-NEXT:    s_endpgm
732;
733; GFX10-WGP-LABEL: flat_system_monotonic_store:
734; GFX10-WGP:       ; %bb.0: ; %entry
735; GFX10-WGP-NEXT:    s_clause 0x1
736; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
737; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
738; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
739; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
740; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
741; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
742; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
743; GFX10-WGP-NEXT:    s_endpgm
744;
745; GFX10-CU-LABEL: flat_system_monotonic_store:
746; GFX10-CU:       ; %bb.0: ; %entry
747; GFX10-CU-NEXT:    s_clause 0x1
748; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
749; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
750; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
751; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
752; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
753; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
754; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
755; GFX10-CU-NEXT:    s_endpgm
756;
757; SKIP-CACHE-INV-LABEL: flat_system_monotonic_store:
758; SKIP-CACHE-INV:       ; %bb.0: ; %entry
759; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
760; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
761; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
762; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
763; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
764; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
765; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
766; SKIP-CACHE-INV-NEXT:    s_endpgm
767;
768; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_store:
769; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
770; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
771; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
772; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
773; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
774; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
775; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
776; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
777;
778; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_store:
779; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
780; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
781; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
782; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
783; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
784; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
785; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
786; GFX90A-TGSPLIT-NEXT:    s_endpgm
787;
788; GFX940-NOTTGSPLIT-LABEL: flat_system_monotonic_store:
789; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
790; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
791; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
792; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
793; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
794; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
795; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
796; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
797;
798; GFX940-TGSPLIT-LABEL: flat_system_monotonic_store:
799; GFX940-TGSPLIT:       ; %bb.0: ; %entry
800; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
801; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
802; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
803; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
804; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
805; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
806; GFX940-TGSPLIT-NEXT:    s_endpgm
807;
808; GFX11-WGP-LABEL: flat_system_monotonic_store:
809; GFX11-WGP:       ; %bb.0: ; %entry
810; GFX11-WGP-NEXT:    s_clause 0x1
811; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
812; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
813; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
814; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
815; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
816; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
817; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
818; GFX11-WGP-NEXT:    s_endpgm
819;
820; GFX11-CU-LABEL: flat_system_monotonic_store:
821; GFX11-CU:       ; %bb.0: ; %entry
822; GFX11-CU-NEXT:    s_clause 0x1
823; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
824; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
825; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
826; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
827; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
828; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
829; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
830; GFX11-CU-NEXT:    s_endpgm
831    i32 %in, i32* %out) {
832entry:
833  store atomic i32 %in, i32* %out monotonic, align 4
834  ret void
835}
836
837define amdgpu_kernel void @flat_system_release_store(
838; GFX7-LABEL: flat_system_release_store:
839; GFX7:       ; %bb.0: ; %entry
840; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
841; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
842; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
843; GFX7-NEXT:    v_mov_b32_e32 v0, s0
844; GFX7-NEXT:    v_mov_b32_e32 v1, s1
845; GFX7-NEXT:    v_mov_b32_e32 v2, s2
846; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
847; GFX7-NEXT:    flat_store_dword v[0:1], v2
848; GFX7-NEXT:    s_endpgm
849;
850; GFX10-WGP-LABEL: flat_system_release_store:
851; GFX10-WGP:       ; %bb.0: ; %entry
852; GFX10-WGP-NEXT:    s_clause 0x1
853; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
854; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
855; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
856; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
857; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
858; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
859; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
860; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
861; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
862; GFX10-WGP-NEXT:    s_endpgm
863;
864; GFX10-CU-LABEL: flat_system_release_store:
865; GFX10-CU:       ; %bb.0: ; %entry
866; GFX10-CU-NEXT:    s_clause 0x1
867; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
868; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
869; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
870; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
871; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
872; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
873; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
874; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
875; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
876; GFX10-CU-NEXT:    s_endpgm
877;
878; SKIP-CACHE-INV-LABEL: flat_system_release_store:
879; SKIP-CACHE-INV:       ; %bb.0: ; %entry
880; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
881; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
882; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
883; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
884; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
885; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
886; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
887; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
888; SKIP-CACHE-INV-NEXT:    s_endpgm
889;
890; GFX90A-NOTTGSPLIT-LABEL: flat_system_release_store:
891; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
892; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
893; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
894; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
895; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
896; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
897; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
898; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
899; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
900; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
901;
902; GFX90A-TGSPLIT-LABEL: flat_system_release_store:
903; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
904; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
905; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
906; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
907; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
908; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
909; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
910; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
911; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
912; GFX90A-TGSPLIT-NEXT:    s_endpgm
913;
914; GFX940-NOTTGSPLIT-LABEL: flat_system_release_store:
915; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
916; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
917; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
918; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
919; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
920; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
921; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
922; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
923; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
924; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
925;
926; GFX940-TGSPLIT-LABEL: flat_system_release_store:
927; GFX940-TGSPLIT:       ; %bb.0: ; %entry
928; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
929; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
930; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
931; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
932; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
933; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
934; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
935; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
936; GFX940-TGSPLIT-NEXT:    s_endpgm
937;
938; GFX11-WGP-LABEL: flat_system_release_store:
939; GFX11-WGP:       ; %bb.0: ; %entry
940; GFX11-WGP-NEXT:    s_clause 0x1
941; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
942; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
943; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
944; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
945; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
946; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
947; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
948; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
949; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
950; GFX11-WGP-NEXT:    s_endpgm
951;
952; GFX11-CU-LABEL: flat_system_release_store:
953; GFX11-CU:       ; %bb.0: ; %entry
954; GFX11-CU-NEXT:    s_clause 0x1
955; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
956; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
957; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
958; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
959; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
960; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
961; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
962; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
963; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
964; GFX11-CU-NEXT:    s_endpgm
965    i32 %in, i32* %out) {
966entry:
967  store atomic i32 %in, i32* %out release, align 4
968  ret void
969}
970
971define amdgpu_kernel void @flat_system_seq_cst_store(
972; GFX7-LABEL: flat_system_seq_cst_store:
973; GFX7:       ; %bb.0: ; %entry
974; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
975; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
976; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
977; GFX7-NEXT:    v_mov_b32_e32 v0, s0
978; GFX7-NEXT:    v_mov_b32_e32 v1, s1
979; GFX7-NEXT:    v_mov_b32_e32 v2, s2
980; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
981; GFX7-NEXT:    flat_store_dword v[0:1], v2
982; GFX7-NEXT:    s_endpgm
983;
984; GFX10-WGP-LABEL: flat_system_seq_cst_store:
985; GFX10-WGP:       ; %bb.0: ; %entry
986; GFX10-WGP-NEXT:    s_clause 0x1
987; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
988; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
989; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
990; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
991; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
992; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
993; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
994; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
995; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
996; GFX10-WGP-NEXT:    s_endpgm
997;
998; GFX10-CU-LABEL: flat_system_seq_cst_store:
999; GFX10-CU:       ; %bb.0: ; %entry
1000; GFX10-CU-NEXT:    s_clause 0x1
1001; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1002; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
1003; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1004; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1005; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1006; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1007; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1008; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1009; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1010; GFX10-CU-NEXT:    s_endpgm
1011;
1012; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_store:
1013; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1014; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
1015; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
1016; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1017; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1018; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1019; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1020; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1021; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1022; SKIP-CACHE-INV-NEXT:    s_endpgm
1023;
1024; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_store:
1025; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1026; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1027; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
1028; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1029; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1030; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1031; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
1032; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1033; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1034; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1035;
1036; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_store:
1037; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1038; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1039; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
1040; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1041; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1042; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1043; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
1044; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1045; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1046; GFX90A-TGSPLIT-NEXT:    s_endpgm
1047;
1048; GFX940-NOTTGSPLIT-LABEL: flat_system_seq_cst_store:
1049; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1050; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
1051; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
1052; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1053; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1054; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1055; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
1056; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1057; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
1058; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1059;
1060; GFX940-TGSPLIT-LABEL: flat_system_seq_cst_store:
1061; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1062; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
1063; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
1064; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1065; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1066; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1067; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
1068; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1069; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
1070; GFX940-TGSPLIT-NEXT:    s_endpgm
1071;
1072; GFX11-WGP-LABEL: flat_system_seq_cst_store:
1073; GFX11-WGP:       ; %bb.0: ; %entry
1074; GFX11-WGP-NEXT:    s_clause 0x1
1075; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
1076; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
1077; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1078; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1079; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1080; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1081; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1082; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1083; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
1084; GFX11-WGP-NEXT:    s_endpgm
1085;
1086; GFX11-CU-LABEL: flat_system_seq_cst_store:
1087; GFX11-CU:       ; %bb.0: ; %entry
1088; GFX11-CU-NEXT:    s_clause 0x1
1089; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
1090; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
1091; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1092; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
1093; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
1094; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1095; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1096; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1097; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
1098; GFX11-CU-NEXT:    s_endpgm
1099    i32 %in, i32* %out) {
1100entry:
1101  store atomic i32 %in, i32* %out seq_cst, align 4
1102  ret void
1103}
1104
1105define amdgpu_kernel void @flat_system_monotonic_atomicrmw(
1106; GFX7-LABEL: flat_system_monotonic_atomicrmw:
1107; GFX7:       ; %bb.0: ; %entry
1108; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1109; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1110; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1111; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1112; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1113; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1114; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1115; GFX7-NEXT:    s_endpgm
1116;
1117; GFX10-WGP-LABEL: flat_system_monotonic_atomicrmw:
1118; GFX10-WGP:       ; %bb.0: ; %entry
1119; GFX10-WGP-NEXT:    s_clause 0x1
1120; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1121; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1122; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1123; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1124; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1125; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1126; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1127; GFX10-WGP-NEXT:    s_endpgm
1128;
1129; GFX10-CU-LABEL: flat_system_monotonic_atomicrmw:
1130; GFX10-CU:       ; %bb.0: ; %entry
1131; GFX10-CU-NEXT:    s_clause 0x1
1132; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1133; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1134; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1135; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1136; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1137; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1138; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1139; GFX10-CU-NEXT:    s_endpgm
1140;
1141; SKIP-CACHE-INV-LABEL: flat_system_monotonic_atomicrmw:
1142; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1143; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1144; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1145; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1146; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1147; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1148; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1149; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1150; SKIP-CACHE-INV-NEXT:    s_endpgm
1151;
1152; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_atomicrmw:
1153; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1154; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1155; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1156; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1157; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1158; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1159; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1160; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1161;
1162; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_atomicrmw:
1163; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1164; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1165; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1166; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1167; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1168; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1169; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1170; GFX90A-TGSPLIT-NEXT:    s_endpgm
1171;
1172; GFX940-NOTTGSPLIT-LABEL: flat_system_monotonic_atomicrmw:
1173; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1174; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1175; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1176; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1177; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1178; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1179; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
1180; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1181;
1182; GFX940-TGSPLIT-LABEL: flat_system_monotonic_atomicrmw:
1183; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1184; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1185; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1186; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1187; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1188; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1189; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
1190; GFX940-TGSPLIT-NEXT:    s_endpgm
1191;
1192; GFX11-WGP-LABEL: flat_system_monotonic_atomicrmw:
1193; GFX11-WGP:       ; %bb.0: ; %entry
1194; GFX11-WGP-NEXT:    s_clause 0x1
1195; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1196; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1197; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1198; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1199; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1200; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1201; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1202; GFX11-WGP-NEXT:    s_endpgm
1203;
1204; GFX11-CU-LABEL: flat_system_monotonic_atomicrmw:
1205; GFX11-CU:       ; %bb.0: ; %entry
1206; GFX11-CU-NEXT:    s_clause 0x1
1207; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1208; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1209; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1210; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
1211; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
1212; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1213; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1214; GFX11-CU-NEXT:    s_endpgm
1215    i32* %out, i32 %in) {
1216entry:
1217  %val = atomicrmw volatile xchg i32* %out, i32 %in monotonic
1218  ret void
1219}
1220
1221define amdgpu_kernel void @flat_system_acquire_atomicrmw(
1222; GFX7-LABEL: flat_system_acquire_atomicrmw:
1223; GFX7:       ; %bb.0: ; %entry
1224; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1225; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1226; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1227; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1228; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1229; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1230; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1231; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1232; GFX7-NEXT:    buffer_wbinvl1_vol
1233; GFX7-NEXT:    s_endpgm
1234;
1235; GFX10-WGP-LABEL: flat_system_acquire_atomicrmw:
1236; GFX10-WGP:       ; %bb.0: ; %entry
1237; GFX10-WGP-NEXT:    s_clause 0x1
1238; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1239; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1240; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1241; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1242; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1243; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1244; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1245; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1246; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1247; GFX10-WGP-NEXT:    buffer_gl0_inv
1248; GFX10-WGP-NEXT:    buffer_gl1_inv
1249; GFX10-WGP-NEXT:    s_endpgm
1250;
1251; GFX10-CU-LABEL: flat_system_acquire_atomicrmw:
1252; GFX10-CU:       ; %bb.0: ; %entry
1253; GFX10-CU-NEXT:    s_clause 0x1
1254; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1255; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1256; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1257; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1258; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1259; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1260; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1261; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1262; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1263; GFX10-CU-NEXT:    buffer_gl0_inv
1264; GFX10-CU-NEXT:    buffer_gl1_inv
1265; GFX10-CU-NEXT:    s_endpgm
1266;
1267; SKIP-CACHE-INV-LABEL: flat_system_acquire_atomicrmw:
1268; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1269; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1270; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1271; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1272; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1273; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1274; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1275; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1276; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1277; SKIP-CACHE-INV-NEXT:    s_endpgm
1278;
1279; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_atomicrmw:
1280; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1281; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1282; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1283; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1284; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1285; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1286; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1287; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1288; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
1289; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1290; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1291;
1292; GFX90A-TGSPLIT-LABEL: flat_system_acquire_atomicrmw:
1293; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1294; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1295; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1296; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1297; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1298; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1299; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1300; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1301; GFX90A-TGSPLIT-NEXT:    buffer_invl2
1302; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1303; GFX90A-TGSPLIT-NEXT:    s_endpgm
1304;
1305; GFX940-NOTTGSPLIT-LABEL: flat_system_acquire_atomicrmw:
1306; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1307; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1308; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1309; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1310; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1311; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1312; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
1313; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1314; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
1315; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1316;
1317; GFX940-TGSPLIT-LABEL: flat_system_acquire_atomicrmw:
1318; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1319; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1320; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1321; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1322; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1323; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1324; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
1325; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1326; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
1327; GFX940-TGSPLIT-NEXT:    s_endpgm
1328;
1329; GFX11-WGP-LABEL: flat_system_acquire_atomicrmw:
1330; GFX11-WGP:       ; %bb.0: ; %entry
1331; GFX11-WGP-NEXT:    s_clause 0x1
1332; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1333; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1334; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1335; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1336; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1337; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1338; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1339; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1340; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1341; GFX11-WGP-NEXT:    buffer_gl0_inv
1342; GFX11-WGP-NEXT:    buffer_gl1_inv
1343; GFX11-WGP-NEXT:    s_endpgm
1344;
1345; GFX11-CU-LABEL: flat_system_acquire_atomicrmw:
1346; GFX11-CU:       ; %bb.0: ; %entry
1347; GFX11-CU-NEXT:    s_clause 0x1
1348; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1349; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1350; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1351; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
1352; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
1353; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1354; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1355; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1356; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1357; GFX11-CU-NEXT:    buffer_gl0_inv
1358; GFX11-CU-NEXT:    buffer_gl1_inv
1359; GFX11-CU-NEXT:    s_endpgm
1360    i32* %out, i32 %in) {
1361entry:
1362  %val = atomicrmw volatile xchg i32* %out, i32 %in acquire
1363  ret void
1364}
1365
1366define amdgpu_kernel void @flat_system_release_atomicrmw(
1367; GFX7-LABEL: flat_system_release_atomicrmw:
1368; GFX7:       ; %bb.0: ; %entry
1369; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1370; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1371; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1372; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1373; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1374; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1375; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1376; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1377; GFX7-NEXT:    s_endpgm
1378;
1379; GFX10-WGP-LABEL: flat_system_release_atomicrmw:
1380; GFX10-WGP:       ; %bb.0: ; %entry
1381; GFX10-WGP-NEXT:    s_clause 0x1
1382; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1383; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1384; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1385; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1386; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1387; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1388; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1389; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1390; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1391; GFX10-WGP-NEXT:    s_endpgm
1392;
1393; GFX10-CU-LABEL: flat_system_release_atomicrmw:
1394; GFX10-CU:       ; %bb.0: ; %entry
1395; GFX10-CU-NEXT:    s_clause 0x1
1396; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1397; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1398; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1399; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1400; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1401; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1402; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1403; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1404; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1405; GFX10-CU-NEXT:    s_endpgm
1406;
1407; SKIP-CACHE-INV-LABEL: flat_system_release_atomicrmw:
1408; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1409; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1410; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1411; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1412; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1413; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1414; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1415; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1416; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1417; SKIP-CACHE-INV-NEXT:    s_endpgm
1418;
1419; GFX90A-NOTTGSPLIT-LABEL: flat_system_release_atomicrmw:
1420; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1421; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1422; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1423; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1424; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1425; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1426; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
1427; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1428; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1429; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1430;
1431; GFX90A-TGSPLIT-LABEL: flat_system_release_atomicrmw:
1432; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1433; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1434; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1435; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1436; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1437; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1438; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
1439; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1440; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1441; GFX90A-TGSPLIT-NEXT:    s_endpgm
1442;
1443; GFX940-NOTTGSPLIT-LABEL: flat_system_release_atomicrmw:
1444; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1445; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1446; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1447; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1448; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1449; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1450; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
1451; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1452; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
1453; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1454;
1455; GFX940-TGSPLIT-LABEL: flat_system_release_atomicrmw:
1456; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1457; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1458; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1459; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1460; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1461; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1462; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
1463; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1464; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
1465; GFX940-TGSPLIT-NEXT:    s_endpgm
1466;
1467; GFX11-WGP-LABEL: flat_system_release_atomicrmw:
1468; GFX11-WGP:       ; %bb.0: ; %entry
1469; GFX11-WGP-NEXT:    s_clause 0x1
1470; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1471; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1472; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1473; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1474; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1475; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1476; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1477; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1478; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1479; GFX11-WGP-NEXT:    s_endpgm
1480;
1481; GFX11-CU-LABEL: flat_system_release_atomicrmw:
1482; GFX11-CU:       ; %bb.0: ; %entry
1483; GFX11-CU-NEXT:    s_clause 0x1
1484; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1485; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1486; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1487; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
1488; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
1489; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1490; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1491; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1492; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1493; GFX11-CU-NEXT:    s_endpgm
1494    i32* %out, i32 %in) {
1495entry:
1496  %val = atomicrmw volatile xchg i32* %out, i32 %in release
1497  ret void
1498}
1499
1500define amdgpu_kernel void @flat_system_acq_rel_atomicrmw(
1501; GFX7-LABEL: flat_system_acq_rel_atomicrmw:
1502; GFX7:       ; %bb.0: ; %entry
1503; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1504; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1505; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1506; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1507; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1508; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1509; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1510; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1511; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1512; GFX7-NEXT:    buffer_wbinvl1_vol
1513; GFX7-NEXT:    s_endpgm
1514;
1515; GFX10-WGP-LABEL: flat_system_acq_rel_atomicrmw:
1516; GFX10-WGP:       ; %bb.0: ; %entry
1517; GFX10-WGP-NEXT:    s_clause 0x1
1518; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1519; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1520; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1521; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1522; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1523; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1524; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1525; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1526; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1527; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1528; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1529; GFX10-WGP-NEXT:    buffer_gl0_inv
1530; GFX10-WGP-NEXT:    buffer_gl1_inv
1531; GFX10-WGP-NEXT:    s_endpgm
1532;
1533; GFX10-CU-LABEL: flat_system_acq_rel_atomicrmw:
1534; GFX10-CU:       ; %bb.0: ; %entry
1535; GFX10-CU-NEXT:    s_clause 0x1
1536; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1537; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1538; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1539; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1540; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1541; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1542; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1543; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1544; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1545; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1546; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1547; GFX10-CU-NEXT:    buffer_gl0_inv
1548; GFX10-CU-NEXT:    buffer_gl1_inv
1549; GFX10-CU-NEXT:    s_endpgm
1550;
1551; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_atomicrmw:
1552; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1553; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1554; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1555; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1556; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1557; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1558; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1559; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1560; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1561; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1562; SKIP-CACHE-INV-NEXT:    s_endpgm
1563;
1564; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_atomicrmw:
1565; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1566; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1567; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1568; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1569; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1570; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1571; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
1572; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1573; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1574; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1575; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
1576; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1577; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1578;
1579; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_atomicrmw:
1580; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1581; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1582; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1583; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1584; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1585; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1586; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
1587; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1588; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1589; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1590; GFX90A-TGSPLIT-NEXT:    buffer_invl2
1591; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1592; GFX90A-TGSPLIT-NEXT:    s_endpgm
1593;
1594; GFX940-NOTTGSPLIT-LABEL: flat_system_acq_rel_atomicrmw:
1595; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1596; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1597; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1598; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1599; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1600; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1601; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
1602; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1603; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
1604; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1605; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
1606; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1607;
1608; GFX940-TGSPLIT-LABEL: flat_system_acq_rel_atomicrmw:
1609; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1610; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1611; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1612; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1613; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1614; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1615; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
1616; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1617; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
1618; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1619; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
1620; GFX940-TGSPLIT-NEXT:    s_endpgm
1621;
1622; GFX11-WGP-LABEL: flat_system_acq_rel_atomicrmw:
1623; GFX11-WGP:       ; %bb.0: ; %entry
1624; GFX11-WGP-NEXT:    s_clause 0x1
1625; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1626; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1627; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1628; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1629; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1630; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1631; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1632; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1633; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1634; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1635; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1636; GFX11-WGP-NEXT:    buffer_gl0_inv
1637; GFX11-WGP-NEXT:    buffer_gl1_inv
1638; GFX11-WGP-NEXT:    s_endpgm
1639;
1640; GFX11-CU-LABEL: flat_system_acq_rel_atomicrmw:
1641; GFX11-CU:       ; %bb.0: ; %entry
1642; GFX11-CU-NEXT:    s_clause 0x1
1643; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1644; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1645; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1646; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
1647; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
1648; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1649; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1650; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1651; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1652; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1653; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1654; GFX11-CU-NEXT:    buffer_gl0_inv
1655; GFX11-CU-NEXT:    buffer_gl1_inv
1656; GFX11-CU-NEXT:    s_endpgm
1657    i32* %out, i32 %in) {
1658entry:
1659  %val = atomicrmw volatile xchg i32* %out, i32 %in acq_rel
1660  ret void
1661}
1662
1663define amdgpu_kernel void @flat_system_seq_cst_atomicrmw(
1664; GFX7-LABEL: flat_system_seq_cst_atomicrmw:
1665; GFX7:       ; %bb.0: ; %entry
1666; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1667; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1668; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1669; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1670; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1671; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1672; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1673; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1674; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1675; GFX7-NEXT:    buffer_wbinvl1_vol
1676; GFX7-NEXT:    s_endpgm
1677;
1678; GFX10-WGP-LABEL: flat_system_seq_cst_atomicrmw:
1679; GFX10-WGP:       ; %bb.0: ; %entry
1680; GFX10-WGP-NEXT:    s_clause 0x1
1681; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1682; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1683; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1684; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1685; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1686; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1687; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1688; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1689; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1690; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1691; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1692; GFX10-WGP-NEXT:    buffer_gl0_inv
1693; GFX10-WGP-NEXT:    buffer_gl1_inv
1694; GFX10-WGP-NEXT:    s_endpgm
1695;
1696; GFX10-CU-LABEL: flat_system_seq_cst_atomicrmw:
1697; GFX10-CU:       ; %bb.0: ; %entry
1698; GFX10-CU-NEXT:    s_clause 0x1
1699; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1700; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1701; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1702; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1703; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1704; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1705; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1706; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1707; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1708; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1709; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1710; GFX10-CU-NEXT:    buffer_gl0_inv
1711; GFX10-CU-NEXT:    buffer_gl1_inv
1712; GFX10-CU-NEXT:    s_endpgm
1713;
1714; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_atomicrmw:
1715; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1716; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1717; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1718; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1719; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1720; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1721; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1722; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1723; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1724; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1725; SKIP-CACHE-INV-NEXT:    s_endpgm
1726;
1727; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_atomicrmw:
1728; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1729; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1730; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1731; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1732; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1733; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1734; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
1735; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1736; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1737; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1738; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
1739; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1740; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1741;
1742; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_atomicrmw:
1743; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1744; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1745; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1746; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1747; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1748; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1749; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
1750; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1751; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1752; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1753; GFX90A-TGSPLIT-NEXT:    buffer_invl2
1754; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1755; GFX90A-TGSPLIT-NEXT:    s_endpgm
1756;
1757; GFX940-NOTTGSPLIT-LABEL: flat_system_seq_cst_atomicrmw:
1758; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1759; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1760; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1761; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1762; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1763; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1764; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
1765; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1766; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
1767; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1768; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
1769; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1770;
1771; GFX940-TGSPLIT-LABEL: flat_system_seq_cst_atomicrmw:
1772; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1773; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1774; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1775; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1776; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1777; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1778; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
1779; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1780; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
1781; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1782; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
1783; GFX940-TGSPLIT-NEXT:    s_endpgm
1784;
1785; GFX11-WGP-LABEL: flat_system_seq_cst_atomicrmw:
1786; GFX11-WGP:       ; %bb.0: ; %entry
1787; GFX11-WGP-NEXT:    s_clause 0x1
1788; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1789; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1790; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1791; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1792; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1793; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1794; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1795; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1796; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1797; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1798; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1799; GFX11-WGP-NEXT:    buffer_gl0_inv
1800; GFX11-WGP-NEXT:    buffer_gl1_inv
1801; GFX11-WGP-NEXT:    s_endpgm
1802;
1803; GFX11-CU-LABEL: flat_system_seq_cst_atomicrmw:
1804; GFX11-CU:       ; %bb.0: ; %entry
1805; GFX11-CU-NEXT:    s_clause 0x1
1806; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1807; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1808; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1809; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
1810; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
1811; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1812; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1813; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1814; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1815; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1816; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1817; GFX11-CU-NEXT:    buffer_gl0_inv
1818; GFX11-CU-NEXT:    buffer_gl1_inv
1819; GFX11-CU-NEXT:    s_endpgm
1820    i32* %out, i32 %in) {
1821entry:
1822  %val = atomicrmw volatile xchg i32* %out, i32 %in seq_cst
1823  ret void
1824}
1825
1826define amdgpu_kernel void @flat_system_acquire_ret_atomicrmw(
1827; GFX7-LABEL: flat_system_acquire_ret_atomicrmw:
1828; GFX7:       ; %bb.0: ; %entry
1829; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1830; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1831; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1832; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1833; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1834; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1835; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1836; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1837; GFX7-NEXT:    buffer_wbinvl1_vol
1838; GFX7-NEXT:    flat_store_dword v[0:1], v2
1839; GFX7-NEXT:    s_endpgm
1840;
1841; GFX10-WGP-LABEL: flat_system_acquire_ret_atomicrmw:
1842; GFX10-WGP:       ; %bb.0: ; %entry
1843; GFX10-WGP-NEXT:    s_clause 0x1
1844; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1845; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1846; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1847; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1848; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1849; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1850; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1851; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1852; GFX10-WGP-NEXT:    buffer_gl0_inv
1853; GFX10-WGP-NEXT:    buffer_gl1_inv
1854; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1855; GFX10-WGP-NEXT:    s_endpgm
1856;
1857; GFX10-CU-LABEL: flat_system_acquire_ret_atomicrmw:
1858; GFX10-CU:       ; %bb.0: ; %entry
1859; GFX10-CU-NEXT:    s_clause 0x1
1860; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1861; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1862; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1863; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1864; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1865; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1866; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1867; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1868; GFX10-CU-NEXT:    buffer_gl0_inv
1869; GFX10-CU-NEXT:    buffer_gl1_inv
1870; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1871; GFX10-CU-NEXT:    s_endpgm
1872;
1873; SKIP-CACHE-INV-LABEL: flat_system_acquire_ret_atomicrmw:
1874; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1875; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1876; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1877; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1878; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1879; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1880; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1881; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1882; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1883; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1884; SKIP-CACHE-INV-NEXT:    s_endpgm
1885;
1886; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_ret_atomicrmw:
1887; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1888; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1889; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1890; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1891; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1892; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1893; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1894; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1895; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
1896; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1897; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1898; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1899;
1900; GFX90A-TGSPLIT-LABEL: flat_system_acquire_ret_atomicrmw:
1901; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1902; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1903; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1904; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1905; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1906; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1907; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1908; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1909; GFX90A-TGSPLIT-NEXT:    buffer_invl2
1910; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1911; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1912; GFX90A-TGSPLIT-NEXT:    s_endpgm
1913;
1914; GFX940-NOTTGSPLIT-LABEL: flat_system_acquire_ret_atomicrmw:
1915; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1916; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1917; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1918; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1919; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1920; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1921; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0 sc1
1922; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1923; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
1924; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1925; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1926;
1927; GFX940-TGSPLIT-LABEL: flat_system_acquire_ret_atomicrmw:
1928; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1929; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1930; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1931; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1932; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1933; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1934; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0 sc1
1935; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1936; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
1937; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1938; GFX940-TGSPLIT-NEXT:    s_endpgm
1939;
1940; GFX11-WGP-LABEL: flat_system_acquire_ret_atomicrmw:
1941; GFX11-WGP:       ; %bb.0: ; %entry
1942; GFX11-WGP-NEXT:    s_clause 0x1
1943; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1944; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1945; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1946; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1947; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1948; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1949; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
1950; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1951; GFX11-WGP-NEXT:    buffer_gl0_inv
1952; GFX11-WGP-NEXT:    buffer_gl1_inv
1953; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
1954; GFX11-WGP-NEXT:    s_endpgm
1955;
1956; GFX11-CU-LABEL: flat_system_acquire_ret_atomicrmw:
1957; GFX11-CU:       ; %bb.0: ; %entry
1958; GFX11-CU-NEXT:    s_clause 0x1
1959; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1960; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1961; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1962; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
1963; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
1964; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1965; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
1966; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1967; GFX11-CU-NEXT:    buffer_gl0_inv
1968; GFX11-CU-NEXT:    buffer_gl1_inv
1969; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
1970; GFX11-CU-NEXT:    s_endpgm
1971    i32* %out, i32 %in) {
1972entry:
1973  %val = atomicrmw volatile xchg i32* %out, i32 %in acquire
1974  store i32 %val, i32* %out, align 4
1975  ret void
1976}
1977
1978define amdgpu_kernel void @flat_system_acq_rel_ret_atomicrmw(
1979; GFX7-LABEL: flat_system_acq_rel_ret_atomicrmw:
1980; GFX7:       ; %bb.0: ; %entry
1981; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1982; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1983; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1984; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1985; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1986; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1987; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1988; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1989; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1990; GFX7-NEXT:    buffer_wbinvl1_vol
1991; GFX7-NEXT:    flat_store_dword v[0:1], v2
1992; GFX7-NEXT:    s_endpgm
1993;
1994; GFX10-WGP-LABEL: flat_system_acq_rel_ret_atomicrmw:
1995; GFX10-WGP:       ; %bb.0: ; %entry
1996; GFX10-WGP-NEXT:    s_clause 0x1
1997; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1998; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1999; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2000; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2001; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2002; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2003; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2004; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2005; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2006; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2007; GFX10-WGP-NEXT:    buffer_gl0_inv
2008; GFX10-WGP-NEXT:    buffer_gl1_inv
2009; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
2010; GFX10-WGP-NEXT:    s_endpgm
2011;
2012; GFX10-CU-LABEL: flat_system_acq_rel_ret_atomicrmw:
2013; GFX10-CU:       ; %bb.0: ; %entry
2014; GFX10-CU-NEXT:    s_clause 0x1
2015; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2016; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
2017; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2018; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2019; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2020; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2021; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2022; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2023; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2024; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2025; GFX10-CU-NEXT:    buffer_gl0_inv
2026; GFX10-CU-NEXT:    buffer_gl1_inv
2027; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
2028; GFX10-CU-NEXT:    s_endpgm
2029;
2030; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_ret_atomicrmw:
2031; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2032; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2033; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
2034; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2035; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2036; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2037; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2038; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2039; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2040; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2041; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
2042; SKIP-CACHE-INV-NEXT:    s_endpgm
2043;
2044; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_ret_atomicrmw:
2045; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2046; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2047; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
2048; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2049; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2050; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
2051; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
2052; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2053; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2054; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2055; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
2056; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2057; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2058; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2059;
2060; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_ret_atomicrmw:
2061; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2062; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2063; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
2064; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2065; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2066; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
2067; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
2068; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2069; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2070; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2071; GFX90A-TGSPLIT-NEXT:    buffer_invl2
2072; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2073; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2074; GFX90A-TGSPLIT-NEXT:    s_endpgm
2075;
2076; GFX940-NOTTGSPLIT-LABEL: flat_system_acq_rel_ret_atomicrmw:
2077; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2078; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2079; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
2080; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2081; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2082; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
2083; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
2084; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2085; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0 sc1
2086; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2087; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
2088; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2089; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2090;
2091; GFX940-TGSPLIT-LABEL: flat_system_acq_rel_ret_atomicrmw:
2092; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2093; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2094; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
2095; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2096; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2097; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
2098; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
2099; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2100; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0 sc1
2101; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2102; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
2103; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2104; GFX940-TGSPLIT-NEXT:    s_endpgm
2105;
2106; GFX11-WGP-LABEL: flat_system_acq_rel_ret_atomicrmw:
2107; GFX11-WGP:       ; %bb.0: ; %entry
2108; GFX11-WGP-NEXT:    s_clause 0x1
2109; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2110; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
2111; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2112; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2113; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2114; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2115; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2116; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2117; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
2118; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2119; GFX11-WGP-NEXT:    buffer_gl0_inv
2120; GFX11-WGP-NEXT:    buffer_gl1_inv
2121; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
2122; GFX11-WGP-NEXT:    s_endpgm
2123;
2124; GFX11-CU-LABEL: flat_system_acq_rel_ret_atomicrmw:
2125; GFX11-CU:       ; %bb.0: ; %entry
2126; GFX11-CU-NEXT:    s_clause 0x1
2127; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2128; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
2129; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2130; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2131; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2132; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2133; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2134; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2135; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
2136; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2137; GFX11-CU-NEXT:    buffer_gl0_inv
2138; GFX11-CU-NEXT:    buffer_gl1_inv
2139; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
2140; GFX11-CU-NEXT:    s_endpgm
2141    i32* %out, i32 %in) {
2142entry:
2143  %val = atomicrmw volatile xchg i32* %out, i32 %in acq_rel
2144  store i32 %val, i32* %out, align 4
2145  ret void
2146}
2147
2148define amdgpu_kernel void @flat_system_seq_cst_ret_atomicrmw(
2149; GFX7-LABEL: flat_system_seq_cst_ret_atomicrmw:
2150; GFX7:       ; %bb.0: ; %entry
2151; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2152; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
2153; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2154; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2155; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2156; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2157; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2158; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2159; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2160; GFX7-NEXT:    buffer_wbinvl1_vol
2161; GFX7-NEXT:    flat_store_dword v[0:1], v2
2162; GFX7-NEXT:    s_endpgm
2163;
2164; GFX10-WGP-LABEL: flat_system_seq_cst_ret_atomicrmw:
2165; GFX10-WGP:       ; %bb.0: ; %entry
2166; GFX10-WGP-NEXT:    s_clause 0x1
2167; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2168; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
2169; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2170; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2171; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2172; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2173; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2174; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2175; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2176; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2177; GFX10-WGP-NEXT:    buffer_gl0_inv
2178; GFX10-WGP-NEXT:    buffer_gl1_inv
2179; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
2180; GFX10-WGP-NEXT:    s_endpgm
2181;
2182; GFX10-CU-LABEL: flat_system_seq_cst_ret_atomicrmw:
2183; GFX10-CU:       ; %bb.0: ; %entry
2184; GFX10-CU-NEXT:    s_clause 0x1
2185; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2186; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
2187; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2188; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2189; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2190; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2191; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2192; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2193; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2194; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2195; GFX10-CU-NEXT:    buffer_gl0_inv
2196; GFX10-CU-NEXT:    buffer_gl1_inv
2197; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
2198; GFX10-CU-NEXT:    s_endpgm
2199;
2200; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_ret_atomicrmw:
2201; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2202; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2203; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
2204; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2205; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2206; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2207; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2208; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2209; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2210; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2211; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
2212; SKIP-CACHE-INV-NEXT:    s_endpgm
2213;
2214; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_ret_atomicrmw:
2215; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2216; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2217; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
2218; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2219; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2220; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
2221; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
2222; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2223; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2224; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2225; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
2226; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2227; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2228; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2229;
2230; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_ret_atomicrmw:
2231; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2232; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2233; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
2234; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2235; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2236; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
2237; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
2238; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2239; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2240; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2241; GFX90A-TGSPLIT-NEXT:    buffer_invl2
2242; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2243; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2244; GFX90A-TGSPLIT-NEXT:    s_endpgm
2245;
2246; GFX940-NOTTGSPLIT-LABEL: flat_system_seq_cst_ret_atomicrmw:
2247; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2248; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2249; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
2250; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2251; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2252; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
2253; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
2254; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2255; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0 sc1
2256; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2257; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
2258; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2259; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2260;
2261; GFX940-TGSPLIT-LABEL: flat_system_seq_cst_ret_atomicrmw:
2262; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2263; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2264; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
2265; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2266; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2267; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
2268; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
2269; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2270; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0 sc1
2271; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2272; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
2273; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2274; GFX940-TGSPLIT-NEXT:    s_endpgm
2275;
2276; GFX11-WGP-LABEL: flat_system_seq_cst_ret_atomicrmw:
2277; GFX11-WGP:       ; %bb.0: ; %entry
2278; GFX11-WGP-NEXT:    s_clause 0x1
2279; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2280; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
2281; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2282; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2283; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2284; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2285; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2286; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2287; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
2288; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2289; GFX11-WGP-NEXT:    buffer_gl0_inv
2290; GFX11-WGP-NEXT:    buffer_gl1_inv
2291; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
2292; GFX11-WGP-NEXT:    s_endpgm
2293;
2294; GFX11-CU-LABEL: flat_system_seq_cst_ret_atomicrmw:
2295; GFX11-CU:       ; %bb.0: ; %entry
2296; GFX11-CU-NEXT:    s_clause 0x1
2297; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2298; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
2299; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2300; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2301; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2302; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2303; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2304; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2305; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
2306; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2307; GFX11-CU-NEXT:    buffer_gl0_inv
2308; GFX11-CU-NEXT:    buffer_gl1_inv
2309; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
2310; GFX11-CU-NEXT:    s_endpgm
2311    i32* %out, i32 %in) {
2312entry:
2313  %val = atomicrmw volatile xchg i32* %out, i32 %in seq_cst
2314  store i32 %val, i32* %out, align 4
2315  ret void
2316}
2317
2318define amdgpu_kernel void @flat_system_monotonic_monotonic_cmpxchg(
2319; GFX7-LABEL: flat_system_monotonic_monotonic_cmpxchg:
2320; GFX7:       ; %bb.0: ; %entry
2321; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2322; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2323; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2324; GFX7-NEXT:    s_add_u32 s0, s0, 16
2325; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2326; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2327; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2328; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2329; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2330; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2331; GFX7-NEXT:    s_endpgm
2332;
2333; GFX10-WGP-LABEL: flat_system_monotonic_monotonic_cmpxchg:
2334; GFX10-WGP:       ; %bb.0: ; %entry
2335; GFX10-WGP-NEXT:    s_clause 0x1
2336; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2337; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2338; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2339; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2340; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2341; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2342; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2343; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2344; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2345; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2346; GFX10-WGP-NEXT:    s_endpgm
2347;
2348; GFX10-CU-LABEL: flat_system_monotonic_monotonic_cmpxchg:
2349; GFX10-CU:       ; %bb.0: ; %entry
2350; GFX10-CU-NEXT:    s_clause 0x1
2351; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2352; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2353; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2354; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2355; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2356; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2357; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2358; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2359; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2360; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2361; GFX10-CU-NEXT:    s_endpgm
2362;
2363; SKIP-CACHE-INV-LABEL: flat_system_monotonic_monotonic_cmpxchg:
2364; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2365; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2366; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2367; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2368; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2369; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2370; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2371; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2372; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2373; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2374; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2375; SKIP-CACHE-INV-NEXT:    s_endpgm
2376;
2377; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_monotonic_cmpxchg:
2378; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2379; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2380; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2381; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2382; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2383; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2384; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2385; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2386;
2387; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_monotonic_cmpxchg:
2388; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2389; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2390; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2391; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2392; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2393; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2394; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2395; GFX90A-TGSPLIT-NEXT:    s_endpgm
2396;
2397; GFX940-NOTTGSPLIT-LABEL: flat_system_monotonic_monotonic_cmpxchg:
2398; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2399; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2400; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2401; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2402; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2403; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2404; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
2405; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2406;
2407; GFX940-TGSPLIT-LABEL: flat_system_monotonic_monotonic_cmpxchg:
2408; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2409; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2410; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2411; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2412; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2413; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2414; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
2415; GFX940-TGSPLIT-NEXT:    s_endpgm
2416;
2417; GFX11-WGP-LABEL: flat_system_monotonic_monotonic_cmpxchg:
2418; GFX11-WGP:       ; %bb.0: ; %entry
2419; GFX11-WGP-NEXT:    s_clause 0x1
2420; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2421; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2422; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2423; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2424; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
2425; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2426; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2427; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2428; GFX11-WGP-NEXT:    s_endpgm
2429;
2430; GFX11-CU-LABEL: flat_system_monotonic_monotonic_cmpxchg:
2431; GFX11-CU:       ; %bb.0: ; %entry
2432; GFX11-CU-NEXT:    s_clause 0x1
2433; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2434; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2435; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2436; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2437; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
2438; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2439; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2440; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2441; GFX11-CU-NEXT:    s_endpgm
2442    i32* %out, i32 %in, i32 %old) {
2443entry:
2444  %gep = getelementptr i32, i32* %out, i32 4
2445  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in monotonic monotonic
2446  ret void
2447}
2448
2449define amdgpu_kernel void @flat_system_acquire_monotonic_cmpxchg(
2450; GFX7-LABEL: flat_system_acquire_monotonic_cmpxchg:
2451; GFX7:       ; %bb.0: ; %entry
2452; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2453; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2454; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2455; GFX7-NEXT:    s_add_u32 s0, s0, 16
2456; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2457; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2458; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2459; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2460; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2461; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2462; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2463; GFX7-NEXT:    buffer_wbinvl1_vol
2464; GFX7-NEXT:    s_endpgm
2465;
2466; GFX10-WGP-LABEL: flat_system_acquire_monotonic_cmpxchg:
2467; GFX10-WGP:       ; %bb.0: ; %entry
2468; GFX10-WGP-NEXT:    s_clause 0x1
2469; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2470; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2471; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2472; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2473; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2474; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2475; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2476; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2477; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2478; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2479; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2480; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2481; GFX10-WGP-NEXT:    buffer_gl0_inv
2482; GFX10-WGP-NEXT:    buffer_gl1_inv
2483; GFX10-WGP-NEXT:    s_endpgm
2484;
2485; GFX10-CU-LABEL: flat_system_acquire_monotonic_cmpxchg:
2486; GFX10-CU:       ; %bb.0: ; %entry
2487; GFX10-CU-NEXT:    s_clause 0x1
2488; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2489; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2490; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2491; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2492; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2493; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2494; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2495; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2496; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2497; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2498; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2499; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2500; GFX10-CU-NEXT:    buffer_gl0_inv
2501; GFX10-CU-NEXT:    buffer_gl1_inv
2502; GFX10-CU-NEXT:    s_endpgm
2503;
2504; SKIP-CACHE-INV-LABEL: flat_system_acquire_monotonic_cmpxchg:
2505; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2506; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2507; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2508; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2509; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2510; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2511; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2512; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2513; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2514; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2515; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2516; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2517; SKIP-CACHE-INV-NEXT:    s_endpgm
2518;
2519; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_monotonic_cmpxchg:
2520; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2521; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2522; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2523; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2524; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2525; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2526; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2527; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2528; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
2529; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2530; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2531;
2532; GFX90A-TGSPLIT-LABEL: flat_system_acquire_monotonic_cmpxchg:
2533; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2534; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2535; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2536; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2537; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2538; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2539; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2540; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2541; GFX90A-TGSPLIT-NEXT:    buffer_invl2
2542; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2543; GFX90A-TGSPLIT-NEXT:    s_endpgm
2544;
2545; GFX940-NOTTGSPLIT-LABEL: flat_system_acquire_monotonic_cmpxchg:
2546; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2547; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2548; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2549; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2550; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2551; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2552; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
2553; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2554; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
2555; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2556;
2557; GFX940-TGSPLIT-LABEL: flat_system_acquire_monotonic_cmpxchg:
2558; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2559; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2560; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2561; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2562; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2563; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2564; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
2565; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2566; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
2567; GFX940-TGSPLIT-NEXT:    s_endpgm
2568;
2569; GFX11-WGP-LABEL: flat_system_acquire_monotonic_cmpxchg:
2570; GFX11-WGP:       ; %bb.0: ; %entry
2571; GFX11-WGP-NEXT:    s_clause 0x1
2572; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2573; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2574; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2575; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2576; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
2577; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2578; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2579; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2580; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2581; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2582; GFX11-WGP-NEXT:    buffer_gl0_inv
2583; GFX11-WGP-NEXT:    buffer_gl1_inv
2584; GFX11-WGP-NEXT:    s_endpgm
2585;
2586; GFX11-CU-LABEL: flat_system_acquire_monotonic_cmpxchg:
2587; GFX11-CU:       ; %bb.0: ; %entry
2588; GFX11-CU-NEXT:    s_clause 0x1
2589; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2590; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2591; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2592; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2593; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
2594; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2595; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2596; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2597; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2598; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2599; GFX11-CU-NEXT:    buffer_gl0_inv
2600; GFX11-CU-NEXT:    buffer_gl1_inv
2601; GFX11-CU-NEXT:    s_endpgm
2602    i32* %out, i32 %in, i32 %old) {
2603entry:
2604  %gep = getelementptr i32, i32* %out, i32 4
2605  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acquire monotonic
2606  ret void
2607}
2608
2609define amdgpu_kernel void @flat_system_release_monotonic_cmpxchg(
2610; GFX7-LABEL: flat_system_release_monotonic_cmpxchg:
2611; GFX7:       ; %bb.0: ; %entry
2612; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2613; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2614; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2615; GFX7-NEXT:    s_add_u32 s0, s0, 16
2616; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2617; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2618; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2619; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2620; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2621; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2622; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2623; GFX7-NEXT:    s_endpgm
2624;
2625; GFX10-WGP-LABEL: flat_system_release_monotonic_cmpxchg:
2626; GFX10-WGP:       ; %bb.0: ; %entry
2627; GFX10-WGP-NEXT:    s_clause 0x1
2628; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2629; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2630; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2631; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2632; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2633; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2634; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2635; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2636; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2637; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2638; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2639; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2640; GFX10-WGP-NEXT:    s_endpgm
2641;
2642; GFX10-CU-LABEL: flat_system_release_monotonic_cmpxchg:
2643; GFX10-CU:       ; %bb.0: ; %entry
2644; GFX10-CU-NEXT:    s_clause 0x1
2645; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2646; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2647; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2648; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2649; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2650; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2651; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2652; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2653; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2654; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2655; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2656; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2657; GFX10-CU-NEXT:    s_endpgm
2658;
2659; SKIP-CACHE-INV-LABEL: flat_system_release_monotonic_cmpxchg:
2660; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2661; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2662; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2663; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2664; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2665; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2666; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2667; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2668; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2669; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2670; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2671; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2672; SKIP-CACHE-INV-NEXT:    s_endpgm
2673;
2674; GFX90A-NOTTGSPLIT-LABEL: flat_system_release_monotonic_cmpxchg:
2675; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2676; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2677; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2678; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2679; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2680; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2681; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
2682; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2683; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2684; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2685;
2686; GFX90A-TGSPLIT-LABEL: flat_system_release_monotonic_cmpxchg:
2687; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2688; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2689; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2690; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2691; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2692; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2693; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
2694; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2695; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2696; GFX90A-TGSPLIT-NEXT:    s_endpgm
2697;
2698; GFX940-NOTTGSPLIT-LABEL: flat_system_release_monotonic_cmpxchg:
2699; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2700; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2701; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2702; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2703; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2704; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2705; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
2706; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2707; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
2708; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2709;
2710; GFX940-TGSPLIT-LABEL: flat_system_release_monotonic_cmpxchg:
2711; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2712; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2713; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2714; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2715; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2716; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2717; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
2718; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2719; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
2720; GFX940-TGSPLIT-NEXT:    s_endpgm
2721;
2722; GFX11-WGP-LABEL: flat_system_release_monotonic_cmpxchg:
2723; GFX11-WGP:       ; %bb.0: ; %entry
2724; GFX11-WGP-NEXT:    s_clause 0x1
2725; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2726; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2727; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2728; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2729; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
2730; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2731; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2732; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2733; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2734; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2735; GFX11-WGP-NEXT:    s_endpgm
2736;
2737; GFX11-CU-LABEL: flat_system_release_monotonic_cmpxchg:
2738; GFX11-CU:       ; %bb.0: ; %entry
2739; GFX11-CU-NEXT:    s_clause 0x1
2740; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2741; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2742; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2743; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2744; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
2745; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2746; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2747; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2748; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2749; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2750; GFX11-CU-NEXT:    s_endpgm
2751    i32* %out, i32 %in, i32 %old) {
2752entry:
2753  %gep = getelementptr i32, i32* %out, i32 4
2754  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in release monotonic
2755  ret void
2756}
2757
2758define amdgpu_kernel void @flat_system_acq_rel_monotonic_cmpxchg(
2759; GFX7-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
2760; GFX7:       ; %bb.0: ; %entry
2761; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2762; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2763; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2764; GFX7-NEXT:    s_add_u32 s0, s0, 16
2765; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2766; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2767; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2768; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2769; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2770; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2771; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2772; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2773; GFX7-NEXT:    buffer_wbinvl1_vol
2774; GFX7-NEXT:    s_endpgm
2775;
2776; GFX10-WGP-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
2777; GFX10-WGP:       ; %bb.0: ; %entry
2778; GFX10-WGP-NEXT:    s_clause 0x1
2779; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2780; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2781; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2782; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2783; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2784; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2785; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2786; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2787; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2788; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2789; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2790; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2791; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2792; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2793; GFX10-WGP-NEXT:    buffer_gl0_inv
2794; GFX10-WGP-NEXT:    buffer_gl1_inv
2795; GFX10-WGP-NEXT:    s_endpgm
2796;
2797; GFX10-CU-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
2798; GFX10-CU:       ; %bb.0: ; %entry
2799; GFX10-CU-NEXT:    s_clause 0x1
2800; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2801; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2802; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2803; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2804; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2805; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2806; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2807; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2808; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2809; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2810; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2811; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2812; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2813; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2814; GFX10-CU-NEXT:    buffer_gl0_inv
2815; GFX10-CU-NEXT:    buffer_gl1_inv
2816; GFX10-CU-NEXT:    s_endpgm
2817;
2818; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
2819; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2820; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2821; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2822; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2823; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2824; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2825; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2826; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2827; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2828; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2829; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2830; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2831; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2832; SKIP-CACHE-INV-NEXT:    s_endpgm
2833;
2834; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
2835; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2836; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2837; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2838; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2839; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2840; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2841; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
2842; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2843; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2844; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2845; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
2846; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2847; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2848;
2849; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
2850; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2851; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2852; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2853; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2854; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2855; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2856; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
2857; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2858; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2859; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2860; GFX90A-TGSPLIT-NEXT:    buffer_invl2
2861; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2862; GFX90A-TGSPLIT-NEXT:    s_endpgm
2863;
2864; GFX940-NOTTGSPLIT-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
2865; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2866; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2867; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2868; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2869; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2870; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2871; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
2872; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2873; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
2874; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2875; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
2876; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2877;
2878; GFX940-TGSPLIT-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
2879; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2880; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2881; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2882; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2883; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2884; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2885; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
2886; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2887; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
2888; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2889; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
2890; GFX940-TGSPLIT-NEXT:    s_endpgm
2891;
2892; GFX11-WGP-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
2893; GFX11-WGP:       ; %bb.0: ; %entry
2894; GFX11-WGP-NEXT:    s_clause 0x1
2895; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2896; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2897; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2898; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2899; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
2900; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2901; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2902; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2903; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2904; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2905; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2906; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2907; GFX11-WGP-NEXT:    buffer_gl0_inv
2908; GFX11-WGP-NEXT:    buffer_gl1_inv
2909; GFX11-WGP-NEXT:    s_endpgm
2910;
2911; GFX11-CU-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
2912; GFX11-CU:       ; %bb.0: ; %entry
2913; GFX11-CU-NEXT:    s_clause 0x1
2914; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2915; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2916; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2917; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2918; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
2919; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2920; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2921; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2922; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2923; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2924; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2925; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2926; GFX11-CU-NEXT:    buffer_gl0_inv
2927; GFX11-CU-NEXT:    buffer_gl1_inv
2928; GFX11-CU-NEXT:    s_endpgm
2929    i32* %out, i32 %in, i32 %old) {
2930entry:
2931  %gep = getelementptr i32, i32* %out, i32 4
2932  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acq_rel monotonic
2933  ret void
2934}
2935
2936define amdgpu_kernel void @flat_system_seq_cst_monotonic_cmpxchg(
2937; GFX7-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
2938; GFX7:       ; %bb.0: ; %entry
2939; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2940; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2941; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2942; GFX7-NEXT:    s_add_u32 s0, s0, 16
2943; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2944; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2945; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2946; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2947; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2948; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2949; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2950; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2951; GFX7-NEXT:    buffer_wbinvl1_vol
2952; GFX7-NEXT:    s_endpgm
2953;
2954; GFX10-WGP-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
2955; GFX10-WGP:       ; %bb.0: ; %entry
2956; GFX10-WGP-NEXT:    s_clause 0x1
2957; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2958; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2959; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2960; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2961; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2962; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2963; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2964; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2965; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2966; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2967; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2968; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2969; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2970; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2971; GFX10-WGP-NEXT:    buffer_gl0_inv
2972; GFX10-WGP-NEXT:    buffer_gl1_inv
2973; GFX10-WGP-NEXT:    s_endpgm
2974;
2975; GFX10-CU-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
2976; GFX10-CU:       ; %bb.0: ; %entry
2977; GFX10-CU-NEXT:    s_clause 0x1
2978; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2979; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2980; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2981; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2982; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2983; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2984; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2985; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2986; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2987; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2988; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2989; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2990; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2991; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2992; GFX10-CU-NEXT:    buffer_gl0_inv
2993; GFX10-CU-NEXT:    buffer_gl1_inv
2994; GFX10-CU-NEXT:    s_endpgm
2995;
2996; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
2997; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2998; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2999; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3000; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3001; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3002; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3003; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3004; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3005; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3006; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3007; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3008; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3009; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3010; SKIP-CACHE-INV-NEXT:    s_endpgm
3011;
3012; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
3013; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3014; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3015; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3016; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3017; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3018; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3019; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
3020; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3021; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3022; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3023; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
3024; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3025; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3026;
3027; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
3028; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3029; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3030; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3031; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3032; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3033; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3034; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
3035; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3036; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3037; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3038; GFX90A-TGSPLIT-NEXT:    buffer_invl2
3039; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3040; GFX90A-TGSPLIT-NEXT:    s_endpgm
3041;
3042; GFX940-NOTTGSPLIT-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
3043; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3044; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3045; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3046; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3047; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3048; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3049; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
3050; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3051; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
3052; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3053; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
3054; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3055;
3056; GFX940-TGSPLIT-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
3057; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3058; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3059; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3060; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3061; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3062; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3063; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
3064; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3065; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
3066; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3067; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
3068; GFX940-TGSPLIT-NEXT:    s_endpgm
3069;
3070; GFX11-WGP-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
3071; GFX11-WGP:       ; %bb.0: ; %entry
3072; GFX11-WGP-NEXT:    s_clause 0x1
3073; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3074; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3075; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3076; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3077; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
3078; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3079; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
3080; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3081; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3082; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3083; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3084; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3085; GFX11-WGP-NEXT:    buffer_gl0_inv
3086; GFX11-WGP-NEXT:    buffer_gl1_inv
3087; GFX11-WGP-NEXT:    s_endpgm
3088;
3089; GFX11-CU-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
3090; GFX11-CU:       ; %bb.0: ; %entry
3091; GFX11-CU-NEXT:    s_clause 0x1
3092; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3093; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3094; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3095; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3096; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
3097; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3098; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
3099; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3100; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3101; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3102; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3103; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3104; GFX11-CU-NEXT:    buffer_gl0_inv
3105; GFX11-CU-NEXT:    buffer_gl1_inv
3106; GFX11-CU-NEXT:    s_endpgm
3107    i32* %out, i32 %in, i32 %old) {
3108entry:
3109  %gep = getelementptr i32, i32* %out, i32 4
3110  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst monotonic
3111  ret void
3112}
3113
3114define amdgpu_kernel void @flat_system_monotonic_acquire_cmpxchg(
3115; GFX7-LABEL: flat_system_monotonic_acquire_cmpxchg:
3116; GFX7:       ; %bb.0: ; %entry
3117; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3118; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3119; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3120; GFX7-NEXT:    s_add_u32 s0, s0, 16
3121; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3122; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3123; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3124; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3125; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3126; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3127; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3128; GFX7-NEXT:    buffer_wbinvl1_vol
3129; GFX7-NEXT:    s_endpgm
3130;
3131; GFX10-WGP-LABEL: flat_system_monotonic_acquire_cmpxchg:
3132; GFX10-WGP:       ; %bb.0: ; %entry
3133; GFX10-WGP-NEXT:    s_clause 0x1
3134; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3135; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3136; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3137; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3138; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3139; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3140; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3141; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3142; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3143; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3144; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3145; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3146; GFX10-WGP-NEXT:    buffer_gl0_inv
3147; GFX10-WGP-NEXT:    buffer_gl1_inv
3148; GFX10-WGP-NEXT:    s_endpgm
3149;
3150; GFX10-CU-LABEL: flat_system_monotonic_acquire_cmpxchg:
3151; GFX10-CU:       ; %bb.0: ; %entry
3152; GFX10-CU-NEXT:    s_clause 0x1
3153; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3154; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3155; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3156; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3157; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3158; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3159; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3160; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3161; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3162; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3163; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3164; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3165; GFX10-CU-NEXT:    buffer_gl0_inv
3166; GFX10-CU-NEXT:    buffer_gl1_inv
3167; GFX10-CU-NEXT:    s_endpgm
3168;
3169; SKIP-CACHE-INV-LABEL: flat_system_monotonic_acquire_cmpxchg:
3170; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3171; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3172; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3173; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3174; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3175; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3176; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3177; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3178; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3179; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3180; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3181; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3182; SKIP-CACHE-INV-NEXT:    s_endpgm
3183;
3184; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_acquire_cmpxchg:
3185; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3186; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3187; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3188; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3189; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3190; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3191; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3192; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3193; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
3194; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3195; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3196;
3197; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_acquire_cmpxchg:
3198; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3199; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3200; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3201; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3202; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3203; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3204; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3205; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3206; GFX90A-TGSPLIT-NEXT:    buffer_invl2
3207; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3208; GFX90A-TGSPLIT-NEXT:    s_endpgm
3209;
3210; GFX940-NOTTGSPLIT-LABEL: flat_system_monotonic_acquire_cmpxchg:
3211; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3212; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3213; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3214; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3215; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3216; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3217; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
3218; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3219; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
3220; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3221;
3222; GFX940-TGSPLIT-LABEL: flat_system_monotonic_acquire_cmpxchg:
3223; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3224; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3225; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3226; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3227; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3228; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3229; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
3230; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3231; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
3232; GFX940-TGSPLIT-NEXT:    s_endpgm
3233;
3234; GFX11-WGP-LABEL: flat_system_monotonic_acquire_cmpxchg:
3235; GFX11-WGP:       ; %bb.0: ; %entry
3236; GFX11-WGP-NEXT:    s_clause 0x1
3237; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3238; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3239; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3240; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3241; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
3242; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3243; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
3244; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3245; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3246; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3247; GFX11-WGP-NEXT:    buffer_gl0_inv
3248; GFX11-WGP-NEXT:    buffer_gl1_inv
3249; GFX11-WGP-NEXT:    s_endpgm
3250;
3251; GFX11-CU-LABEL: flat_system_monotonic_acquire_cmpxchg:
3252; GFX11-CU:       ; %bb.0: ; %entry
3253; GFX11-CU-NEXT:    s_clause 0x1
3254; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3255; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3256; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3257; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3258; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
3259; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3260; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
3261; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3262; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3263; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3264; GFX11-CU-NEXT:    buffer_gl0_inv
3265; GFX11-CU-NEXT:    buffer_gl1_inv
3266; GFX11-CU-NEXT:    s_endpgm
3267    i32* %out, i32 %in, i32 %old) {
3268entry:
3269  %gep = getelementptr i32, i32* %out, i32 4
3270  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in monotonic acquire
3271  ret void
3272}
3273
3274define amdgpu_kernel void @flat_system_acquire_acquire_cmpxchg(
3275; GFX7-LABEL: flat_system_acquire_acquire_cmpxchg:
3276; GFX7:       ; %bb.0: ; %entry
3277; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3278; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3279; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3280; GFX7-NEXT:    s_add_u32 s0, s0, 16
3281; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3282; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3283; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3284; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3285; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3286; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3287; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3288; GFX7-NEXT:    buffer_wbinvl1_vol
3289; GFX7-NEXT:    s_endpgm
3290;
3291; GFX10-WGP-LABEL: flat_system_acquire_acquire_cmpxchg:
3292; GFX10-WGP:       ; %bb.0: ; %entry
3293; GFX10-WGP-NEXT:    s_clause 0x1
3294; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3295; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3296; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3297; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3298; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3299; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3300; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3301; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3302; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3303; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3304; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3305; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3306; GFX10-WGP-NEXT:    buffer_gl0_inv
3307; GFX10-WGP-NEXT:    buffer_gl1_inv
3308; GFX10-WGP-NEXT:    s_endpgm
3309;
3310; GFX10-CU-LABEL: flat_system_acquire_acquire_cmpxchg:
3311; GFX10-CU:       ; %bb.0: ; %entry
3312; GFX10-CU-NEXT:    s_clause 0x1
3313; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3314; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3315; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3316; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3317; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3318; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3319; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3320; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3321; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3322; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3323; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3324; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3325; GFX10-CU-NEXT:    buffer_gl0_inv
3326; GFX10-CU-NEXT:    buffer_gl1_inv
3327; GFX10-CU-NEXT:    s_endpgm
3328;
3329; SKIP-CACHE-INV-LABEL: flat_system_acquire_acquire_cmpxchg:
3330; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3331; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3332; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3333; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3334; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3335; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3336; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3337; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3338; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3339; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3340; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3341; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3342; SKIP-CACHE-INV-NEXT:    s_endpgm
3343;
3344; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_acquire_cmpxchg:
3345; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3346; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3347; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3348; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3349; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3350; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3351; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3352; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3353; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
3354; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3355; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3356;
3357; GFX90A-TGSPLIT-LABEL: flat_system_acquire_acquire_cmpxchg:
3358; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3359; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3360; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3361; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3362; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3363; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3364; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3365; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3366; GFX90A-TGSPLIT-NEXT:    buffer_invl2
3367; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3368; GFX90A-TGSPLIT-NEXT:    s_endpgm
3369;
3370; GFX940-NOTTGSPLIT-LABEL: flat_system_acquire_acquire_cmpxchg:
3371; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3372; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3373; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3374; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3375; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3376; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3377; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
3378; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3379; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
3380; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3381;
3382; GFX940-TGSPLIT-LABEL: flat_system_acquire_acquire_cmpxchg:
3383; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3384; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3385; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3386; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3387; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3388; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3389; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
3390; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3391; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
3392; GFX940-TGSPLIT-NEXT:    s_endpgm
3393;
3394; GFX11-WGP-LABEL: flat_system_acquire_acquire_cmpxchg:
3395; GFX11-WGP:       ; %bb.0: ; %entry
3396; GFX11-WGP-NEXT:    s_clause 0x1
3397; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3398; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3399; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3400; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3401; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
3402; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3403; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
3404; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3405; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3406; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3407; GFX11-WGP-NEXT:    buffer_gl0_inv
3408; GFX11-WGP-NEXT:    buffer_gl1_inv
3409; GFX11-WGP-NEXT:    s_endpgm
3410;
3411; GFX11-CU-LABEL: flat_system_acquire_acquire_cmpxchg:
3412; GFX11-CU:       ; %bb.0: ; %entry
3413; GFX11-CU-NEXT:    s_clause 0x1
3414; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3415; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3416; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3417; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3418; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
3419; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3420; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
3421; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3422; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3423; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3424; GFX11-CU-NEXT:    buffer_gl0_inv
3425; GFX11-CU-NEXT:    buffer_gl1_inv
3426; GFX11-CU-NEXT:    s_endpgm
3427    i32* %out, i32 %in, i32 %old) {
3428entry:
3429  %gep = getelementptr i32, i32* %out, i32 4
3430  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acquire acquire
3431  ret void
3432}
3433
3434define amdgpu_kernel void @flat_system_release_acquire_cmpxchg(
3435; GFX7-LABEL: flat_system_release_acquire_cmpxchg:
3436; GFX7:       ; %bb.0: ; %entry
3437; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3438; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3439; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3440; GFX7-NEXT:    s_add_u32 s0, s0, 16
3441; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3442; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3443; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3444; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3445; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3446; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3447; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3448; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3449; GFX7-NEXT:    buffer_wbinvl1_vol
3450; GFX7-NEXT:    s_endpgm
3451;
3452; GFX10-WGP-LABEL: flat_system_release_acquire_cmpxchg:
3453; GFX10-WGP:       ; %bb.0: ; %entry
3454; GFX10-WGP-NEXT:    s_clause 0x1
3455; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3456; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3457; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3458; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3459; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3460; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3461; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3462; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3463; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3464; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3465; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3466; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3467; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3468; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3469; GFX10-WGP-NEXT:    buffer_gl0_inv
3470; GFX10-WGP-NEXT:    buffer_gl1_inv
3471; GFX10-WGP-NEXT:    s_endpgm
3472;
3473; GFX10-CU-LABEL: flat_system_release_acquire_cmpxchg:
3474; GFX10-CU:       ; %bb.0: ; %entry
3475; GFX10-CU-NEXT:    s_clause 0x1
3476; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3477; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3478; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3479; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3480; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3481; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3482; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3483; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3484; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3485; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3486; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3487; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3488; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3489; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3490; GFX10-CU-NEXT:    buffer_gl0_inv
3491; GFX10-CU-NEXT:    buffer_gl1_inv
3492; GFX10-CU-NEXT:    s_endpgm
3493;
3494; SKIP-CACHE-INV-LABEL: flat_system_release_acquire_cmpxchg:
3495; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3496; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3497; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3498; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3499; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3500; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3501; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3502; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3503; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3504; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3505; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3506; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3507; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3508; SKIP-CACHE-INV-NEXT:    s_endpgm
3509;
3510; GFX90A-NOTTGSPLIT-LABEL: flat_system_release_acquire_cmpxchg:
3511; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3512; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3513; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3514; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3515; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3516; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3517; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
3518; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3519; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3520; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3521; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
3522; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3523; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3524;
3525; GFX90A-TGSPLIT-LABEL: flat_system_release_acquire_cmpxchg:
3526; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3527; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3528; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3529; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3530; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3531; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3532; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
3533; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3534; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3535; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3536; GFX90A-TGSPLIT-NEXT:    buffer_invl2
3537; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3538; GFX90A-TGSPLIT-NEXT:    s_endpgm
3539;
3540; GFX940-NOTTGSPLIT-LABEL: flat_system_release_acquire_cmpxchg:
3541; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3542; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3543; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3544; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3545; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3546; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3547; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
3548; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3549; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
3550; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3551; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
3552; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3553;
3554; GFX940-TGSPLIT-LABEL: flat_system_release_acquire_cmpxchg:
3555; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3556; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3557; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3558; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3559; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3560; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3561; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
3562; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3563; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
3564; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3565; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
3566; GFX940-TGSPLIT-NEXT:    s_endpgm
3567;
3568; GFX11-WGP-LABEL: flat_system_release_acquire_cmpxchg:
3569; GFX11-WGP:       ; %bb.0: ; %entry
3570; GFX11-WGP-NEXT:    s_clause 0x1
3571; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3572; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3573; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3574; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3575; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
3576; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3577; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
3578; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3579; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3580; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3581; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3582; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3583; GFX11-WGP-NEXT:    buffer_gl0_inv
3584; GFX11-WGP-NEXT:    buffer_gl1_inv
3585; GFX11-WGP-NEXT:    s_endpgm
3586;
3587; GFX11-CU-LABEL: flat_system_release_acquire_cmpxchg:
3588; GFX11-CU:       ; %bb.0: ; %entry
3589; GFX11-CU-NEXT:    s_clause 0x1
3590; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3591; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3592; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3593; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3594; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
3595; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3596; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
3597; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3598; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3599; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3600; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3601; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3602; GFX11-CU-NEXT:    buffer_gl0_inv
3603; GFX11-CU-NEXT:    buffer_gl1_inv
3604; GFX11-CU-NEXT:    s_endpgm
3605    i32* %out, i32 %in, i32 %old) {
3606entry:
3607  %gep = getelementptr i32, i32* %out, i32 4
3608  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in release acquire
3609  ret void
3610}
3611
3612define amdgpu_kernel void @flat_system_acq_rel_acquire_cmpxchg(
3613; GFX7-LABEL: flat_system_acq_rel_acquire_cmpxchg:
3614; GFX7:       ; %bb.0: ; %entry
3615; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3616; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3617; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3618; GFX7-NEXT:    s_add_u32 s0, s0, 16
3619; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3620; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3621; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3622; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3623; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3624; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3625; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3626; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3627; GFX7-NEXT:    buffer_wbinvl1_vol
3628; GFX7-NEXT:    s_endpgm
3629;
3630; GFX10-WGP-LABEL: flat_system_acq_rel_acquire_cmpxchg:
3631; GFX10-WGP:       ; %bb.0: ; %entry
3632; GFX10-WGP-NEXT:    s_clause 0x1
3633; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3634; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3635; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3636; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3637; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3638; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3639; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3640; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3641; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3642; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3643; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3644; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3645; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3646; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3647; GFX10-WGP-NEXT:    buffer_gl0_inv
3648; GFX10-WGP-NEXT:    buffer_gl1_inv
3649; GFX10-WGP-NEXT:    s_endpgm
3650;
3651; GFX10-CU-LABEL: flat_system_acq_rel_acquire_cmpxchg:
3652; GFX10-CU:       ; %bb.0: ; %entry
3653; GFX10-CU-NEXT:    s_clause 0x1
3654; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3655; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3656; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3657; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3658; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3659; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3660; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3661; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3662; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3663; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3664; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3665; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3666; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3667; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3668; GFX10-CU-NEXT:    buffer_gl0_inv
3669; GFX10-CU-NEXT:    buffer_gl1_inv
3670; GFX10-CU-NEXT:    s_endpgm
3671;
3672; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_acquire_cmpxchg:
3673; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3674; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3675; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3676; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3677; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3678; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3679; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3680; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3681; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3682; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3683; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3684; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3685; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3686; SKIP-CACHE-INV-NEXT:    s_endpgm
3687;
3688; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_acquire_cmpxchg:
3689; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3690; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3691; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3692; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3693; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3694; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3695; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
3696; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3697; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3698; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3699; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
3700; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3701; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3702;
3703; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_acquire_cmpxchg:
3704; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3705; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3706; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3707; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3708; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3709; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3710; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
3711; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3712; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3713; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3714; GFX90A-TGSPLIT-NEXT:    buffer_invl2
3715; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3716; GFX90A-TGSPLIT-NEXT:    s_endpgm
3717;
3718; GFX940-NOTTGSPLIT-LABEL: flat_system_acq_rel_acquire_cmpxchg:
3719; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3720; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3721; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3722; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3723; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3724; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3725; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
3726; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3727; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
3728; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3729; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
3730; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3731;
3732; GFX940-TGSPLIT-LABEL: flat_system_acq_rel_acquire_cmpxchg:
3733; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3734; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3735; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3736; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3737; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3738; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3739; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
3740; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3741; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
3742; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3743; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
3744; GFX940-TGSPLIT-NEXT:    s_endpgm
3745;
3746; GFX11-WGP-LABEL: flat_system_acq_rel_acquire_cmpxchg:
3747; GFX11-WGP:       ; %bb.0: ; %entry
3748; GFX11-WGP-NEXT:    s_clause 0x1
3749; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3750; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3751; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3752; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3753; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
3754; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3755; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
3756; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3757; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3758; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3759; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3760; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3761; GFX11-WGP-NEXT:    buffer_gl0_inv
3762; GFX11-WGP-NEXT:    buffer_gl1_inv
3763; GFX11-WGP-NEXT:    s_endpgm
3764;
3765; GFX11-CU-LABEL: flat_system_acq_rel_acquire_cmpxchg:
3766; GFX11-CU:       ; %bb.0: ; %entry
3767; GFX11-CU-NEXT:    s_clause 0x1
3768; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3769; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3770; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3771; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3772; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
3773; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3774; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
3775; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3776; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3777; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3778; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3779; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3780; GFX11-CU-NEXT:    buffer_gl0_inv
3781; GFX11-CU-NEXT:    buffer_gl1_inv
3782; GFX11-CU-NEXT:    s_endpgm
3783    i32* %out, i32 %in, i32 %old) {
3784entry:
3785  %gep = getelementptr i32, i32* %out, i32 4
3786  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acq_rel acquire
3787  ret void
3788}
3789
3790define amdgpu_kernel void @flat_system_seq_cst_acquire_cmpxchg(
3791; GFX7-LABEL: flat_system_seq_cst_acquire_cmpxchg:
3792; GFX7:       ; %bb.0: ; %entry
3793; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3794; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3795; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3796; GFX7-NEXT:    s_add_u32 s0, s0, 16
3797; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3798; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3799; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3800; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3801; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3802; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3803; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3804; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3805; GFX7-NEXT:    buffer_wbinvl1_vol
3806; GFX7-NEXT:    s_endpgm
3807;
3808; GFX10-WGP-LABEL: flat_system_seq_cst_acquire_cmpxchg:
3809; GFX10-WGP:       ; %bb.0: ; %entry
3810; GFX10-WGP-NEXT:    s_clause 0x1
3811; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3812; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3813; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3814; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3815; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3816; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3817; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3818; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3819; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3820; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3821; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3822; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3823; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3824; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3825; GFX10-WGP-NEXT:    buffer_gl0_inv
3826; GFX10-WGP-NEXT:    buffer_gl1_inv
3827; GFX10-WGP-NEXT:    s_endpgm
3828;
3829; GFX10-CU-LABEL: flat_system_seq_cst_acquire_cmpxchg:
3830; GFX10-CU:       ; %bb.0: ; %entry
3831; GFX10-CU-NEXT:    s_clause 0x1
3832; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3833; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3834; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3835; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3836; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3837; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3838; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3839; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3840; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3841; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3842; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3843; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3844; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3845; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3846; GFX10-CU-NEXT:    buffer_gl0_inv
3847; GFX10-CU-NEXT:    buffer_gl1_inv
3848; GFX10-CU-NEXT:    s_endpgm
3849;
3850; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_acquire_cmpxchg:
3851; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3852; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3853; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3854; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3855; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3856; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3857; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3858; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3859; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3860; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3861; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3862; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3863; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3864; SKIP-CACHE-INV-NEXT:    s_endpgm
3865;
3866; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_acquire_cmpxchg:
3867; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3868; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3869; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3870; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3871; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3872; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3873; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
3874; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3875; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3876; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3877; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
3878; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3879; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3880;
3881; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_acquire_cmpxchg:
3882; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3883; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3884; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3885; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3886; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3887; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3888; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
3889; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3890; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3891; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3892; GFX90A-TGSPLIT-NEXT:    buffer_invl2
3893; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3894; GFX90A-TGSPLIT-NEXT:    s_endpgm
3895;
3896; GFX940-NOTTGSPLIT-LABEL: flat_system_seq_cst_acquire_cmpxchg:
3897; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3898; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3899; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3900; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3901; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3902; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3903; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
3904; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3905; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
3906; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3907; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
3908; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3909;
3910; GFX940-TGSPLIT-LABEL: flat_system_seq_cst_acquire_cmpxchg:
3911; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3912; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3913; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3914; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3915; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3916; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3917; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
3918; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3919; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
3920; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3921; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
3922; GFX940-TGSPLIT-NEXT:    s_endpgm
3923;
3924; GFX11-WGP-LABEL: flat_system_seq_cst_acquire_cmpxchg:
3925; GFX11-WGP:       ; %bb.0: ; %entry
3926; GFX11-WGP-NEXT:    s_clause 0x1
3927; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3928; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3929; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3930; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3931; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
3932; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3933; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
3934; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3935; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3936; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3937; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3938; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3939; GFX11-WGP-NEXT:    buffer_gl0_inv
3940; GFX11-WGP-NEXT:    buffer_gl1_inv
3941; GFX11-WGP-NEXT:    s_endpgm
3942;
3943; GFX11-CU-LABEL: flat_system_seq_cst_acquire_cmpxchg:
3944; GFX11-CU:       ; %bb.0: ; %entry
3945; GFX11-CU-NEXT:    s_clause 0x1
3946; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3947; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3948; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3949; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3950; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
3951; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3952; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
3953; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3954; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3955; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3956; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3957; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3958; GFX11-CU-NEXT:    buffer_gl0_inv
3959; GFX11-CU-NEXT:    buffer_gl1_inv
3960; GFX11-CU-NEXT:    s_endpgm
3961    i32* %out, i32 %in, i32 %old) {
3962entry:
3963  %gep = getelementptr i32, i32* %out, i32 4
3964  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst acquire
3965  ret void
3966}
3967
3968define amdgpu_kernel void @flat_system_monotonic_seq_cst_cmpxchg(
3969; GFX7-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
3970; GFX7:       ; %bb.0: ; %entry
3971; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3972; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3973; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3974; GFX7-NEXT:    s_add_u32 s0, s0, 16
3975; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3976; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3977; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3978; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3979; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3980; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3981; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3982; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3983; GFX7-NEXT:    buffer_wbinvl1_vol
3984; GFX7-NEXT:    s_endpgm
3985;
3986; GFX10-WGP-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
3987; GFX10-WGP:       ; %bb.0: ; %entry
3988; GFX10-WGP-NEXT:    s_clause 0x1
3989; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3990; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3991; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3992; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3993; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3994; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3995; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3996; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3997; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3998; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3999; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4000; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4001; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4002; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4003; GFX10-WGP-NEXT:    buffer_gl0_inv
4004; GFX10-WGP-NEXT:    buffer_gl1_inv
4005; GFX10-WGP-NEXT:    s_endpgm
4006;
4007; GFX10-CU-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
4008; GFX10-CU:       ; %bb.0: ; %entry
4009; GFX10-CU-NEXT:    s_clause 0x1
4010; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4011; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4012; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4013; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
4014; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
4015; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4016; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4017; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4018; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4019; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4020; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4021; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4022; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4023; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4024; GFX10-CU-NEXT:    buffer_gl0_inv
4025; GFX10-CU-NEXT:    buffer_gl1_inv
4026; GFX10-CU-NEXT:    s_endpgm
4027;
4028; SKIP-CACHE-INV-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
4029; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4030; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4031; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4032; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4033; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
4034; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
4035; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4036; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4037; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4038; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4039; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4040; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4041; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4042; SKIP-CACHE-INV-NEXT:    s_endpgm
4043;
4044; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
4045; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4046; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4047; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4048; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4049; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4050; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4051; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
4052; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4053; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4054; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4055; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
4056; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4057; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4058;
4059; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
4060; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4061; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4062; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4063; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4064; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4065; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4066; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
4067; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4068; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4069; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4070; GFX90A-TGSPLIT-NEXT:    buffer_invl2
4071; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4072; GFX90A-TGSPLIT-NEXT:    s_endpgm
4073;
4074; GFX940-NOTTGSPLIT-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
4075; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4076; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4077; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4078; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4079; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4080; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4081; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
4082; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4083; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
4084; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4085; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
4086; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4087;
4088; GFX940-TGSPLIT-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
4089; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4090; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4091; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4092; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4093; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4094; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4095; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
4096; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4097; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
4098; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4099; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
4100; GFX940-TGSPLIT-NEXT:    s_endpgm
4101;
4102; GFX11-WGP-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
4103; GFX11-WGP:       ; %bb.0: ; %entry
4104; GFX11-WGP-NEXT:    s_clause 0x1
4105; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4106; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4107; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4108; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
4109; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
4110; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
4111; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
4112; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4113; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4114; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4115; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4116; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4117; GFX11-WGP-NEXT:    buffer_gl0_inv
4118; GFX11-WGP-NEXT:    buffer_gl1_inv
4119; GFX11-WGP-NEXT:    s_endpgm
4120;
4121; GFX11-CU-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
4122; GFX11-CU:       ; %bb.0: ; %entry
4123; GFX11-CU-NEXT:    s_clause 0x1
4124; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4125; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4126; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4127; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
4128; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
4129; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
4130; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
4131; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4132; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4133; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4134; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4135; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4136; GFX11-CU-NEXT:    buffer_gl0_inv
4137; GFX11-CU-NEXT:    buffer_gl1_inv
4138; GFX11-CU-NEXT:    s_endpgm
4139    i32* %out, i32 %in, i32 %old) {
4140entry:
4141  %gep = getelementptr i32, i32* %out, i32 4
4142  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in monotonic seq_cst
4143  ret void
4144}
4145
4146define amdgpu_kernel void @flat_system_acquire_seq_cst_cmpxchg(
4147; GFX7-LABEL: flat_system_acquire_seq_cst_cmpxchg:
4148; GFX7:       ; %bb.0: ; %entry
4149; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4150; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4151; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4152; GFX7-NEXT:    s_add_u32 s0, s0, 16
4153; GFX7-NEXT:    s_addc_u32 s1, s1, 0
4154; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4155; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4156; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4157; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4158; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4159; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4160; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4161; GFX7-NEXT:    buffer_wbinvl1_vol
4162; GFX7-NEXT:    s_endpgm
4163;
4164; GFX10-WGP-LABEL: flat_system_acquire_seq_cst_cmpxchg:
4165; GFX10-WGP:       ; %bb.0: ; %entry
4166; GFX10-WGP-NEXT:    s_clause 0x1
4167; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4168; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4169; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4170; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
4171; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
4172; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4173; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4174; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4175; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4176; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4177; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4178; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4179; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4180; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4181; GFX10-WGP-NEXT:    buffer_gl0_inv
4182; GFX10-WGP-NEXT:    buffer_gl1_inv
4183; GFX10-WGP-NEXT:    s_endpgm
4184;
4185; GFX10-CU-LABEL: flat_system_acquire_seq_cst_cmpxchg:
4186; GFX10-CU:       ; %bb.0: ; %entry
4187; GFX10-CU-NEXT:    s_clause 0x1
4188; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4189; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4190; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4191; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
4192; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
4193; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4194; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4195; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4196; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4197; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4198; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4199; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4200; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4201; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4202; GFX10-CU-NEXT:    buffer_gl0_inv
4203; GFX10-CU-NEXT:    buffer_gl1_inv
4204; GFX10-CU-NEXT:    s_endpgm
4205;
4206; SKIP-CACHE-INV-LABEL: flat_system_acquire_seq_cst_cmpxchg:
4207; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4208; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4209; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4210; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4211; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
4212; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
4213; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4214; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4215; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4216; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4217; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4218; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4219; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4220; SKIP-CACHE-INV-NEXT:    s_endpgm
4221;
4222; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_seq_cst_cmpxchg:
4223; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4224; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4225; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4226; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4227; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4228; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4229; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
4230; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4231; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4232; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4233; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
4234; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4235; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4236;
4237; GFX90A-TGSPLIT-LABEL: flat_system_acquire_seq_cst_cmpxchg:
4238; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4239; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4240; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4241; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4242; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4243; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4244; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
4245; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4246; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4247; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4248; GFX90A-TGSPLIT-NEXT:    buffer_invl2
4249; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4250; GFX90A-TGSPLIT-NEXT:    s_endpgm
4251;
4252; GFX940-NOTTGSPLIT-LABEL: flat_system_acquire_seq_cst_cmpxchg:
4253; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4254; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4255; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4256; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4257; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4258; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4259; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
4260; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4261; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
4262; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4263; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
4264; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4265;
4266; GFX940-TGSPLIT-LABEL: flat_system_acquire_seq_cst_cmpxchg:
4267; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4268; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4269; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4270; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4271; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4272; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4273; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
4274; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4275; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
4276; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4277; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
4278; GFX940-TGSPLIT-NEXT:    s_endpgm
4279;
4280; GFX11-WGP-LABEL: flat_system_acquire_seq_cst_cmpxchg:
4281; GFX11-WGP:       ; %bb.0: ; %entry
4282; GFX11-WGP-NEXT:    s_clause 0x1
4283; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4284; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4285; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4286; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
4287; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
4288; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
4289; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
4290; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4291; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4292; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4293; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4294; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4295; GFX11-WGP-NEXT:    buffer_gl0_inv
4296; GFX11-WGP-NEXT:    buffer_gl1_inv
4297; GFX11-WGP-NEXT:    s_endpgm
4298;
4299; GFX11-CU-LABEL: flat_system_acquire_seq_cst_cmpxchg:
4300; GFX11-CU:       ; %bb.0: ; %entry
4301; GFX11-CU-NEXT:    s_clause 0x1
4302; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4303; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4304; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4305; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
4306; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
4307; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
4308; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
4309; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4310; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4311; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4312; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4313; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4314; GFX11-CU-NEXT:    buffer_gl0_inv
4315; GFX11-CU-NEXT:    buffer_gl1_inv
4316; GFX11-CU-NEXT:    s_endpgm
4317    i32* %out, i32 %in, i32 %old) {
4318entry:
4319  %gep = getelementptr i32, i32* %out, i32 4
4320  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acquire seq_cst
4321  ret void
4322}
4323
4324define amdgpu_kernel void @flat_system_release_seq_cst_cmpxchg(
4325; GFX7-LABEL: flat_system_release_seq_cst_cmpxchg:
4326; GFX7:       ; %bb.0: ; %entry
4327; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4328; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4329; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4330; GFX7-NEXT:    s_add_u32 s0, s0, 16
4331; GFX7-NEXT:    s_addc_u32 s1, s1, 0
4332; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4333; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4334; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4335; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4336; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4337; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4338; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4339; GFX7-NEXT:    buffer_wbinvl1_vol
4340; GFX7-NEXT:    s_endpgm
4341;
4342; GFX10-WGP-LABEL: flat_system_release_seq_cst_cmpxchg:
4343; GFX10-WGP:       ; %bb.0: ; %entry
4344; GFX10-WGP-NEXT:    s_clause 0x1
4345; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4346; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4347; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4348; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
4349; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
4350; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4351; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4352; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4353; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4354; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4355; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4356; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4357; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4358; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4359; GFX10-WGP-NEXT:    buffer_gl0_inv
4360; GFX10-WGP-NEXT:    buffer_gl1_inv
4361; GFX10-WGP-NEXT:    s_endpgm
4362;
4363; GFX10-CU-LABEL: flat_system_release_seq_cst_cmpxchg:
4364; GFX10-CU:       ; %bb.0: ; %entry
4365; GFX10-CU-NEXT:    s_clause 0x1
4366; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4367; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4368; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4369; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
4370; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
4371; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4372; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4373; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4374; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4375; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4376; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4377; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4378; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4379; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4380; GFX10-CU-NEXT:    buffer_gl0_inv
4381; GFX10-CU-NEXT:    buffer_gl1_inv
4382; GFX10-CU-NEXT:    s_endpgm
4383;
4384; SKIP-CACHE-INV-LABEL: flat_system_release_seq_cst_cmpxchg:
4385; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4386; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4387; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4388; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4389; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
4390; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
4391; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4392; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4393; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4394; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4395; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4396; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4397; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4398; SKIP-CACHE-INV-NEXT:    s_endpgm
4399;
4400; GFX90A-NOTTGSPLIT-LABEL: flat_system_release_seq_cst_cmpxchg:
4401; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4402; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4403; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4404; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4405; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4406; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4407; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
4408; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4409; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4410; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4411; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
4412; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4413; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4414;
4415; GFX90A-TGSPLIT-LABEL: flat_system_release_seq_cst_cmpxchg:
4416; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4417; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4418; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4419; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4420; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4421; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4422; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
4423; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4424; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4425; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4426; GFX90A-TGSPLIT-NEXT:    buffer_invl2
4427; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4428; GFX90A-TGSPLIT-NEXT:    s_endpgm
4429;
4430; GFX940-NOTTGSPLIT-LABEL: flat_system_release_seq_cst_cmpxchg:
4431; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4432; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4433; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4434; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4435; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4436; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4437; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
4438; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4439; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
4440; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4441; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
4442; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4443;
4444; GFX940-TGSPLIT-LABEL: flat_system_release_seq_cst_cmpxchg:
4445; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4446; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4447; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4448; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4449; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4450; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4451; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
4452; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4453; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
4454; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4455; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
4456; GFX940-TGSPLIT-NEXT:    s_endpgm
4457;
4458; GFX11-WGP-LABEL: flat_system_release_seq_cst_cmpxchg:
4459; GFX11-WGP:       ; %bb.0: ; %entry
4460; GFX11-WGP-NEXT:    s_clause 0x1
4461; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4462; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4463; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4464; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
4465; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
4466; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
4467; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
4468; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4469; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4470; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4471; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4472; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4473; GFX11-WGP-NEXT:    buffer_gl0_inv
4474; GFX11-WGP-NEXT:    buffer_gl1_inv
4475; GFX11-WGP-NEXT:    s_endpgm
4476;
4477; GFX11-CU-LABEL: flat_system_release_seq_cst_cmpxchg:
4478; GFX11-CU:       ; %bb.0: ; %entry
4479; GFX11-CU-NEXT:    s_clause 0x1
4480; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4481; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4482; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4483; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
4484; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
4485; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
4486; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
4487; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4488; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4489; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4490; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4491; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4492; GFX11-CU-NEXT:    buffer_gl0_inv
4493; GFX11-CU-NEXT:    buffer_gl1_inv
4494; GFX11-CU-NEXT:    s_endpgm
4495    i32* %out, i32 %in, i32 %old) {
4496entry:
4497  %gep = getelementptr i32, i32* %out, i32 4
4498  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in release seq_cst
4499  ret void
4500}
4501
4502define amdgpu_kernel void @flat_system_acq_rel_seq_cst_cmpxchg(
4503; GFX7-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
4504; GFX7:       ; %bb.0: ; %entry
4505; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4506; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4507; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4508; GFX7-NEXT:    s_add_u32 s0, s0, 16
4509; GFX7-NEXT:    s_addc_u32 s1, s1, 0
4510; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4511; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4512; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4513; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4514; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4515; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4516; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4517; GFX7-NEXT:    buffer_wbinvl1_vol
4518; GFX7-NEXT:    s_endpgm
4519;
4520; GFX10-WGP-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
4521; GFX10-WGP:       ; %bb.0: ; %entry
4522; GFX10-WGP-NEXT:    s_clause 0x1
4523; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4524; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4525; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4526; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
4527; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
4528; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4529; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4530; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4531; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4532; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4533; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4534; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4535; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4536; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4537; GFX10-WGP-NEXT:    buffer_gl0_inv
4538; GFX10-WGP-NEXT:    buffer_gl1_inv
4539; GFX10-WGP-NEXT:    s_endpgm
4540;
4541; GFX10-CU-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
4542; GFX10-CU:       ; %bb.0: ; %entry
4543; GFX10-CU-NEXT:    s_clause 0x1
4544; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4545; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4546; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4547; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
4548; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
4549; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4550; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4551; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4552; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4553; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4554; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4555; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4556; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4557; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4558; GFX10-CU-NEXT:    buffer_gl0_inv
4559; GFX10-CU-NEXT:    buffer_gl1_inv
4560; GFX10-CU-NEXT:    s_endpgm
4561;
4562; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
4563; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4564; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4565; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4566; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4567; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
4568; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
4569; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4570; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4571; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4572; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4573; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4574; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4575; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4576; SKIP-CACHE-INV-NEXT:    s_endpgm
4577;
4578; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
4579; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4580; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4581; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4582; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4583; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4584; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4585; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
4586; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4587; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4588; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4589; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
4590; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4591; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4592;
4593; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
4594; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4595; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4596; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4597; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4598; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4599; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4600; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
4601; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4602; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4603; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4604; GFX90A-TGSPLIT-NEXT:    buffer_invl2
4605; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4606; GFX90A-TGSPLIT-NEXT:    s_endpgm
4607;
4608; GFX940-NOTTGSPLIT-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
4609; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4610; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4611; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4612; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4613; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4614; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4615; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
4616; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4617; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
4618; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4619; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
4620; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4621;
4622; GFX940-TGSPLIT-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
4623; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4624; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4625; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4626; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4627; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4628; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4629; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
4630; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4631; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
4632; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4633; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
4634; GFX940-TGSPLIT-NEXT:    s_endpgm
4635;
4636; GFX11-WGP-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
4637; GFX11-WGP:       ; %bb.0: ; %entry
4638; GFX11-WGP-NEXT:    s_clause 0x1
4639; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4640; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4641; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4642; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
4643; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
4644; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
4645; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
4646; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4647; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4648; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4649; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4650; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4651; GFX11-WGP-NEXT:    buffer_gl0_inv
4652; GFX11-WGP-NEXT:    buffer_gl1_inv
4653; GFX11-WGP-NEXT:    s_endpgm
4654;
4655; GFX11-CU-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
4656; GFX11-CU:       ; %bb.0: ; %entry
4657; GFX11-CU-NEXT:    s_clause 0x1
4658; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4659; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4660; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4661; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
4662; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
4663; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
4664; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
4665; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4666; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4667; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4668; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4669; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4670; GFX11-CU-NEXT:    buffer_gl0_inv
4671; GFX11-CU-NEXT:    buffer_gl1_inv
4672; GFX11-CU-NEXT:    s_endpgm
4673    i32* %out, i32 %in, i32 %old) {
4674entry:
4675  %gep = getelementptr i32, i32* %out, i32 4
4676  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acq_rel seq_cst
4677  ret void
4678}
4679
4680define amdgpu_kernel void @flat_system_seq_cst_seq_cst_cmpxchg(
4681; GFX7-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
4682; GFX7:       ; %bb.0: ; %entry
4683; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4684; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4685; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4686; GFX7-NEXT:    s_add_u32 s0, s0, 16
4687; GFX7-NEXT:    s_addc_u32 s1, s1, 0
4688; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4689; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4690; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4691; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4692; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4693; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4694; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4695; GFX7-NEXT:    buffer_wbinvl1_vol
4696; GFX7-NEXT:    s_endpgm
4697;
4698; GFX10-WGP-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
4699; GFX10-WGP:       ; %bb.0: ; %entry
4700; GFX10-WGP-NEXT:    s_clause 0x1
4701; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4702; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4703; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4704; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
4705; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
4706; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4707; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4708; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4709; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4710; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4711; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4712; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4713; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4714; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4715; GFX10-WGP-NEXT:    buffer_gl0_inv
4716; GFX10-WGP-NEXT:    buffer_gl1_inv
4717; GFX10-WGP-NEXT:    s_endpgm
4718;
4719; GFX10-CU-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
4720; GFX10-CU:       ; %bb.0: ; %entry
4721; GFX10-CU-NEXT:    s_clause 0x1
4722; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4723; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4724; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4725; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
4726; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
4727; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4728; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4729; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4730; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4731; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4732; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4733; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4734; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4735; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4736; GFX10-CU-NEXT:    buffer_gl0_inv
4737; GFX10-CU-NEXT:    buffer_gl1_inv
4738; GFX10-CU-NEXT:    s_endpgm
4739;
4740; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
4741; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4742; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4743; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4744; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4745; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
4746; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
4747; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4748; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4749; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4750; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4751; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4752; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4753; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4754; SKIP-CACHE-INV-NEXT:    s_endpgm
4755;
4756; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
4757; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4758; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4759; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4760; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4761; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4762; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4763; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
4764; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4765; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4766; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4767; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
4768; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4769; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4770;
4771; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
4772; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4773; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4774; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4775; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4776; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4777; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4778; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
4779; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4780; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4781; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4782; GFX90A-TGSPLIT-NEXT:    buffer_invl2
4783; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4784; GFX90A-TGSPLIT-NEXT:    s_endpgm
4785;
4786; GFX940-NOTTGSPLIT-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
4787; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4788; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4789; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4790; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4791; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4792; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4793; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
4794; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4795; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
4796; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4797; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
4798; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4799;
4800; GFX940-TGSPLIT-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
4801; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4802; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4803; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4804; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4805; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4806; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4807; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
4808; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4809; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
4810; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4811; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
4812; GFX940-TGSPLIT-NEXT:    s_endpgm
4813;
4814; GFX11-WGP-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
4815; GFX11-WGP:       ; %bb.0: ; %entry
4816; GFX11-WGP-NEXT:    s_clause 0x1
4817; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4818; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4819; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4820; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
4821; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
4822; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
4823; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
4824; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4825; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4826; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4827; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4828; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4829; GFX11-WGP-NEXT:    buffer_gl0_inv
4830; GFX11-WGP-NEXT:    buffer_gl1_inv
4831; GFX11-WGP-NEXT:    s_endpgm
4832;
4833; GFX11-CU-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
4834; GFX11-CU:       ; %bb.0: ; %entry
4835; GFX11-CU-NEXT:    s_clause 0x1
4836; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4837; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4838; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4839; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
4840; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
4841; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
4842; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
4843; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4844; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4845; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4846; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4847; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4848; GFX11-CU-NEXT:    buffer_gl0_inv
4849; GFX11-CU-NEXT:    buffer_gl1_inv
4850; GFX11-CU-NEXT:    s_endpgm
4851    i32* %out, i32 %in, i32 %old) {
4852entry:
4853  %gep = getelementptr i32, i32* %out, i32 4
4854  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst seq_cst
4855  ret void
4856}
4857
4858define amdgpu_kernel void @flat_system_monotonic_monotonic_ret_cmpxchg(
4859; GFX7-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
4860; GFX7:       ; %bb.0: ; %entry
4861; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4862; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4863; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4864; GFX7-NEXT:    s_add_u32 s4, s0, 16
4865; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4866; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4867; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4868; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4869; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4870; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4871; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4872; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4873; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4874; GFX7-NEXT:    flat_store_dword v[0:1], v2
4875; GFX7-NEXT:    s_endpgm
4876;
4877; GFX10-WGP-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
4878; GFX10-WGP:       ; %bb.0: ; %entry
4879; GFX10-WGP-NEXT:    s_clause 0x1
4880; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4881; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4882; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4883; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4884; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4885; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4886; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4887; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4888; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4889; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4890; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4891; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4892; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4893; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4894; GFX10-WGP-NEXT:    s_endpgm
4895;
4896; GFX10-CU-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
4897; GFX10-CU:       ; %bb.0: ; %entry
4898; GFX10-CU-NEXT:    s_clause 0x1
4899; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4900; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4901; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4902; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4903; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4904; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4905; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4906; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4907; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4908; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4909; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4910; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4911; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4912; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4913; GFX10-CU-NEXT:    s_endpgm
4914;
4915; SKIP-CACHE-INV-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
4916; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4917; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4918; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4919; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4920; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4921; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4922; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4923; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4924; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4925; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4926; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4927; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4928; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4929; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4930; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4931; SKIP-CACHE-INV-NEXT:    s_endpgm
4932;
4933; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
4934; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4935; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4936; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4937; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4938; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4939; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4940; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4941; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4942; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4943; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4944;
4945; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
4946; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4947; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4948; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4949; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4950; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4951; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4952; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4953; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4954; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4955; GFX90A-TGSPLIT-NEXT:    s_endpgm
4956;
4957; GFX940-NOTTGSPLIT-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
4958; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4959; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4960; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4961; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4962; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4963; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4964; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
4965; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4966; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4967; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4968;
4969; GFX940-TGSPLIT-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
4970; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4971; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4972; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4973; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4974; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4975; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4976; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
4977; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4978; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4979; GFX940-TGSPLIT-NEXT:    s_endpgm
4980;
4981; GFX11-WGP-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
4982; GFX11-WGP:       ; %bb.0: ; %entry
4983; GFX11-WGP-NEXT:    s_clause 0x1
4984; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4985; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4986; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4987; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
4988; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
4989; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
4990; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
4991; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4992; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4993; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
4994; GFX11-WGP-NEXT:    s_endpgm
4995;
4996; GFX11-CU-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
4997; GFX11-CU:       ; %bb.0: ; %entry
4998; GFX11-CU-NEXT:    s_clause 0x1
4999; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5000; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5001; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5002; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
5003; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
5004; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
5005; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
5006; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5007; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5008; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5009; GFX11-CU-NEXT:    s_endpgm
5010    i32* %out, i32 %in, i32 %old) {
5011entry:
5012  %gep = getelementptr i32, i32* %out, i32 4
5013  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in monotonic monotonic
5014  %val0 = extractvalue { i32, i1 } %val, 0
5015  store i32 %val0, i32* %out, align 4
5016  ret void
5017}
5018
5019define amdgpu_kernel void @flat_system_acquire_monotonic_ret_cmpxchg(
5020; GFX7-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
5021; GFX7:       ; %bb.0: ; %entry
5022; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5023; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5024; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5025; GFX7-NEXT:    s_add_u32 s4, s0, 16
5026; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5027; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5028; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5029; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5030; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5031; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5032; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5033; GFX7-NEXT:    buffer_wbinvl1_vol
5034; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5035; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5036; GFX7-NEXT:    flat_store_dword v[0:1], v2
5037; GFX7-NEXT:    s_endpgm
5038;
5039; GFX10-WGP-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
5040; GFX10-WGP:       ; %bb.0: ; %entry
5041; GFX10-WGP-NEXT:    s_clause 0x1
5042; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5043; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5044; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5045; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5046; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5047; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5048; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5049; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5050; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5051; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5052; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5053; GFX10-WGP-NEXT:    buffer_gl0_inv
5054; GFX10-WGP-NEXT:    buffer_gl1_inv
5055; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5056; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5057; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5058; GFX10-WGP-NEXT:    s_endpgm
5059;
5060; GFX10-CU-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
5061; GFX10-CU:       ; %bb.0: ; %entry
5062; GFX10-CU-NEXT:    s_clause 0x1
5063; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5064; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5065; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5066; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5067; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5068; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5069; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5070; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5071; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5072; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5073; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5074; GFX10-CU-NEXT:    buffer_gl0_inv
5075; GFX10-CU-NEXT:    buffer_gl1_inv
5076; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5077; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5078; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5079; GFX10-CU-NEXT:    s_endpgm
5080;
5081; SKIP-CACHE-INV-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
5082; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5083; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5084; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5085; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5086; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5087; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5088; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5089; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5090; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5091; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5092; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5093; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5094; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5095; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5096; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5097; SKIP-CACHE-INV-NEXT:    s_endpgm
5098;
5099; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
5100; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5101; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5102; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5103; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5104; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5105; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5106; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5107; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5108; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
5109; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5110; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5111; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5112;
5113; GFX90A-TGSPLIT-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
5114; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5115; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5116; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5117; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5118; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5119; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5120; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5121; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5122; GFX90A-TGSPLIT-NEXT:    buffer_invl2
5123; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5124; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5125; GFX90A-TGSPLIT-NEXT:    s_endpgm
5126;
5127; GFX940-NOTTGSPLIT-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
5128; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5129; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5130; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5131; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5132; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5133; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5134; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
5135; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5136; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
5137; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5138; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5139;
5140; GFX940-TGSPLIT-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
5141; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5142; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5143; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5144; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5145; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5146; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5147; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
5148; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5149; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
5150; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5151; GFX940-TGSPLIT-NEXT:    s_endpgm
5152;
5153; GFX11-WGP-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
5154; GFX11-WGP:       ; %bb.0: ; %entry
5155; GFX11-WGP-NEXT:    s_clause 0x1
5156; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5157; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5158; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5159; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
5160; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
5161; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
5162; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
5163; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5164; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5165; GFX11-WGP-NEXT:    buffer_gl0_inv
5166; GFX11-WGP-NEXT:    buffer_gl1_inv
5167; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5168; GFX11-WGP-NEXT:    s_endpgm
5169;
5170; GFX11-CU-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
5171; GFX11-CU:       ; %bb.0: ; %entry
5172; GFX11-CU-NEXT:    s_clause 0x1
5173; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5174; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5175; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5176; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
5177; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
5178; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
5179; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
5180; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5181; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5182; GFX11-CU-NEXT:    buffer_gl0_inv
5183; GFX11-CU-NEXT:    buffer_gl1_inv
5184; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5185; GFX11-CU-NEXT:    s_endpgm
5186    i32* %out, i32 %in, i32 %old) {
5187entry:
5188  %gep = getelementptr i32, i32* %out, i32 4
5189  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acquire monotonic
5190  %val0 = extractvalue { i32, i1 } %val, 0
5191  store i32 %val0, i32* %out, align 4
5192  ret void
5193}
5194
5195define amdgpu_kernel void @flat_system_release_monotonic_ret_cmpxchg(
5196; GFX7-LABEL: flat_system_release_monotonic_ret_cmpxchg:
5197; GFX7:       ; %bb.0: ; %entry
5198; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5199; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5200; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5201; GFX7-NEXT:    s_add_u32 s4, s0, 16
5202; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5203; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5204; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5205; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5206; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5207; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5208; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5209; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5210; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5211; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5212; GFX7-NEXT:    flat_store_dword v[0:1], v2
5213; GFX7-NEXT:    s_endpgm
5214;
5215; GFX10-WGP-LABEL: flat_system_release_monotonic_ret_cmpxchg:
5216; GFX10-WGP:       ; %bb.0: ; %entry
5217; GFX10-WGP-NEXT:    s_clause 0x1
5218; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5219; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5220; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5221; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5222; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5223; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5224; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5225; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5226; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5227; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5228; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5229; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5230; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5231; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5232; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5233; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5234; GFX10-WGP-NEXT:    s_endpgm
5235;
5236; GFX10-CU-LABEL: flat_system_release_monotonic_ret_cmpxchg:
5237; GFX10-CU:       ; %bb.0: ; %entry
5238; GFX10-CU-NEXT:    s_clause 0x1
5239; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5240; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5241; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5242; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5243; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5244; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5245; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5246; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5247; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5248; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5249; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5250; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5251; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5252; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5253; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5254; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5255; GFX10-CU-NEXT:    s_endpgm
5256;
5257; SKIP-CACHE-INV-LABEL: flat_system_release_monotonic_ret_cmpxchg:
5258; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5259; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5260; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5261; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5262; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5263; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5264; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5265; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5266; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5267; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5268; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5269; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5270; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5271; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5272; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5273; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5274; SKIP-CACHE-INV-NEXT:    s_endpgm
5275;
5276; GFX90A-NOTTGSPLIT-LABEL: flat_system_release_monotonic_ret_cmpxchg:
5277; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5278; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5279; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5280; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5281; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5282; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5283; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
5284; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5285; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5286; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5287; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5288; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5289;
5290; GFX90A-TGSPLIT-LABEL: flat_system_release_monotonic_ret_cmpxchg:
5291; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5292; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5293; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5294; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5295; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5296; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5297; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
5298; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5299; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5300; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5301; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5302; GFX90A-TGSPLIT-NEXT:    s_endpgm
5303;
5304; GFX940-NOTTGSPLIT-LABEL: flat_system_release_monotonic_ret_cmpxchg:
5305; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5306; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5307; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5308; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5309; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5310; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5311; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
5312; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5313; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
5314; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5315; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5316; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5317;
5318; GFX940-TGSPLIT-LABEL: flat_system_release_monotonic_ret_cmpxchg:
5319; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5320; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5321; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5322; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5323; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5324; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5325; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
5326; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5327; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
5328; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5329; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5330; GFX940-TGSPLIT-NEXT:    s_endpgm
5331;
5332; GFX11-WGP-LABEL: flat_system_release_monotonic_ret_cmpxchg:
5333; GFX11-WGP:       ; %bb.0: ; %entry
5334; GFX11-WGP-NEXT:    s_clause 0x1
5335; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5336; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5337; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5338; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
5339; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
5340; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
5341; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
5342; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5343; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5344; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5345; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5346; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5347; GFX11-WGP-NEXT:    s_endpgm
5348;
5349; GFX11-CU-LABEL: flat_system_release_monotonic_ret_cmpxchg:
5350; GFX11-CU:       ; %bb.0: ; %entry
5351; GFX11-CU-NEXT:    s_clause 0x1
5352; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5353; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5354; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5355; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
5356; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
5357; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
5358; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
5359; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5360; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5361; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5362; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5363; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5364; GFX11-CU-NEXT:    s_endpgm
5365    i32* %out, i32 %in, i32 %old) {
5366entry:
5367  %gep = getelementptr i32, i32* %out, i32 4
5368  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in release monotonic
5369  %val0 = extractvalue { i32, i1 } %val, 0
5370  store i32 %val0, i32* %out, align 4
5371  ret void
5372}
5373
5374define amdgpu_kernel void @flat_system_acq_rel_monotonic_ret_cmpxchg(
5375; GFX7-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
5376; GFX7:       ; %bb.0: ; %entry
5377; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5378; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5379; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5380; GFX7-NEXT:    s_add_u32 s4, s0, 16
5381; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5382; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5383; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5384; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5385; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5386; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5387; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5388; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5389; GFX7-NEXT:    buffer_wbinvl1_vol
5390; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5391; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5392; GFX7-NEXT:    flat_store_dword v[0:1], v2
5393; GFX7-NEXT:    s_endpgm
5394;
5395; GFX10-WGP-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
5396; GFX10-WGP:       ; %bb.0: ; %entry
5397; GFX10-WGP-NEXT:    s_clause 0x1
5398; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5399; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5400; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5401; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5402; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5403; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5404; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5405; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5406; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5407; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5408; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5409; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5410; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5411; GFX10-WGP-NEXT:    buffer_gl0_inv
5412; GFX10-WGP-NEXT:    buffer_gl1_inv
5413; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5414; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5415; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5416; GFX10-WGP-NEXT:    s_endpgm
5417;
5418; GFX10-CU-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
5419; GFX10-CU:       ; %bb.0: ; %entry
5420; GFX10-CU-NEXT:    s_clause 0x1
5421; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5422; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5423; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5424; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5425; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5426; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5427; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5428; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5429; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5430; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5431; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5432; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5433; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5434; GFX10-CU-NEXT:    buffer_gl0_inv
5435; GFX10-CU-NEXT:    buffer_gl1_inv
5436; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5437; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5438; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5439; GFX10-CU-NEXT:    s_endpgm
5440;
5441; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
5442; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5443; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5444; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5445; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5446; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5447; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5448; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5449; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5450; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5451; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5452; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5453; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5454; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5455; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5456; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5457; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5458; SKIP-CACHE-INV-NEXT:    s_endpgm
5459;
5460; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
5461; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5462; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5463; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5464; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5465; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5466; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5467; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
5468; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5469; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5470; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5471; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
5472; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5473; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5474; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5475;
5476; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
5477; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5478; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5479; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5480; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5481; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5482; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5483; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
5484; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5485; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5486; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5487; GFX90A-TGSPLIT-NEXT:    buffer_invl2
5488; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5489; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5490; GFX90A-TGSPLIT-NEXT:    s_endpgm
5491;
5492; GFX940-NOTTGSPLIT-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
5493; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5494; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5495; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5496; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5497; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5498; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5499; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
5500; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5501; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
5502; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5503; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
5504; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5505; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5506;
5507; GFX940-TGSPLIT-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
5508; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5509; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5510; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5511; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5512; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5513; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5514; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
5515; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5516; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
5517; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5518; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
5519; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5520; GFX940-TGSPLIT-NEXT:    s_endpgm
5521;
5522; GFX11-WGP-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
5523; GFX11-WGP:       ; %bb.0: ; %entry
5524; GFX11-WGP-NEXT:    s_clause 0x1
5525; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5526; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5527; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5528; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
5529; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
5530; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
5531; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
5532; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5533; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5534; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5535; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5536; GFX11-WGP-NEXT:    buffer_gl0_inv
5537; GFX11-WGP-NEXT:    buffer_gl1_inv
5538; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5539; GFX11-WGP-NEXT:    s_endpgm
5540;
5541; GFX11-CU-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
5542; GFX11-CU:       ; %bb.0: ; %entry
5543; GFX11-CU-NEXT:    s_clause 0x1
5544; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5545; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5546; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5547; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
5548; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
5549; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
5550; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
5551; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5552; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5553; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5554; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5555; GFX11-CU-NEXT:    buffer_gl0_inv
5556; GFX11-CU-NEXT:    buffer_gl1_inv
5557; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5558; GFX11-CU-NEXT:    s_endpgm
5559    i32* %out, i32 %in, i32 %old) {
5560entry:
5561  %gep = getelementptr i32, i32* %out, i32 4
5562  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acq_rel monotonic
5563  %val0 = extractvalue { i32, i1 } %val, 0
5564  store i32 %val0, i32* %out, align 4
5565  ret void
5566}
5567
5568define amdgpu_kernel void @flat_system_seq_cst_monotonic_ret_cmpxchg(
5569; GFX7-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
5570; GFX7:       ; %bb.0: ; %entry
5571; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5572; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5573; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5574; GFX7-NEXT:    s_add_u32 s4, s0, 16
5575; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5576; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5577; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5578; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5579; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5580; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5581; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5582; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5583; GFX7-NEXT:    buffer_wbinvl1_vol
5584; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5585; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5586; GFX7-NEXT:    flat_store_dword v[0:1], v2
5587; GFX7-NEXT:    s_endpgm
5588;
5589; GFX10-WGP-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
5590; GFX10-WGP:       ; %bb.0: ; %entry
5591; GFX10-WGP-NEXT:    s_clause 0x1
5592; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5593; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5594; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5595; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5596; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5597; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5598; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5599; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5600; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5601; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5602; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5603; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5604; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5605; GFX10-WGP-NEXT:    buffer_gl0_inv
5606; GFX10-WGP-NEXT:    buffer_gl1_inv
5607; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5608; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5609; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5610; GFX10-WGP-NEXT:    s_endpgm
5611;
5612; GFX10-CU-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
5613; GFX10-CU:       ; %bb.0: ; %entry
5614; GFX10-CU-NEXT:    s_clause 0x1
5615; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5616; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5617; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5618; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5619; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5620; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5621; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5622; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5623; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5624; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5625; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5626; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5627; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5628; GFX10-CU-NEXT:    buffer_gl0_inv
5629; GFX10-CU-NEXT:    buffer_gl1_inv
5630; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5631; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5632; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5633; GFX10-CU-NEXT:    s_endpgm
5634;
5635; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
5636; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5637; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5638; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5639; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5640; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5641; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5642; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5643; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5644; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5645; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5646; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5647; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5648; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5649; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5650; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5651; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5652; SKIP-CACHE-INV-NEXT:    s_endpgm
5653;
5654; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
5655; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5656; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5657; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5658; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5659; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5660; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5661; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
5662; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5663; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5664; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5665; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
5666; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5667; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5668; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5669;
5670; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
5671; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5672; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5673; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5674; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5675; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5676; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5677; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
5678; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5679; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5680; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5681; GFX90A-TGSPLIT-NEXT:    buffer_invl2
5682; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5683; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5684; GFX90A-TGSPLIT-NEXT:    s_endpgm
5685;
5686; GFX940-NOTTGSPLIT-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
5687; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5688; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5689; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5690; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5691; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5692; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5693; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
5694; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5695; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
5696; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5697; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
5698; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5699; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5700;
5701; GFX940-TGSPLIT-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
5702; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5703; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5704; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5705; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5706; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5707; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5708; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
5709; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5710; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
5711; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5712; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
5713; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5714; GFX940-TGSPLIT-NEXT:    s_endpgm
5715;
5716; GFX11-WGP-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
5717; GFX11-WGP:       ; %bb.0: ; %entry
5718; GFX11-WGP-NEXT:    s_clause 0x1
5719; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5720; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5721; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5722; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
5723; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
5724; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
5725; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
5726; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5727; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5728; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5729; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5730; GFX11-WGP-NEXT:    buffer_gl0_inv
5731; GFX11-WGP-NEXT:    buffer_gl1_inv
5732; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5733; GFX11-WGP-NEXT:    s_endpgm
5734;
5735; GFX11-CU-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
5736; GFX11-CU:       ; %bb.0: ; %entry
5737; GFX11-CU-NEXT:    s_clause 0x1
5738; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5739; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5740; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5741; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
5742; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
5743; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
5744; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
5745; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5746; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5747; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5748; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5749; GFX11-CU-NEXT:    buffer_gl0_inv
5750; GFX11-CU-NEXT:    buffer_gl1_inv
5751; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5752; GFX11-CU-NEXT:    s_endpgm
5753    i32* %out, i32 %in, i32 %old) {
5754entry:
5755  %gep = getelementptr i32, i32* %out, i32 4
5756  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst monotonic
5757  %val0 = extractvalue { i32, i1 } %val, 0
5758  store i32 %val0, i32* %out, align 4
5759  ret void
5760}
5761
5762define amdgpu_kernel void @flat_system_monotonic_acquire_ret_cmpxchg(
5763; GFX7-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
5764; GFX7:       ; %bb.0: ; %entry
5765; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5766; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5767; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5768; GFX7-NEXT:    s_add_u32 s4, s0, 16
5769; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5770; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5771; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5772; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5773; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5774; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5775; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5776; GFX7-NEXT:    buffer_wbinvl1_vol
5777; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5778; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5779; GFX7-NEXT:    flat_store_dword v[0:1], v2
5780; GFX7-NEXT:    s_endpgm
5781;
5782; GFX10-WGP-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
5783; GFX10-WGP:       ; %bb.0: ; %entry
5784; GFX10-WGP-NEXT:    s_clause 0x1
5785; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5786; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5787; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5788; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5789; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5790; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5791; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5792; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5793; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5794; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5795; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5796; GFX10-WGP-NEXT:    buffer_gl0_inv
5797; GFX10-WGP-NEXT:    buffer_gl1_inv
5798; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5799; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5800; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5801; GFX10-WGP-NEXT:    s_endpgm
5802;
5803; GFX10-CU-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
5804; GFX10-CU:       ; %bb.0: ; %entry
5805; GFX10-CU-NEXT:    s_clause 0x1
5806; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5807; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5808; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5809; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5810; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5811; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5812; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5813; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5814; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5815; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5816; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5817; GFX10-CU-NEXT:    buffer_gl0_inv
5818; GFX10-CU-NEXT:    buffer_gl1_inv
5819; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5820; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5821; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5822; GFX10-CU-NEXT:    s_endpgm
5823;
5824; SKIP-CACHE-INV-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
5825; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5826; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5827; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5828; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5829; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5830; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5831; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5832; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5833; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5834; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5835; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5836; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5837; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5838; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5839; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5840; SKIP-CACHE-INV-NEXT:    s_endpgm
5841;
5842; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
5843; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5844; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5845; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5846; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5847; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5848; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5849; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5850; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5851; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
5852; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5853; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5854; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5855;
5856; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
5857; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5858; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5859; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5860; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5861; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5862; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5863; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5864; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5865; GFX90A-TGSPLIT-NEXT:    buffer_invl2
5866; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5867; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5868; GFX90A-TGSPLIT-NEXT:    s_endpgm
5869;
5870; GFX940-NOTTGSPLIT-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
5871; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5872; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5873; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5874; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5875; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5876; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5877; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
5878; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5879; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
5880; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5881; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5882;
5883; GFX940-TGSPLIT-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
5884; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5885; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5886; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5887; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5888; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5889; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5890; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
5891; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5892; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
5893; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5894; GFX940-TGSPLIT-NEXT:    s_endpgm
5895;
5896; GFX11-WGP-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
5897; GFX11-WGP:       ; %bb.0: ; %entry
5898; GFX11-WGP-NEXT:    s_clause 0x1
5899; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5900; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5901; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5902; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
5903; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
5904; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
5905; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
5906; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5907; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5908; GFX11-WGP-NEXT:    buffer_gl0_inv
5909; GFX11-WGP-NEXT:    buffer_gl1_inv
5910; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5911; GFX11-WGP-NEXT:    s_endpgm
5912;
5913; GFX11-CU-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
5914; GFX11-CU:       ; %bb.0: ; %entry
5915; GFX11-CU-NEXT:    s_clause 0x1
5916; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5917; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5918; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5919; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
5920; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
5921; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
5922; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
5923; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5924; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5925; GFX11-CU-NEXT:    buffer_gl0_inv
5926; GFX11-CU-NEXT:    buffer_gl1_inv
5927; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5928; GFX11-CU-NEXT:    s_endpgm
5929    i32* %out, i32 %in, i32 %old) {
5930entry:
5931  %gep = getelementptr i32, i32* %out, i32 4
5932  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in monotonic acquire
5933  %val0 = extractvalue { i32, i1 } %val, 0
5934  store i32 %val0, i32* %out, align 4
5935  ret void
5936}
5937
5938define amdgpu_kernel void @flat_system_acquire_acquire_ret_cmpxchg(
5939; GFX7-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
5940; GFX7:       ; %bb.0: ; %entry
5941; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5942; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5943; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5944; GFX7-NEXT:    s_add_u32 s4, s0, 16
5945; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5946; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5947; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5948; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5949; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5950; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5951; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5952; GFX7-NEXT:    buffer_wbinvl1_vol
5953; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5954; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5955; GFX7-NEXT:    flat_store_dword v[0:1], v2
5956; GFX7-NEXT:    s_endpgm
5957;
5958; GFX10-WGP-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
5959; GFX10-WGP:       ; %bb.0: ; %entry
5960; GFX10-WGP-NEXT:    s_clause 0x1
5961; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5962; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5963; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5964; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5965; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5966; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5967; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5968; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5969; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5970; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5971; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5972; GFX10-WGP-NEXT:    buffer_gl0_inv
5973; GFX10-WGP-NEXT:    buffer_gl1_inv
5974; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5975; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5976; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5977; GFX10-WGP-NEXT:    s_endpgm
5978;
5979; GFX10-CU-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
5980; GFX10-CU:       ; %bb.0: ; %entry
5981; GFX10-CU-NEXT:    s_clause 0x1
5982; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5983; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5984; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5985; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5986; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5987; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5988; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5989; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5990; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5991; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5992; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5993; GFX10-CU-NEXT:    buffer_gl0_inv
5994; GFX10-CU-NEXT:    buffer_gl1_inv
5995; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5996; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5997; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5998; GFX10-CU-NEXT:    s_endpgm
5999;
6000; SKIP-CACHE-INV-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
6001; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6002; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6003; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6004; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6005; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6006; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6007; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6008; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6009; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6010; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6011; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6012; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6013; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6014; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6015; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6016; SKIP-CACHE-INV-NEXT:    s_endpgm
6017;
6018; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
6019; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6020; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6021; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6022; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6023; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6024; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6025; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6026; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6027; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
6028; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6029; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6030; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6031;
6032; GFX90A-TGSPLIT-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
6033; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6034; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6035; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6036; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6037; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6038; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6039; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6040; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6041; GFX90A-TGSPLIT-NEXT:    buffer_invl2
6042; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6043; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6044; GFX90A-TGSPLIT-NEXT:    s_endpgm
6045;
6046; GFX940-NOTTGSPLIT-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
6047; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6048; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6049; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6050; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6051; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6052; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6053; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
6054; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6055; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
6056; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6057; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6058;
6059; GFX940-TGSPLIT-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
6060; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6061; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6062; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6063; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6064; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6065; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6066; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
6067; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6068; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
6069; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6070; GFX940-TGSPLIT-NEXT:    s_endpgm
6071;
6072; GFX11-WGP-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
6073; GFX11-WGP:       ; %bb.0: ; %entry
6074; GFX11-WGP-NEXT:    s_clause 0x1
6075; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6076; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6077; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6078; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
6079; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
6080; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
6081; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
6082; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6083; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6084; GFX11-WGP-NEXT:    buffer_gl0_inv
6085; GFX11-WGP-NEXT:    buffer_gl1_inv
6086; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6087; GFX11-WGP-NEXT:    s_endpgm
6088;
6089; GFX11-CU-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
6090; GFX11-CU:       ; %bb.0: ; %entry
6091; GFX11-CU-NEXT:    s_clause 0x1
6092; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6093; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6094; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6095; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
6096; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
6097; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
6098; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
6099; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6100; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6101; GFX11-CU-NEXT:    buffer_gl0_inv
6102; GFX11-CU-NEXT:    buffer_gl1_inv
6103; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6104; GFX11-CU-NEXT:    s_endpgm
6105    i32* %out, i32 %in, i32 %old) {
6106entry:
6107  %gep = getelementptr i32, i32* %out, i32 4
6108  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acquire acquire
6109  %val0 = extractvalue { i32, i1 } %val, 0
6110  store i32 %val0, i32* %out, align 4
6111  ret void
6112}
6113
6114define amdgpu_kernel void @flat_system_release_acquire_ret_cmpxchg(
6115; GFX7-LABEL: flat_system_release_acquire_ret_cmpxchg:
6116; GFX7:       ; %bb.0: ; %entry
6117; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6118; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6119; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6120; GFX7-NEXT:    s_add_u32 s4, s0, 16
6121; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6122; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6123; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6124; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6125; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6126; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6127; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6128; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6129; GFX7-NEXT:    buffer_wbinvl1_vol
6130; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6131; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6132; GFX7-NEXT:    flat_store_dword v[0:1], v2
6133; GFX7-NEXT:    s_endpgm
6134;
6135; GFX10-WGP-LABEL: flat_system_release_acquire_ret_cmpxchg:
6136; GFX10-WGP:       ; %bb.0: ; %entry
6137; GFX10-WGP-NEXT:    s_clause 0x1
6138; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6139; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6140; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6141; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6142; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6143; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6144; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6145; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6146; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6147; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6148; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6149; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6150; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6151; GFX10-WGP-NEXT:    buffer_gl0_inv
6152; GFX10-WGP-NEXT:    buffer_gl1_inv
6153; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6154; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6155; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6156; GFX10-WGP-NEXT:    s_endpgm
6157;
6158; GFX10-CU-LABEL: flat_system_release_acquire_ret_cmpxchg:
6159; GFX10-CU:       ; %bb.0: ; %entry
6160; GFX10-CU-NEXT:    s_clause 0x1
6161; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6162; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6163; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6164; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6165; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6166; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6167; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6168; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6169; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6170; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6171; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6172; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6173; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6174; GFX10-CU-NEXT:    buffer_gl0_inv
6175; GFX10-CU-NEXT:    buffer_gl1_inv
6176; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6177; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6178; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6179; GFX10-CU-NEXT:    s_endpgm
6180;
6181; SKIP-CACHE-INV-LABEL: flat_system_release_acquire_ret_cmpxchg:
6182; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6183; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6184; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6185; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6186; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6187; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6188; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6189; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6190; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6191; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6192; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6193; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6194; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6195; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6196; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6197; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6198; SKIP-CACHE-INV-NEXT:    s_endpgm
6199;
6200; GFX90A-NOTTGSPLIT-LABEL: flat_system_release_acquire_ret_cmpxchg:
6201; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6202; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6203; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6204; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6205; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6206; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6207; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
6208; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6209; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6210; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6211; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
6212; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6213; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6214; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6215;
6216; GFX90A-TGSPLIT-LABEL: flat_system_release_acquire_ret_cmpxchg:
6217; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6218; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6219; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6220; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6221; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6222; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6223; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
6224; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6225; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6226; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6227; GFX90A-TGSPLIT-NEXT:    buffer_invl2
6228; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6229; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6230; GFX90A-TGSPLIT-NEXT:    s_endpgm
6231;
6232; GFX940-NOTTGSPLIT-LABEL: flat_system_release_acquire_ret_cmpxchg:
6233; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6234; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6235; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6236; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6237; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6238; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6239; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
6240; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6241; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
6242; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6243; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
6244; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6245; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6246;
6247; GFX940-TGSPLIT-LABEL: flat_system_release_acquire_ret_cmpxchg:
6248; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6249; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6250; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6251; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6252; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6253; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6254; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
6255; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6256; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
6257; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6258; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
6259; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6260; GFX940-TGSPLIT-NEXT:    s_endpgm
6261;
6262; GFX11-WGP-LABEL: flat_system_release_acquire_ret_cmpxchg:
6263; GFX11-WGP:       ; %bb.0: ; %entry
6264; GFX11-WGP-NEXT:    s_clause 0x1
6265; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6266; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6267; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6268; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
6269; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
6270; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
6271; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
6272; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6273; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6274; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6275; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6276; GFX11-WGP-NEXT:    buffer_gl0_inv
6277; GFX11-WGP-NEXT:    buffer_gl1_inv
6278; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6279; GFX11-WGP-NEXT:    s_endpgm
6280;
6281; GFX11-CU-LABEL: flat_system_release_acquire_ret_cmpxchg:
6282; GFX11-CU:       ; %bb.0: ; %entry
6283; GFX11-CU-NEXT:    s_clause 0x1
6284; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6285; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6286; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6287; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
6288; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
6289; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
6290; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
6291; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6292; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6293; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6294; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6295; GFX11-CU-NEXT:    buffer_gl0_inv
6296; GFX11-CU-NEXT:    buffer_gl1_inv
6297; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6298; GFX11-CU-NEXT:    s_endpgm
6299    i32* %out, i32 %in, i32 %old) {
6300entry:
6301  %gep = getelementptr i32, i32* %out, i32 4
6302  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in release acquire
6303  %val0 = extractvalue { i32, i1 } %val, 0
6304  store i32 %val0, i32* %out, align 4
6305  ret void
6306}
6307
6308define amdgpu_kernel void @flat_system_acq_rel_acquire_ret_cmpxchg(
6309; GFX7-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
6310; GFX7:       ; %bb.0: ; %entry
6311; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6312; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6313; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6314; GFX7-NEXT:    s_add_u32 s4, s0, 16
6315; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6316; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6317; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6318; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6319; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6320; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6321; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6322; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6323; GFX7-NEXT:    buffer_wbinvl1_vol
6324; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6325; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6326; GFX7-NEXT:    flat_store_dword v[0:1], v2
6327; GFX7-NEXT:    s_endpgm
6328;
6329; GFX10-WGP-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
6330; GFX10-WGP:       ; %bb.0: ; %entry
6331; GFX10-WGP-NEXT:    s_clause 0x1
6332; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6333; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6334; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6335; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6336; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6337; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6338; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6339; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6340; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6341; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6342; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6343; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6344; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6345; GFX10-WGP-NEXT:    buffer_gl0_inv
6346; GFX10-WGP-NEXT:    buffer_gl1_inv
6347; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6348; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6349; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6350; GFX10-WGP-NEXT:    s_endpgm
6351;
6352; GFX10-CU-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
6353; GFX10-CU:       ; %bb.0: ; %entry
6354; GFX10-CU-NEXT:    s_clause 0x1
6355; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6356; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6357; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6358; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6359; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6360; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6361; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6362; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6363; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6364; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6365; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6366; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6367; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6368; GFX10-CU-NEXT:    buffer_gl0_inv
6369; GFX10-CU-NEXT:    buffer_gl1_inv
6370; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6371; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6372; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6373; GFX10-CU-NEXT:    s_endpgm
6374;
6375; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
6376; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6377; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6378; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6379; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6380; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6381; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6382; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6383; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6384; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6385; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6386; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6387; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6388; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6389; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6390; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6391; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6392; SKIP-CACHE-INV-NEXT:    s_endpgm
6393;
6394; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
6395; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6396; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6397; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6398; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6399; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6400; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6401; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
6402; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6403; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6404; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6405; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
6406; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6407; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6408; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6409;
6410; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
6411; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6412; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6413; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6414; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6415; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6416; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6417; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
6418; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6419; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6420; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6421; GFX90A-TGSPLIT-NEXT:    buffer_invl2
6422; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6423; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6424; GFX90A-TGSPLIT-NEXT:    s_endpgm
6425;
6426; GFX940-NOTTGSPLIT-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
6427; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6428; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6429; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6430; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6431; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6432; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6433; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
6434; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6435; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
6436; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6437; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
6438; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6439; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6440;
6441; GFX940-TGSPLIT-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
6442; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6443; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6444; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6445; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6446; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6447; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6448; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
6449; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6450; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
6451; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6452; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
6453; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6454; GFX940-TGSPLIT-NEXT:    s_endpgm
6455;
6456; GFX11-WGP-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
6457; GFX11-WGP:       ; %bb.0: ; %entry
6458; GFX11-WGP-NEXT:    s_clause 0x1
6459; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6460; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6461; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6462; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
6463; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
6464; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
6465; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
6466; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6467; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6468; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6469; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6470; GFX11-WGP-NEXT:    buffer_gl0_inv
6471; GFX11-WGP-NEXT:    buffer_gl1_inv
6472; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6473; GFX11-WGP-NEXT:    s_endpgm
6474;
6475; GFX11-CU-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
6476; GFX11-CU:       ; %bb.0: ; %entry
6477; GFX11-CU-NEXT:    s_clause 0x1
6478; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6479; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6480; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6481; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
6482; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
6483; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
6484; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
6485; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6486; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6487; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6488; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6489; GFX11-CU-NEXT:    buffer_gl0_inv
6490; GFX11-CU-NEXT:    buffer_gl1_inv
6491; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6492; GFX11-CU-NEXT:    s_endpgm
6493    i32* %out, i32 %in, i32 %old) {
6494entry:
6495  %gep = getelementptr i32, i32* %out, i32 4
6496  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acq_rel acquire
6497  %val0 = extractvalue { i32, i1 } %val, 0
6498  store i32 %val0, i32* %out, align 4
6499  ret void
6500}
6501
6502define amdgpu_kernel void @flat_system_seq_cst_acquire_ret_cmpxchg(
6503; GFX7-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
6504; GFX7:       ; %bb.0: ; %entry
6505; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6506; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6507; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6508; GFX7-NEXT:    s_add_u32 s4, s0, 16
6509; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6510; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6511; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6512; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6513; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6514; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6515; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6516; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6517; GFX7-NEXT:    buffer_wbinvl1_vol
6518; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6519; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6520; GFX7-NEXT:    flat_store_dword v[0:1], v2
6521; GFX7-NEXT:    s_endpgm
6522;
6523; GFX10-WGP-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
6524; GFX10-WGP:       ; %bb.0: ; %entry
6525; GFX10-WGP-NEXT:    s_clause 0x1
6526; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6527; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6528; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6529; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6530; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6531; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6532; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6533; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6534; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6535; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6536; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6537; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6538; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6539; GFX10-WGP-NEXT:    buffer_gl0_inv
6540; GFX10-WGP-NEXT:    buffer_gl1_inv
6541; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6542; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6543; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6544; GFX10-WGP-NEXT:    s_endpgm
6545;
6546; GFX10-CU-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
6547; GFX10-CU:       ; %bb.0: ; %entry
6548; GFX10-CU-NEXT:    s_clause 0x1
6549; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6550; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6551; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6552; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6553; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6554; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6555; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6556; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6557; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6558; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6559; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6560; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6561; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6562; GFX10-CU-NEXT:    buffer_gl0_inv
6563; GFX10-CU-NEXT:    buffer_gl1_inv
6564; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6565; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6566; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6567; GFX10-CU-NEXT:    s_endpgm
6568;
6569; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
6570; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6571; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6572; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6573; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6574; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6575; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6576; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6577; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6578; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6579; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6580; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6581; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6582; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6583; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6584; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6585; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6586; SKIP-CACHE-INV-NEXT:    s_endpgm
6587;
6588; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
6589; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6590; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6591; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6592; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6593; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6594; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6595; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
6596; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6597; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6598; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6599; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
6600; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6601; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6602; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6603;
6604; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
6605; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6606; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6607; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6608; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6609; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6610; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6611; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
6612; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6613; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6614; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6615; GFX90A-TGSPLIT-NEXT:    buffer_invl2
6616; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6617; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6618; GFX90A-TGSPLIT-NEXT:    s_endpgm
6619;
6620; GFX940-NOTTGSPLIT-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
6621; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6622; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6623; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6624; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6625; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6626; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6627; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
6628; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6629; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
6630; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6631; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
6632; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6633; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6634;
6635; GFX940-TGSPLIT-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
6636; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6637; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6638; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6639; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6640; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6641; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6642; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
6643; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6644; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
6645; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6646; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
6647; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6648; GFX940-TGSPLIT-NEXT:    s_endpgm
6649;
6650; GFX11-WGP-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
6651; GFX11-WGP:       ; %bb.0: ; %entry
6652; GFX11-WGP-NEXT:    s_clause 0x1
6653; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6654; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6655; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6656; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
6657; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
6658; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
6659; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
6660; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6661; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6662; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6663; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6664; GFX11-WGP-NEXT:    buffer_gl0_inv
6665; GFX11-WGP-NEXT:    buffer_gl1_inv
6666; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6667; GFX11-WGP-NEXT:    s_endpgm
6668;
6669; GFX11-CU-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
6670; GFX11-CU:       ; %bb.0: ; %entry
6671; GFX11-CU-NEXT:    s_clause 0x1
6672; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6673; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6674; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6675; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
6676; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
6677; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
6678; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
6679; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6680; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6681; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6682; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6683; GFX11-CU-NEXT:    buffer_gl0_inv
6684; GFX11-CU-NEXT:    buffer_gl1_inv
6685; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6686; GFX11-CU-NEXT:    s_endpgm
6687    i32* %out, i32 %in, i32 %old) {
6688entry:
6689  %gep = getelementptr i32, i32* %out, i32 4
6690  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst acquire
6691  %val0 = extractvalue { i32, i1 } %val, 0
6692  store i32 %val0, i32* %out, align 4
6693  ret void
6694}
6695
6696define amdgpu_kernel void @flat_system_monotonic_seq_cst_ret_cmpxchg(
6697; GFX7-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
6698; GFX7:       ; %bb.0: ; %entry
6699; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6700; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6701; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6702; GFX7-NEXT:    s_add_u32 s4, s0, 16
6703; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6704; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6705; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6706; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6707; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6708; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6709; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6710; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6711; GFX7-NEXT:    buffer_wbinvl1_vol
6712; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6713; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6714; GFX7-NEXT:    flat_store_dword v[0:1], v2
6715; GFX7-NEXT:    s_endpgm
6716;
6717; GFX10-WGP-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
6718; GFX10-WGP:       ; %bb.0: ; %entry
6719; GFX10-WGP-NEXT:    s_clause 0x1
6720; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6721; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6722; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6723; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6724; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6725; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6726; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6727; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6728; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6729; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6730; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6731; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6732; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6733; GFX10-WGP-NEXT:    buffer_gl0_inv
6734; GFX10-WGP-NEXT:    buffer_gl1_inv
6735; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6736; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6737; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6738; GFX10-WGP-NEXT:    s_endpgm
6739;
6740; GFX10-CU-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
6741; GFX10-CU:       ; %bb.0: ; %entry
6742; GFX10-CU-NEXT:    s_clause 0x1
6743; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6744; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6745; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6746; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6747; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6748; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6749; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6750; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6751; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6752; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6753; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6754; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6755; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6756; GFX10-CU-NEXT:    buffer_gl0_inv
6757; GFX10-CU-NEXT:    buffer_gl1_inv
6758; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6759; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6760; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6761; GFX10-CU-NEXT:    s_endpgm
6762;
6763; SKIP-CACHE-INV-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
6764; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6765; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6766; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6767; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6768; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6769; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6770; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6771; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6772; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6773; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6774; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6775; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6776; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6777; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6778; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6779; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6780; SKIP-CACHE-INV-NEXT:    s_endpgm
6781;
6782; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
6783; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6784; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6785; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6786; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6787; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6788; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6789; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
6790; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6791; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6792; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6793; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
6794; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6795; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6796; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6797;
6798; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
6799; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6800; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6801; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6802; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6803; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6804; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6805; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
6806; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6807; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6808; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6809; GFX90A-TGSPLIT-NEXT:    buffer_invl2
6810; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6811; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6812; GFX90A-TGSPLIT-NEXT:    s_endpgm
6813;
6814; GFX940-NOTTGSPLIT-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
6815; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6816; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6817; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6818; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6819; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6820; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6821; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
6822; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6823; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
6824; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6825; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
6826; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6827; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6828;
6829; GFX940-TGSPLIT-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
6830; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6831; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6832; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6833; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6834; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6835; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6836; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
6837; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6838; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
6839; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6840; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
6841; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6842; GFX940-TGSPLIT-NEXT:    s_endpgm
6843;
6844; GFX11-WGP-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
6845; GFX11-WGP:       ; %bb.0: ; %entry
6846; GFX11-WGP-NEXT:    s_clause 0x1
6847; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6848; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6849; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6850; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
6851; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
6852; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
6853; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
6854; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6855; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6856; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6857; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6858; GFX11-WGP-NEXT:    buffer_gl0_inv
6859; GFX11-WGP-NEXT:    buffer_gl1_inv
6860; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6861; GFX11-WGP-NEXT:    s_endpgm
6862;
6863; GFX11-CU-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
6864; GFX11-CU:       ; %bb.0: ; %entry
6865; GFX11-CU-NEXT:    s_clause 0x1
6866; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6867; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6868; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6869; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
6870; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
6871; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
6872; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
6873; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6874; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6875; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6876; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6877; GFX11-CU-NEXT:    buffer_gl0_inv
6878; GFX11-CU-NEXT:    buffer_gl1_inv
6879; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6880; GFX11-CU-NEXT:    s_endpgm
6881    i32* %out, i32 %in, i32 %old) {
6882entry:
6883  %gep = getelementptr i32, i32* %out, i32 4
6884  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in monotonic seq_cst
6885  %val0 = extractvalue { i32, i1 } %val, 0
6886  store i32 %val0, i32* %out, align 4
6887  ret void
6888}
6889
6890define amdgpu_kernel void @flat_system_acquire_seq_cst_ret_cmpxchg(
6891; GFX7-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
6892; GFX7:       ; %bb.0: ; %entry
6893; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6894; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6895; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6896; GFX7-NEXT:    s_add_u32 s4, s0, 16
6897; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6898; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6899; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6900; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6901; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6902; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6903; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6904; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6905; GFX7-NEXT:    buffer_wbinvl1_vol
6906; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6907; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6908; GFX7-NEXT:    flat_store_dword v[0:1], v2
6909; GFX7-NEXT:    s_endpgm
6910;
6911; GFX10-WGP-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
6912; GFX10-WGP:       ; %bb.0: ; %entry
6913; GFX10-WGP-NEXT:    s_clause 0x1
6914; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6915; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6916; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6917; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6918; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6919; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6920; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6921; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6922; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6923; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6924; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6925; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6926; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6927; GFX10-WGP-NEXT:    buffer_gl0_inv
6928; GFX10-WGP-NEXT:    buffer_gl1_inv
6929; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6930; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6931; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6932; GFX10-WGP-NEXT:    s_endpgm
6933;
6934; GFX10-CU-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
6935; GFX10-CU:       ; %bb.0: ; %entry
6936; GFX10-CU-NEXT:    s_clause 0x1
6937; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6938; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6939; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6940; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6941; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6942; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6943; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6944; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6945; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6946; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6947; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6948; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6949; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6950; GFX10-CU-NEXT:    buffer_gl0_inv
6951; GFX10-CU-NEXT:    buffer_gl1_inv
6952; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6953; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6954; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6955; GFX10-CU-NEXT:    s_endpgm
6956;
6957; SKIP-CACHE-INV-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
6958; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6959; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6960; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6961; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6962; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6963; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6964; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6965; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6966; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6967; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6968; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6969; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6970; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6971; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6972; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6973; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6974; SKIP-CACHE-INV-NEXT:    s_endpgm
6975;
6976; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
6977; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6978; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6979; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6980; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6981; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6982; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6983; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
6984; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6985; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6986; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6987; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
6988; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6989; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6990; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6991;
6992; GFX90A-TGSPLIT-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
6993; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6994; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6995; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6996; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6997; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6998; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6999; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
7000; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7001; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7002; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7003; GFX90A-TGSPLIT-NEXT:    buffer_invl2
7004; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7005; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7006; GFX90A-TGSPLIT-NEXT:    s_endpgm
7007;
7008; GFX940-NOTTGSPLIT-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
7009; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7010; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7011; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7012; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7013; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7014; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7015; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
7016; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7017; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
7018; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7019; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
7020; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7021; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7022;
7023; GFX940-TGSPLIT-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
7024; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7025; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7026; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7027; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7028; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7029; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7030; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
7031; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7032; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
7033; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7034; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
7035; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7036; GFX940-TGSPLIT-NEXT:    s_endpgm
7037;
7038; GFX11-WGP-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
7039; GFX11-WGP:       ; %bb.0: ; %entry
7040; GFX11-WGP-NEXT:    s_clause 0x1
7041; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7042; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7043; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7044; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7045; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
7046; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7047; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7048; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7049; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7050; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7051; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7052; GFX11-WGP-NEXT:    buffer_gl0_inv
7053; GFX11-WGP-NEXT:    buffer_gl1_inv
7054; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7055; GFX11-WGP-NEXT:    s_endpgm
7056;
7057; GFX11-CU-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
7058; GFX11-CU:       ; %bb.0: ; %entry
7059; GFX11-CU-NEXT:    s_clause 0x1
7060; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7061; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7062; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7063; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7064; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
7065; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7066; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7067; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7068; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7069; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7070; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7071; GFX11-CU-NEXT:    buffer_gl0_inv
7072; GFX11-CU-NEXT:    buffer_gl1_inv
7073; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7074; GFX11-CU-NEXT:    s_endpgm
7075    i32* %out, i32 %in, i32 %old) {
7076entry:
7077  %gep = getelementptr i32, i32* %out, i32 4
7078  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acquire seq_cst
7079  %val0 = extractvalue { i32, i1 } %val, 0
7080  store i32 %val0, i32* %out, align 4
7081  ret void
7082}
7083
7084define amdgpu_kernel void @flat_system_release_seq_cst_ret_cmpxchg(
7085; GFX7-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
7086; GFX7:       ; %bb.0: ; %entry
7087; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7088; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7089; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7090; GFX7-NEXT:    s_add_u32 s4, s0, 16
7091; GFX7-NEXT:    s_addc_u32 s5, s1, 0
7092; GFX7-NEXT:    v_mov_b32_e32 v0, s4
7093; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7094; GFX7-NEXT:    v_mov_b32_e32 v1, s5
7095; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7096; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7097; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7098; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7099; GFX7-NEXT:    buffer_wbinvl1_vol
7100; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7101; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7102; GFX7-NEXT:    flat_store_dword v[0:1], v2
7103; GFX7-NEXT:    s_endpgm
7104;
7105; GFX10-WGP-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
7106; GFX10-WGP:       ; %bb.0: ; %entry
7107; GFX10-WGP-NEXT:    s_clause 0x1
7108; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7109; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7110; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7111; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
7112; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
7113; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
7114; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7115; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
7116; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7117; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7118; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7119; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7120; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7121; GFX10-WGP-NEXT:    buffer_gl0_inv
7122; GFX10-WGP-NEXT:    buffer_gl1_inv
7123; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7124; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7125; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7126; GFX10-WGP-NEXT:    s_endpgm
7127;
7128; GFX10-CU-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
7129; GFX10-CU:       ; %bb.0: ; %entry
7130; GFX10-CU-NEXT:    s_clause 0x1
7131; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7132; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7133; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7134; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
7135; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
7136; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
7137; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7138; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
7139; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7140; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7141; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7142; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7143; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7144; GFX10-CU-NEXT:    buffer_gl0_inv
7145; GFX10-CU-NEXT:    buffer_gl1_inv
7146; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7147; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7148; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7149; GFX10-CU-NEXT:    s_endpgm
7150;
7151; SKIP-CACHE-INV-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
7152; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7153; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7154; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7155; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7156; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
7157; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
7158; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7159; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7160; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7161; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7162; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7163; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7164; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7165; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7166; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7167; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7168; SKIP-CACHE-INV-NEXT:    s_endpgm
7169;
7170; GFX90A-NOTTGSPLIT-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
7171; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7172; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7173; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7174; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7175; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7176; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7177; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
7178; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7179; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7180; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7181; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
7182; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7183; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7184; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7185;
7186; GFX90A-TGSPLIT-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
7187; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7188; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7189; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7190; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7191; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7192; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7193; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
7194; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7195; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7196; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7197; GFX90A-TGSPLIT-NEXT:    buffer_invl2
7198; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7199; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7200; GFX90A-TGSPLIT-NEXT:    s_endpgm
7201;
7202; GFX940-NOTTGSPLIT-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
7203; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7204; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7205; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7206; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7207; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7208; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7209; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
7210; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7211; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
7212; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7213; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
7214; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7215; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7216;
7217; GFX940-TGSPLIT-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
7218; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7219; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7220; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7221; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7222; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7223; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7224; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
7225; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7226; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
7227; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7228; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
7229; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7230; GFX940-TGSPLIT-NEXT:    s_endpgm
7231;
7232; GFX11-WGP-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
7233; GFX11-WGP:       ; %bb.0: ; %entry
7234; GFX11-WGP-NEXT:    s_clause 0x1
7235; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7236; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7237; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7238; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7239; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
7240; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7241; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7242; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7243; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7244; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7245; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7246; GFX11-WGP-NEXT:    buffer_gl0_inv
7247; GFX11-WGP-NEXT:    buffer_gl1_inv
7248; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7249; GFX11-WGP-NEXT:    s_endpgm
7250;
7251; GFX11-CU-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
7252; GFX11-CU:       ; %bb.0: ; %entry
7253; GFX11-CU-NEXT:    s_clause 0x1
7254; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7255; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7256; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7257; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7258; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
7259; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7260; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7261; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7262; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7263; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7264; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7265; GFX11-CU-NEXT:    buffer_gl0_inv
7266; GFX11-CU-NEXT:    buffer_gl1_inv
7267; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7268; GFX11-CU-NEXT:    s_endpgm
7269    i32* %out, i32 %in, i32 %old) {
7270entry:
7271  %gep = getelementptr i32, i32* %out, i32 4
7272  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in release seq_cst
7273  %val0 = extractvalue { i32, i1 } %val, 0
7274  store i32 %val0, i32* %out, align 4
7275  ret void
7276}
7277
7278define amdgpu_kernel void @flat_system_acq_rel_seq_cst_ret_cmpxchg(
7279; GFX7-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
7280; GFX7:       ; %bb.0: ; %entry
7281; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7282; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7283; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7284; GFX7-NEXT:    s_add_u32 s4, s0, 16
7285; GFX7-NEXT:    s_addc_u32 s5, s1, 0
7286; GFX7-NEXT:    v_mov_b32_e32 v0, s4
7287; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7288; GFX7-NEXT:    v_mov_b32_e32 v1, s5
7289; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7290; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7291; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7292; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7293; GFX7-NEXT:    buffer_wbinvl1_vol
7294; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7295; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7296; GFX7-NEXT:    flat_store_dword v[0:1], v2
7297; GFX7-NEXT:    s_endpgm
7298;
7299; GFX10-WGP-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
7300; GFX10-WGP:       ; %bb.0: ; %entry
7301; GFX10-WGP-NEXT:    s_clause 0x1
7302; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7303; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7304; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7305; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
7306; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
7307; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
7308; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7309; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
7310; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7311; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7312; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7313; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7314; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7315; GFX10-WGP-NEXT:    buffer_gl0_inv
7316; GFX10-WGP-NEXT:    buffer_gl1_inv
7317; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7318; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7319; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7320; GFX10-WGP-NEXT:    s_endpgm
7321;
7322; GFX10-CU-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
7323; GFX10-CU:       ; %bb.0: ; %entry
7324; GFX10-CU-NEXT:    s_clause 0x1
7325; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7326; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7327; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7328; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
7329; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
7330; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
7331; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7332; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
7333; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7334; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7335; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7336; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7337; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7338; GFX10-CU-NEXT:    buffer_gl0_inv
7339; GFX10-CU-NEXT:    buffer_gl1_inv
7340; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7341; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7342; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7343; GFX10-CU-NEXT:    s_endpgm
7344;
7345; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
7346; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7347; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7348; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7349; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7350; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
7351; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
7352; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7353; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7354; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7355; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7356; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7357; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7358; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7359; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7360; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7361; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7362; SKIP-CACHE-INV-NEXT:    s_endpgm
7363;
7364; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
7365; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7366; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7367; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7368; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7369; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7370; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7371; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
7372; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7373; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7374; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7375; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
7376; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7377; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7378; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7379;
7380; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
7381; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7382; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7383; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7384; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7385; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7386; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7387; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
7388; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7389; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7390; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7391; GFX90A-TGSPLIT-NEXT:    buffer_invl2
7392; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7393; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7394; GFX90A-TGSPLIT-NEXT:    s_endpgm
7395;
7396; GFX940-NOTTGSPLIT-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
7397; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7398; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7399; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7400; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7401; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7402; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7403; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
7404; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7405; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
7406; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7407; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
7408; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7409; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7410;
7411; GFX940-TGSPLIT-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
7412; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7413; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7414; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7415; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7416; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7417; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7418; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
7419; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7420; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
7421; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7422; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
7423; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7424; GFX940-TGSPLIT-NEXT:    s_endpgm
7425;
7426; GFX11-WGP-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
7427; GFX11-WGP:       ; %bb.0: ; %entry
7428; GFX11-WGP-NEXT:    s_clause 0x1
7429; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7430; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7431; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7432; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7433; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
7434; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7435; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7436; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7437; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7438; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7439; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7440; GFX11-WGP-NEXT:    buffer_gl0_inv
7441; GFX11-WGP-NEXT:    buffer_gl1_inv
7442; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7443; GFX11-WGP-NEXT:    s_endpgm
7444;
7445; GFX11-CU-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
7446; GFX11-CU:       ; %bb.0: ; %entry
7447; GFX11-CU-NEXT:    s_clause 0x1
7448; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7449; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7450; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7451; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7452; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
7453; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7454; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7455; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7456; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7457; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7458; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7459; GFX11-CU-NEXT:    buffer_gl0_inv
7460; GFX11-CU-NEXT:    buffer_gl1_inv
7461; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7462; GFX11-CU-NEXT:    s_endpgm
7463    i32* %out, i32 %in, i32 %old) {
7464entry:
7465  %gep = getelementptr i32, i32* %out, i32 4
7466  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acq_rel seq_cst
7467  %val0 = extractvalue { i32, i1 } %val, 0
7468  store i32 %val0, i32* %out, align 4
7469  ret void
7470}
7471
7472define amdgpu_kernel void @flat_system_seq_cst_seq_cst_ret_cmpxchg(
7473; GFX7-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
7474; GFX7:       ; %bb.0: ; %entry
7475; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7476; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7477; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7478; GFX7-NEXT:    s_add_u32 s4, s0, 16
7479; GFX7-NEXT:    s_addc_u32 s5, s1, 0
7480; GFX7-NEXT:    v_mov_b32_e32 v0, s4
7481; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7482; GFX7-NEXT:    v_mov_b32_e32 v1, s5
7483; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7484; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7485; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7486; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7487; GFX7-NEXT:    buffer_wbinvl1_vol
7488; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7489; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7490; GFX7-NEXT:    flat_store_dword v[0:1], v2
7491; GFX7-NEXT:    s_endpgm
7492;
7493; GFX10-WGP-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
7494; GFX10-WGP:       ; %bb.0: ; %entry
7495; GFX10-WGP-NEXT:    s_clause 0x1
7496; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7497; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7498; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7499; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
7500; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
7501; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
7502; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7503; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
7504; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7505; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7506; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7507; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7508; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7509; GFX10-WGP-NEXT:    buffer_gl0_inv
7510; GFX10-WGP-NEXT:    buffer_gl1_inv
7511; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7512; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7513; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7514; GFX10-WGP-NEXT:    s_endpgm
7515;
7516; GFX10-CU-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
7517; GFX10-CU:       ; %bb.0: ; %entry
7518; GFX10-CU-NEXT:    s_clause 0x1
7519; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7520; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7521; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7522; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
7523; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
7524; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
7525; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7526; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
7527; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7528; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7529; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7530; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7531; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7532; GFX10-CU-NEXT:    buffer_gl0_inv
7533; GFX10-CU-NEXT:    buffer_gl1_inv
7534; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7535; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7536; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7537; GFX10-CU-NEXT:    s_endpgm
7538;
7539; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
7540; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7541; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7542; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7543; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7544; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
7545; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
7546; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7547; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7548; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7549; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7550; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7551; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7552; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7553; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7554; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7555; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7556; SKIP-CACHE-INV-NEXT:    s_endpgm
7557;
7558; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
7559; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7560; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7561; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7562; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7563; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7564; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7565; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
7566; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7567; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7568; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7569; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
7570; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7571; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7572; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7573;
7574; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
7575; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7576; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7577; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7578; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7579; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7580; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7581; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
7582; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7583; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7584; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7585; GFX90A-TGSPLIT-NEXT:    buffer_invl2
7586; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7587; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7588; GFX90A-TGSPLIT-NEXT:    s_endpgm
7589;
7590; GFX940-NOTTGSPLIT-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
7591; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7592; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7593; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7594; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7595; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7596; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7597; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
7598; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7599; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
7600; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7601; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
7602; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7603; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7604;
7605; GFX940-TGSPLIT-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
7606; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7607; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7608; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7609; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7610; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7611; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7612; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
7613; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7614; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
7615; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7616; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
7617; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7618; GFX940-TGSPLIT-NEXT:    s_endpgm
7619;
7620; GFX11-WGP-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
7621; GFX11-WGP:       ; %bb.0: ; %entry
7622; GFX11-WGP-NEXT:    s_clause 0x1
7623; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7624; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7625; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7626; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7627; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
7628; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7629; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7630; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7631; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7632; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7633; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7634; GFX11-WGP-NEXT:    buffer_gl0_inv
7635; GFX11-WGP-NEXT:    buffer_gl1_inv
7636; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7637; GFX11-WGP-NEXT:    s_endpgm
7638;
7639; GFX11-CU-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
7640; GFX11-CU:       ; %bb.0: ; %entry
7641; GFX11-CU-NEXT:    s_clause 0x1
7642; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7643; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7644; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7645; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7646; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
7647; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7648; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7649; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7650; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7651; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7652; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7653; GFX11-CU-NEXT:    buffer_gl0_inv
7654; GFX11-CU-NEXT:    buffer_gl1_inv
7655; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7656; GFX11-CU-NEXT:    s_endpgm
7657    i32* %out, i32 %in, i32 %old) {
7658entry:
7659  %gep = getelementptr i32, i32* %out, i32 4
7660  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst seq_cst
7661  %val0 = extractvalue { i32, i1 } %val, 0
7662  store i32 %val0, i32* %out, align 4
7663  ret void
7664}
7665
7666define amdgpu_kernel void @flat_system_one_as_unordered_load(
7667; GFX7-LABEL: flat_system_one_as_unordered_load:
7668; GFX7:       ; %bb.0: ; %entry
7669; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7670; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7671; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7672; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7673; GFX7-NEXT:    flat_load_dword v2, v[0:1]
7674; GFX7-NEXT:    v_mov_b32_e32 v0, s2
7675; GFX7-NEXT:    v_mov_b32_e32 v1, s3
7676; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7677; GFX7-NEXT:    flat_store_dword v[0:1], v2
7678; GFX7-NEXT:    s_endpgm
7679;
7680; GFX10-WGP-LABEL: flat_system_one_as_unordered_load:
7681; GFX10-WGP:       ; %bb.0: ; %entry
7682; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7683; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7684; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7685; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7686; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
7687; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
7688; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
7689; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7690; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7691; GFX10-WGP-NEXT:    s_endpgm
7692;
7693; GFX10-CU-LABEL: flat_system_one_as_unordered_load:
7694; GFX10-CU:       ; %bb.0: ; %entry
7695; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7696; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7697; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7698; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7699; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
7700; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
7701; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
7702; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7703; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7704; GFX10-CU-NEXT:    s_endpgm
7705;
7706; SKIP-CACHE-INV-LABEL: flat_system_one_as_unordered_load:
7707; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7708; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7709; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7710; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7711; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
7712; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
7713; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7714; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7715; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7716; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7717; SKIP-CACHE-INV-NEXT:    s_endpgm
7718;
7719; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_unordered_load:
7720; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7721; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7722; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7723; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7724; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7725; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
7726; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7727; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7728; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7729; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7730; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7731;
7732; GFX90A-TGSPLIT-LABEL: flat_system_one_as_unordered_load:
7733; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7734; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7735; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7736; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7737; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7738; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
7739; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7740; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7741; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7742; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7743; GFX90A-TGSPLIT-NEXT:    s_endpgm
7744;
7745; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_unordered_load:
7746; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7747; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7748; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7749; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7750; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7751; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
7752; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7753; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7754; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7755; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7756; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7757;
7758; GFX940-TGSPLIT-LABEL: flat_system_one_as_unordered_load:
7759; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7760; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7761; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7762; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7763; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7764; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
7765; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7766; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7767; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7768; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7769; GFX940-TGSPLIT-NEXT:    s_endpgm
7770;
7771; GFX11-WGP-LABEL: flat_system_one_as_unordered_load:
7772; GFX11-WGP:       ; %bb.0: ; %entry
7773; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7774; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7775; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
7776; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
7777; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1]
7778; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7779; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7780; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7781; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7782; GFX11-WGP-NEXT:    s_endpgm
7783;
7784; GFX11-CU-LABEL: flat_system_one_as_unordered_load:
7785; GFX11-CU:       ; %bb.0: ; %entry
7786; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7787; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7788; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
7789; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
7790; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
7791; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7792; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7793; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7794; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7795; GFX11-CU-NEXT:    s_endpgm
7796    i32* %in, i32* %out) {
7797entry:
7798  %val = load atomic i32, i32* %in syncscope("one-as") unordered, align 4
7799  store i32 %val, i32* %out
7800  ret void
7801}
7802
7803define amdgpu_kernel void @flat_system_one_as_monotonic_load(
7804; GFX7-LABEL: flat_system_one_as_monotonic_load:
7805; GFX7:       ; %bb.0: ; %entry
7806; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7807; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7808; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7809; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7810; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
7811; GFX7-NEXT:    v_mov_b32_e32 v0, s2
7812; GFX7-NEXT:    v_mov_b32_e32 v1, s3
7813; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7814; GFX7-NEXT:    flat_store_dword v[0:1], v2
7815; GFX7-NEXT:    s_endpgm
7816;
7817; GFX10-WGP-LABEL: flat_system_one_as_monotonic_load:
7818; GFX10-WGP:       ; %bb.0: ; %entry
7819; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7820; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7821; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7822; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7823; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
7824; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
7825; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
7826; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7827; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7828; GFX10-WGP-NEXT:    s_endpgm
7829;
7830; GFX10-CU-LABEL: flat_system_one_as_monotonic_load:
7831; GFX10-CU:       ; %bb.0: ; %entry
7832; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7833; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7834; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7835; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7836; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
7837; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
7838; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
7839; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7840; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7841; GFX10-CU-NEXT:    s_endpgm
7842;
7843; SKIP-CACHE-INV-LABEL: flat_system_one_as_monotonic_load:
7844; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7845; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7846; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7847; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7848; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
7849; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
7850; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7851; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7852; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7853; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7854; SKIP-CACHE-INV-NEXT:    s_endpgm
7855;
7856; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_load:
7857; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7858; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7859; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7860; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7861; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7862; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
7863; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7864; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7865; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7866; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7867; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7868;
7869; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_load:
7870; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7871; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7872; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7873; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7874; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7875; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
7876; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7877; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7878; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7879; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7880; GFX90A-TGSPLIT-NEXT:    s_endpgm
7881;
7882; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_load:
7883; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7884; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7885; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7886; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7887; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7888; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0 sc1
7889; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7890; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7891; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7892; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7893; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7894;
7895; GFX940-TGSPLIT-LABEL: flat_system_one_as_monotonic_load:
7896; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7897; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7898; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7899; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7900; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7901; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0 sc1
7902; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7903; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7904; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7905; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7906; GFX940-TGSPLIT-NEXT:    s_endpgm
7907;
7908; GFX11-WGP-LABEL: flat_system_one_as_monotonic_load:
7909; GFX11-WGP:       ; %bb.0: ; %entry
7910; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7911; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7912; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
7913; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
7914; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc
7915; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7916; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7917; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7918; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7919; GFX11-WGP-NEXT:    s_endpgm
7920;
7921; GFX11-CU-LABEL: flat_system_one_as_monotonic_load:
7922; GFX11-CU:       ; %bb.0: ; %entry
7923; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7924; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7925; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
7926; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
7927; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1] glc
7928; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7929; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7930; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7931; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7932; GFX11-CU-NEXT:    s_endpgm
7933    i32* %in, i32* %out) {
7934entry:
7935  %val = load atomic i32, i32* %in syncscope("one-as") monotonic, align 4
7936  store i32 %val, i32* %out
7937  ret void
7938}
7939
7940define amdgpu_kernel void @flat_system_one_as_acquire_load(
7941; GFX7-LABEL: flat_system_one_as_acquire_load:
7942; GFX7:       ; %bb.0: ; %entry
7943; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7944; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7945; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7946; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7947; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
7948; GFX7-NEXT:    s_waitcnt vmcnt(0)
7949; GFX7-NEXT:    buffer_wbinvl1_vol
7950; GFX7-NEXT:    v_mov_b32_e32 v0, s2
7951; GFX7-NEXT:    v_mov_b32_e32 v1, s3
7952; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7953; GFX7-NEXT:    flat_store_dword v[0:1], v2
7954; GFX7-NEXT:    s_endpgm
7955;
7956; GFX10-WGP-LABEL: flat_system_one_as_acquire_load:
7957; GFX10-WGP:       ; %bb.0: ; %entry
7958; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7959; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7960; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7961; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7962; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
7963; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7964; GFX10-WGP-NEXT:    buffer_gl0_inv
7965; GFX10-WGP-NEXT:    buffer_gl1_inv
7966; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
7967; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
7968; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7969; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7970; GFX10-WGP-NEXT:    s_endpgm
7971;
7972; GFX10-CU-LABEL: flat_system_one_as_acquire_load:
7973; GFX10-CU:       ; %bb.0: ; %entry
7974; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7975; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7976; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7977; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7978; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
7979; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7980; GFX10-CU-NEXT:    buffer_gl0_inv
7981; GFX10-CU-NEXT:    buffer_gl1_inv
7982; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
7983; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
7984; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7985; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7986; GFX10-CU-NEXT:    s_endpgm
7987;
7988; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_load:
7989; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7990; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7991; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7992; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7993; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
7994; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
7995; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7996; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7997; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7998; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7999; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8000; SKIP-CACHE-INV-NEXT:    s_endpgm
8001;
8002; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_load:
8003; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8004; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8005; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8006; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8007; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
8008; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
8009; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8010; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
8011; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
8012; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
8013; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
8014; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8015; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8016; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8017;
8018; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_load:
8019; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8020; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8021; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8022; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8023; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
8024; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
8025; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8026; GFX90A-TGSPLIT-NEXT:    buffer_invl2
8027; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8028; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
8029; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
8030; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8031; GFX90A-TGSPLIT-NEXT:    s_endpgm
8032;
8033; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_load:
8034; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8035; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8036; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8037; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8038; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
8039; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0 sc1
8040; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8041; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
8042; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
8043; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
8044; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8045; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8046; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8047;
8048; GFX940-TGSPLIT-LABEL: flat_system_one_as_acquire_load:
8049; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8050; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8051; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8052; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8053; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
8054; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0 sc1
8055; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8056; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
8057; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
8058; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
8059; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8060; GFX940-TGSPLIT-NEXT:    s_endpgm
8061;
8062; GFX11-WGP-LABEL: flat_system_one_as_acquire_load:
8063; GFX11-WGP:       ; %bb.0: ; %entry
8064; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8065; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8066; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
8067; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
8068; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc
8069; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8070; GFX11-WGP-NEXT:    buffer_gl0_inv
8071; GFX11-WGP-NEXT:    buffer_gl1_inv
8072; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8073; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8074; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8075; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
8076; GFX11-WGP-NEXT:    s_endpgm
8077;
8078; GFX11-CU-LABEL: flat_system_one_as_acquire_load:
8079; GFX11-CU:       ; %bb.0: ; %entry
8080; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8081; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8082; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
8083; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
8084; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1] glc
8085; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
8086; GFX11-CU-NEXT:    buffer_gl0_inv
8087; GFX11-CU-NEXT:    buffer_gl1_inv
8088; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8089; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8090; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8091; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8092; GFX11-CU-NEXT:    s_endpgm
8093    i32* %in, i32* %out) {
8094entry:
8095  %val = load atomic i32, i32* %in syncscope("one-as") acquire, align 4
8096  store i32 %val, i32* %out
8097  ret void
8098}
8099
8100define amdgpu_kernel void @flat_system_one_as_seq_cst_load(
8101; GFX7-LABEL: flat_system_one_as_seq_cst_load:
8102; GFX7:       ; %bb.0: ; %entry
8103; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8104; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8105; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8106; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8107; GFX7-NEXT:    s_waitcnt vmcnt(0)
8108; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
8109; GFX7-NEXT:    s_waitcnt vmcnt(0)
8110; GFX7-NEXT:    buffer_wbinvl1_vol
8111; GFX7-NEXT:    v_mov_b32_e32 v0, s2
8112; GFX7-NEXT:    v_mov_b32_e32 v1, s3
8113; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8114; GFX7-NEXT:    flat_store_dword v[0:1], v2
8115; GFX7-NEXT:    s_endpgm
8116;
8117; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_load:
8118; GFX10-WGP:       ; %bb.0: ; %entry
8119; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8120; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8121; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8122; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8123; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8124; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8125; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
8126; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8127; GFX10-WGP-NEXT:    buffer_gl0_inv
8128; GFX10-WGP-NEXT:    buffer_gl1_inv
8129; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
8130; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
8131; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8132; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8133; GFX10-WGP-NEXT:    s_endpgm
8134;
8135; GFX10-CU-LABEL: flat_system_one_as_seq_cst_load:
8136; GFX10-CU:       ; %bb.0: ; %entry
8137; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8138; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8139; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8140; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8141; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8142; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8143; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
8144; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8145; GFX10-CU-NEXT:    buffer_gl0_inv
8146; GFX10-CU-NEXT:    buffer_gl1_inv
8147; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
8148; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
8149; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8150; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8151; GFX10-CU-NEXT:    s_endpgm
8152;
8153; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_load:
8154; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8155; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8156; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8157; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
8158; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
8159; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8160; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
8161; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8162; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8163; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8164; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8165; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8166; SKIP-CACHE-INV-NEXT:    s_endpgm
8167;
8168; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_load:
8169; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8170; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8171; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8172; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8173; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
8174; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8175; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
8176; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8177; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
8178; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
8179; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
8180; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
8181; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8182; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8183; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8184;
8185; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_load:
8186; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8187; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8188; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8189; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8190; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
8191; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8192; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
8193; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8194; GFX90A-TGSPLIT-NEXT:    buffer_invl2
8195; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8196; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
8197; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
8198; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8199; GFX90A-TGSPLIT-NEXT:    s_endpgm
8200;
8201; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_load:
8202; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8203; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8204; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8205; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8206; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
8207; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8208; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0 sc1
8209; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8210; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
8211; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
8212; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
8213; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8214; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8215; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8216;
8217; GFX940-TGSPLIT-LABEL: flat_system_one_as_seq_cst_load:
8218; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8219; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8220; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8221; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8222; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
8223; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8224; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0 sc1
8225; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8226; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
8227; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
8228; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
8229; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8230; GFX940-TGSPLIT-NEXT:    s_endpgm
8231;
8232; GFX11-WGP-LABEL: flat_system_one_as_seq_cst_load:
8233; GFX11-WGP:       ; %bb.0: ; %entry
8234; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8235; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8236; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
8237; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
8238; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8239; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8240; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc
8241; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8242; GFX11-WGP-NEXT:    buffer_gl0_inv
8243; GFX11-WGP-NEXT:    buffer_gl1_inv
8244; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8245; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8246; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8247; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
8248; GFX11-WGP-NEXT:    s_endpgm
8249;
8250; GFX11-CU-LABEL: flat_system_one_as_seq_cst_load:
8251; GFX11-CU:       ; %bb.0: ; %entry
8252; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8253; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8254; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
8255; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
8256; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
8257; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8258; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1] glc
8259; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
8260; GFX11-CU-NEXT:    buffer_gl0_inv
8261; GFX11-CU-NEXT:    buffer_gl1_inv
8262; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8263; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8264; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8265; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8266; GFX11-CU-NEXT:    s_endpgm
8267    i32* %in, i32* %out) {
8268entry:
8269  %val = load atomic i32, i32* %in syncscope("one-as") seq_cst, align 4
8270  store i32 %val, i32* %out
8271  ret void
8272}
8273
8274define amdgpu_kernel void @flat_system_one_as_unordered_store(
8275; GFX7-LABEL: flat_system_one_as_unordered_store:
8276; GFX7:       ; %bb.0: ; %entry
8277; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
8278; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
8279; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8280; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8281; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8282; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8283; GFX7-NEXT:    flat_store_dword v[0:1], v2
8284; GFX7-NEXT:    s_endpgm
8285;
8286; GFX10-WGP-LABEL: flat_system_one_as_unordered_store:
8287; GFX10-WGP:       ; %bb.0: ; %entry
8288; GFX10-WGP-NEXT:    s_clause 0x1
8289; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8290; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
8291; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8292; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8293; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8294; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8295; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8296; GFX10-WGP-NEXT:    s_endpgm
8297;
8298; GFX10-CU-LABEL: flat_system_one_as_unordered_store:
8299; GFX10-CU:       ; %bb.0: ; %entry
8300; GFX10-CU-NEXT:    s_clause 0x1
8301; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8302; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
8303; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8304; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8305; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8306; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8307; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8308; GFX10-CU-NEXT:    s_endpgm
8309;
8310; SKIP-CACHE-INV-LABEL: flat_system_one_as_unordered_store:
8311; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8312; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
8313; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
8314; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8315; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8316; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8317; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8318; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8319; SKIP-CACHE-INV-NEXT:    s_endpgm
8320;
8321; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_unordered_store:
8322; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8323; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8324; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
8325; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8326; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8327; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8328; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8329; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8330;
8331; GFX90A-TGSPLIT-LABEL: flat_system_one_as_unordered_store:
8332; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8333; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8334; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
8335; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8336; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8337; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8338; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8339; GFX90A-TGSPLIT-NEXT:    s_endpgm
8340;
8341; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_unordered_store:
8342; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8343; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8344; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
8345; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8346; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8347; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8348; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8349; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8350;
8351; GFX940-TGSPLIT-LABEL: flat_system_one_as_unordered_store:
8352; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8353; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8354; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
8355; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8356; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8357; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8358; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8359; GFX940-TGSPLIT-NEXT:    s_endpgm
8360;
8361; GFX11-WGP-LABEL: flat_system_one_as_unordered_store:
8362; GFX11-WGP:       ; %bb.0: ; %entry
8363; GFX11-WGP-NEXT:    s_clause 0x1
8364; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8365; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
8366; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8367; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8368; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8369; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8370; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
8371; GFX11-WGP-NEXT:    s_endpgm
8372;
8373; GFX11-CU-LABEL: flat_system_one_as_unordered_store:
8374; GFX11-CU:       ; %bb.0: ; %entry
8375; GFX11-CU-NEXT:    s_clause 0x1
8376; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8377; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
8378; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8379; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8380; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8381; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8382; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8383; GFX11-CU-NEXT:    s_endpgm
8384    i32 %in, i32* %out) {
8385entry:
8386  store atomic i32 %in, i32* %out syncscope("one-as") unordered, align 4
8387  ret void
8388}
8389
8390define amdgpu_kernel void @flat_system_one_as_monotonic_store(
8391; GFX7-LABEL: flat_system_one_as_monotonic_store:
8392; GFX7:       ; %bb.0: ; %entry
8393; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
8394; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
8395; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8396; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8397; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8398; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8399; GFX7-NEXT:    flat_store_dword v[0:1], v2
8400; GFX7-NEXT:    s_endpgm
8401;
8402; GFX10-WGP-LABEL: flat_system_one_as_monotonic_store:
8403; GFX10-WGP:       ; %bb.0: ; %entry
8404; GFX10-WGP-NEXT:    s_clause 0x1
8405; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8406; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
8407; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8408; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8409; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8410; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8411; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8412; GFX10-WGP-NEXT:    s_endpgm
8413;
8414; GFX10-CU-LABEL: flat_system_one_as_monotonic_store:
8415; GFX10-CU:       ; %bb.0: ; %entry
8416; GFX10-CU-NEXT:    s_clause 0x1
8417; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8418; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
8419; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8420; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8421; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8422; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8423; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8424; GFX10-CU-NEXT:    s_endpgm
8425;
8426; SKIP-CACHE-INV-LABEL: flat_system_one_as_monotonic_store:
8427; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8428; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
8429; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
8430; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8431; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8432; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8433; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8434; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8435; SKIP-CACHE-INV-NEXT:    s_endpgm
8436;
8437; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_store:
8438; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8439; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8440; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
8441; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8442; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8443; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8444; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8445; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8446;
8447; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_store:
8448; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8449; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8450; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
8451; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8452; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8453; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8454; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8455; GFX90A-TGSPLIT-NEXT:    s_endpgm
8456;
8457; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_store:
8458; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8459; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8460; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
8461; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8462; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8463; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8464; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
8465; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8466;
8467; GFX940-TGSPLIT-LABEL: flat_system_one_as_monotonic_store:
8468; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8469; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8470; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
8471; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8472; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8473; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8474; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
8475; GFX940-TGSPLIT-NEXT:    s_endpgm
8476;
8477; GFX11-WGP-LABEL: flat_system_one_as_monotonic_store:
8478; GFX11-WGP:       ; %bb.0: ; %entry
8479; GFX11-WGP-NEXT:    s_clause 0x1
8480; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8481; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
8482; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8483; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8484; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8485; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8486; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
8487; GFX11-WGP-NEXT:    s_endpgm
8488;
8489; GFX11-CU-LABEL: flat_system_one_as_monotonic_store:
8490; GFX11-CU:       ; %bb.0: ; %entry
8491; GFX11-CU-NEXT:    s_clause 0x1
8492; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8493; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
8494; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8495; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8496; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8497; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8498; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8499; GFX11-CU-NEXT:    s_endpgm
8500    i32 %in, i32* %out) {
8501entry:
8502  store atomic i32 %in, i32* %out syncscope("one-as") monotonic, align 4
8503  ret void
8504}
8505
8506define amdgpu_kernel void @flat_system_one_as_release_store(
8507; GFX7-LABEL: flat_system_one_as_release_store:
8508; GFX7:       ; %bb.0: ; %entry
8509; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
8510; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
8511; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8512; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8513; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8514; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8515; GFX7-NEXT:    s_waitcnt vmcnt(0)
8516; GFX7-NEXT:    flat_store_dword v[0:1], v2
8517; GFX7-NEXT:    s_endpgm
8518;
8519; GFX10-WGP-LABEL: flat_system_one_as_release_store:
8520; GFX10-WGP:       ; %bb.0: ; %entry
8521; GFX10-WGP-NEXT:    s_clause 0x1
8522; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8523; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
8524; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8525; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8526; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8527; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8528; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8529; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8530; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8531; GFX10-WGP-NEXT:    s_endpgm
8532;
8533; GFX10-CU-LABEL: flat_system_one_as_release_store:
8534; GFX10-CU:       ; %bb.0: ; %entry
8535; GFX10-CU-NEXT:    s_clause 0x1
8536; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8537; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
8538; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8539; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8540; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8541; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8542; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8543; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8544; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8545; GFX10-CU-NEXT:    s_endpgm
8546;
8547; SKIP-CACHE-INV-LABEL: flat_system_one_as_release_store:
8548; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8549; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
8550; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
8551; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8552; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8553; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8554; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8555; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8556; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8557; SKIP-CACHE-INV-NEXT:    s_endpgm
8558;
8559; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_release_store:
8560; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8561; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8562; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
8563; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8564; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8565; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8566; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
8567; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8568; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8569; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8570;
8571; GFX90A-TGSPLIT-LABEL: flat_system_one_as_release_store:
8572; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8573; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8574; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
8575; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8576; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8577; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8578; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
8579; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8580; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8581; GFX90A-TGSPLIT-NEXT:    s_endpgm
8582;
8583; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_release_store:
8584; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8585; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8586; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
8587; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8588; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8589; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8590; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
8591; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8592; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
8593; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8594;
8595; GFX940-TGSPLIT-LABEL: flat_system_one_as_release_store:
8596; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8597; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8598; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
8599; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8600; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8601; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8602; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
8603; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8604; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
8605; GFX940-TGSPLIT-NEXT:    s_endpgm
8606;
8607; GFX11-WGP-LABEL: flat_system_one_as_release_store:
8608; GFX11-WGP:       ; %bb.0: ; %entry
8609; GFX11-WGP-NEXT:    s_clause 0x1
8610; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8611; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
8612; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8613; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8614; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8615; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8616; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8617; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8618; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
8619; GFX11-WGP-NEXT:    s_endpgm
8620;
8621; GFX11-CU-LABEL: flat_system_one_as_release_store:
8622; GFX11-CU:       ; %bb.0: ; %entry
8623; GFX11-CU-NEXT:    s_clause 0x1
8624; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8625; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
8626; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8627; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8628; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8629; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8630; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
8631; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8632; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8633; GFX11-CU-NEXT:    s_endpgm
8634    i32 %in, i32* %out) {
8635entry:
8636  store atomic i32 %in, i32* %out syncscope("one-as") release, align 4
8637  ret void
8638}
8639
8640define amdgpu_kernel void @flat_system_one_as_seq_cst_store(
8641; GFX7-LABEL: flat_system_one_as_seq_cst_store:
8642; GFX7:       ; %bb.0: ; %entry
8643; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
8644; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
8645; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8646; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8647; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8648; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8649; GFX7-NEXT:    s_waitcnt vmcnt(0)
8650; GFX7-NEXT:    flat_store_dword v[0:1], v2
8651; GFX7-NEXT:    s_endpgm
8652;
8653; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_store:
8654; GFX10-WGP:       ; %bb.0: ; %entry
8655; GFX10-WGP-NEXT:    s_clause 0x1
8656; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8657; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
8658; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8659; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8660; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8661; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8662; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8663; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8664; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8665; GFX10-WGP-NEXT:    s_endpgm
8666;
8667; GFX10-CU-LABEL: flat_system_one_as_seq_cst_store:
8668; GFX10-CU:       ; %bb.0: ; %entry
8669; GFX10-CU-NEXT:    s_clause 0x1
8670; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8671; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
8672; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8673; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8674; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8675; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8676; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8677; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8678; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8679; GFX10-CU-NEXT:    s_endpgm
8680;
8681; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_store:
8682; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8683; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
8684; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
8685; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8686; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8687; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8688; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8689; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8690; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8691; SKIP-CACHE-INV-NEXT:    s_endpgm
8692;
8693; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_store:
8694; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8695; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8696; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
8697; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8698; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8699; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8700; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
8701; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8702; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8703; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8704;
8705; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_store:
8706; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8707; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8708; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
8709; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8710; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8711; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8712; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
8713; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8714; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8715; GFX90A-TGSPLIT-NEXT:    s_endpgm
8716;
8717; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_store:
8718; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8719; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8720; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
8721; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8722; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8723; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8724; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
8725; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8726; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
8727; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8728;
8729; GFX940-TGSPLIT-LABEL: flat_system_one_as_seq_cst_store:
8730; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8731; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8732; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
8733; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8734; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8735; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8736; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
8737; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8738; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
8739; GFX940-TGSPLIT-NEXT:    s_endpgm
8740;
8741; GFX11-WGP-LABEL: flat_system_one_as_seq_cst_store:
8742; GFX11-WGP:       ; %bb.0: ; %entry
8743; GFX11-WGP-NEXT:    s_clause 0x1
8744; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8745; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
8746; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8747; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8748; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8749; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8750; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8751; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8752; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
8753; GFX11-WGP-NEXT:    s_endpgm
8754;
8755; GFX11-CU-LABEL: flat_system_one_as_seq_cst_store:
8756; GFX11-CU:       ; %bb.0: ; %entry
8757; GFX11-CU-NEXT:    s_clause 0x1
8758; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8759; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
8760; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8761; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8762; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8763; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8764; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
8765; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8766; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8767; GFX11-CU-NEXT:    s_endpgm
8768    i32 %in, i32* %out) {
8769entry:
8770  store atomic i32 %in, i32* %out syncscope("one-as") seq_cst, align 4
8771  ret void
8772}
8773
8774define amdgpu_kernel void @flat_system_one_as_monotonic_atomicrmw(
8775; GFX7-LABEL: flat_system_one_as_monotonic_atomicrmw:
8776; GFX7:       ; %bb.0: ; %entry
8777; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8778; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
8779; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8780; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8781; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8782; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8783; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
8784; GFX7-NEXT:    s_endpgm
8785;
8786; GFX10-WGP-LABEL: flat_system_one_as_monotonic_atomicrmw:
8787; GFX10-WGP:       ; %bb.0: ; %entry
8788; GFX10-WGP-NEXT:    s_clause 0x1
8789; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8790; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
8791; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8792; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8793; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8794; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8795; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
8796; GFX10-WGP-NEXT:    s_endpgm
8797;
8798; GFX10-CU-LABEL: flat_system_one_as_monotonic_atomicrmw:
8799; GFX10-CU:       ; %bb.0: ; %entry
8800; GFX10-CU-NEXT:    s_clause 0x1
8801; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8802; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8803; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8804; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8805; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8806; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8807; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
8808; GFX10-CU-NEXT:    s_endpgm
8809;
8810; SKIP-CACHE-INV-LABEL: flat_system_one_as_monotonic_atomicrmw:
8811; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8812; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8813; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
8814; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8815; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8816; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8817; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8818; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
8819; SKIP-CACHE-INV-NEXT:    s_endpgm
8820;
8821; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_atomicrmw:
8822; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8823; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8824; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8825; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8826; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8827; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8828; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8829; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8830;
8831; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_atomicrmw:
8832; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8833; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8834; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8835; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8836; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8837; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8838; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8839; GFX90A-TGSPLIT-NEXT:    s_endpgm
8840;
8841; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_atomicrmw:
8842; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8843; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8844; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8845; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8846; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8847; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8848; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
8849; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8850;
8851; GFX940-TGSPLIT-LABEL: flat_system_one_as_monotonic_atomicrmw:
8852; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8853; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8854; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8855; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8856; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8857; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8858; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
8859; GFX940-TGSPLIT-NEXT:    s_endpgm
8860;
8861; GFX11-WGP-LABEL: flat_system_one_as_monotonic_atomicrmw:
8862; GFX11-WGP:       ; %bb.0: ; %entry
8863; GFX11-WGP-NEXT:    s_clause 0x1
8864; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8865; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
8866; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8867; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8868; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8869; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8870; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
8871; GFX11-WGP-NEXT:    s_endpgm
8872;
8873; GFX11-CU-LABEL: flat_system_one_as_monotonic_atomicrmw:
8874; GFX11-CU:       ; %bb.0: ; %entry
8875; GFX11-CU-NEXT:    s_clause 0x1
8876; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8877; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
8878; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8879; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8880; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8881; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8882; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
8883; GFX11-CU-NEXT:    s_endpgm
8884    i32* %out, i32 %in) {
8885entry:
8886  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") monotonic
8887  ret void
8888}
8889
8890define amdgpu_kernel void @flat_system_one_as_acquire_atomicrmw(
8891; GFX7-LABEL: flat_system_one_as_acquire_atomicrmw:
8892; GFX7:       ; %bb.0: ; %entry
8893; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8894; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
8895; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8896; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8897; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8898; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8899; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
8900; GFX7-NEXT:    s_waitcnt vmcnt(0)
8901; GFX7-NEXT:    buffer_wbinvl1_vol
8902; GFX7-NEXT:    s_endpgm
8903;
8904; GFX10-WGP-LABEL: flat_system_one_as_acquire_atomicrmw:
8905; GFX10-WGP:       ; %bb.0: ; %entry
8906; GFX10-WGP-NEXT:    s_clause 0x1
8907; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8908; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
8909; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8910; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8911; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8912; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8913; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
8914; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8915; GFX10-WGP-NEXT:    buffer_gl0_inv
8916; GFX10-WGP-NEXT:    buffer_gl1_inv
8917; GFX10-WGP-NEXT:    s_endpgm
8918;
8919; GFX10-CU-LABEL: flat_system_one_as_acquire_atomicrmw:
8920; GFX10-CU:       ; %bb.0: ; %entry
8921; GFX10-CU-NEXT:    s_clause 0x1
8922; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8923; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8924; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8925; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8926; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8927; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8928; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
8929; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8930; GFX10-CU-NEXT:    buffer_gl0_inv
8931; GFX10-CU-NEXT:    buffer_gl1_inv
8932; GFX10-CU-NEXT:    s_endpgm
8933;
8934; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_atomicrmw:
8935; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8936; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8937; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
8938; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8939; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8940; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8941; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8942; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
8943; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8944; SKIP-CACHE-INV-NEXT:    s_endpgm
8945;
8946; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_atomicrmw:
8947; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8948; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8949; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8950; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8951; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8952; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8953; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8954; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8955; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
8956; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
8957; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8958;
8959; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_atomicrmw:
8960; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8961; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8962; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8963; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8964; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8965; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8966; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8967; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8968; GFX90A-TGSPLIT-NEXT:    buffer_invl2
8969; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8970; GFX90A-TGSPLIT-NEXT:    s_endpgm
8971;
8972; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_atomicrmw:
8973; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8974; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8975; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8976; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8977; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8978; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8979; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
8980; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8981; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
8982; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8983;
8984; GFX940-TGSPLIT-LABEL: flat_system_one_as_acquire_atomicrmw:
8985; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8986; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8987; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8988; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8989; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8990; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8991; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
8992; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8993; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
8994; GFX940-TGSPLIT-NEXT:    s_endpgm
8995;
8996; GFX11-WGP-LABEL: flat_system_one_as_acquire_atomicrmw:
8997; GFX11-WGP:       ; %bb.0: ; %entry
8998; GFX11-WGP-NEXT:    s_clause 0x1
8999; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9000; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
9001; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9002; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9003; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9004; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9005; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
9006; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9007; GFX11-WGP-NEXT:    buffer_gl0_inv
9008; GFX11-WGP-NEXT:    buffer_gl1_inv
9009; GFX11-WGP-NEXT:    s_endpgm
9010;
9011; GFX11-CU-LABEL: flat_system_one_as_acquire_atomicrmw:
9012; GFX11-CU:       ; %bb.0: ; %entry
9013; GFX11-CU-NEXT:    s_clause 0x1
9014; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9015; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
9016; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9017; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9018; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9019; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9020; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
9021; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9022; GFX11-CU-NEXT:    buffer_gl0_inv
9023; GFX11-CU-NEXT:    buffer_gl1_inv
9024; GFX11-CU-NEXT:    s_endpgm
9025    i32* %out, i32 %in) {
9026entry:
9027  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") acquire
9028  ret void
9029}
9030
9031define amdgpu_kernel void @flat_system_one_as_release_atomicrmw(
9032; GFX7-LABEL: flat_system_one_as_release_atomicrmw:
9033; GFX7:       ; %bb.0: ; %entry
9034; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9035; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
9036; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9037; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9038; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9039; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9040; GFX7-NEXT:    s_waitcnt vmcnt(0)
9041; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
9042; GFX7-NEXT:    s_endpgm
9043;
9044; GFX10-WGP-LABEL: flat_system_one_as_release_atomicrmw:
9045; GFX10-WGP:       ; %bb.0: ; %entry
9046; GFX10-WGP-NEXT:    s_clause 0x1
9047; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9048; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
9049; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9050; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9051; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9052; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9053; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9054; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9055; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
9056; GFX10-WGP-NEXT:    s_endpgm
9057;
9058; GFX10-CU-LABEL: flat_system_one_as_release_atomicrmw:
9059; GFX10-CU:       ; %bb.0: ; %entry
9060; GFX10-CU-NEXT:    s_clause 0x1
9061; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9062; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
9063; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9064; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9065; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9066; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9067; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9068; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9069; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
9070; GFX10-CU-NEXT:    s_endpgm
9071;
9072; SKIP-CACHE-INV-LABEL: flat_system_one_as_release_atomicrmw:
9073; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9074; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9075; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
9076; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9077; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9078; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9079; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9080; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9081; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
9082; SKIP-CACHE-INV-NEXT:    s_endpgm
9083;
9084; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_release_atomicrmw:
9085; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9086; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9087; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9088; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9089; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9090; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9091; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
9092; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9093; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
9094; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9095;
9096; GFX90A-TGSPLIT-LABEL: flat_system_one_as_release_atomicrmw:
9097; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9098; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9099; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9100; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9101; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9102; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9103; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
9104; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9105; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
9106; GFX90A-TGSPLIT-NEXT:    s_endpgm
9107;
9108; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_release_atomicrmw:
9109; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9110; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9111; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9112; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9113; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9114; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9115; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
9116; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9117; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
9118; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9119;
9120; GFX940-TGSPLIT-LABEL: flat_system_one_as_release_atomicrmw:
9121; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9122; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9123; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9124; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9125; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9126; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9127; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
9128; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9129; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
9130; GFX940-TGSPLIT-NEXT:    s_endpgm
9131;
9132; GFX11-WGP-LABEL: flat_system_one_as_release_atomicrmw:
9133; GFX11-WGP:       ; %bb.0: ; %entry
9134; GFX11-WGP-NEXT:    s_clause 0x1
9135; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9136; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
9137; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9138; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9139; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9140; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9141; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9142; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9143; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
9144; GFX11-WGP-NEXT:    s_endpgm
9145;
9146; GFX11-CU-LABEL: flat_system_one_as_release_atomicrmw:
9147; GFX11-CU:       ; %bb.0: ; %entry
9148; GFX11-CU-NEXT:    s_clause 0x1
9149; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9150; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
9151; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9152; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9153; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9154; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9155; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9156; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9157; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
9158; GFX11-CU-NEXT:    s_endpgm
9159    i32* %out, i32 %in) {
9160entry:
9161  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") release
9162  ret void
9163}
9164
9165define amdgpu_kernel void @flat_system_one_as_acq_rel_atomicrmw(
9166; GFX7-LABEL: flat_system_one_as_acq_rel_atomicrmw:
9167; GFX7:       ; %bb.0: ; %entry
9168; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9169; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
9170; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9171; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9172; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9173; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9174; GFX7-NEXT:    s_waitcnt vmcnt(0)
9175; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
9176; GFX7-NEXT:    s_waitcnt vmcnt(0)
9177; GFX7-NEXT:    buffer_wbinvl1_vol
9178; GFX7-NEXT:    s_endpgm
9179;
9180; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_atomicrmw:
9181; GFX10-WGP:       ; %bb.0: ; %entry
9182; GFX10-WGP-NEXT:    s_clause 0x1
9183; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9184; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
9185; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9186; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9187; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9188; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9189; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9190; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9191; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
9192; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9193; GFX10-WGP-NEXT:    buffer_gl0_inv
9194; GFX10-WGP-NEXT:    buffer_gl1_inv
9195; GFX10-WGP-NEXT:    s_endpgm
9196;
9197; GFX10-CU-LABEL: flat_system_one_as_acq_rel_atomicrmw:
9198; GFX10-CU:       ; %bb.0: ; %entry
9199; GFX10-CU-NEXT:    s_clause 0x1
9200; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9201; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
9202; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9203; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9204; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9205; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9206; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9207; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9208; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
9209; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9210; GFX10-CU-NEXT:    buffer_gl0_inv
9211; GFX10-CU-NEXT:    buffer_gl1_inv
9212; GFX10-CU-NEXT:    s_endpgm
9213;
9214; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_atomicrmw:
9215; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9216; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9217; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
9218; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9219; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9220; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9221; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9222; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9223; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
9224; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9225; SKIP-CACHE-INV-NEXT:    s_endpgm
9226;
9227; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_atomicrmw:
9228; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9229; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9230; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9231; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9232; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9233; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9234; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
9235; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9236; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
9237; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9238; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
9239; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9240; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9241;
9242; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_atomicrmw:
9243; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9244; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9245; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9246; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9247; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9248; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9249; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
9250; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9251; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
9252; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9253; GFX90A-TGSPLIT-NEXT:    buffer_invl2
9254; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9255; GFX90A-TGSPLIT-NEXT:    s_endpgm
9256;
9257; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_atomicrmw:
9258; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9259; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9260; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9261; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9262; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9263; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9264; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
9265; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9266; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
9267; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9268; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
9269; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9270;
9271; GFX940-TGSPLIT-LABEL: flat_system_one_as_acq_rel_atomicrmw:
9272; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9273; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9274; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9275; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9276; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9277; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9278; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
9279; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9280; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
9281; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9282; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
9283; GFX940-TGSPLIT-NEXT:    s_endpgm
9284;
9285; GFX11-WGP-LABEL: flat_system_one_as_acq_rel_atomicrmw:
9286; GFX11-WGP:       ; %bb.0: ; %entry
9287; GFX11-WGP-NEXT:    s_clause 0x1
9288; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9289; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
9290; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9291; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9292; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9293; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9294; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9295; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9296; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
9297; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9298; GFX11-WGP-NEXT:    buffer_gl0_inv
9299; GFX11-WGP-NEXT:    buffer_gl1_inv
9300; GFX11-WGP-NEXT:    s_endpgm
9301;
9302; GFX11-CU-LABEL: flat_system_one_as_acq_rel_atomicrmw:
9303; GFX11-CU:       ; %bb.0: ; %entry
9304; GFX11-CU-NEXT:    s_clause 0x1
9305; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9306; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
9307; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9308; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9309; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9310; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9311; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9312; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9313; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
9314; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9315; GFX11-CU-NEXT:    buffer_gl0_inv
9316; GFX11-CU-NEXT:    buffer_gl1_inv
9317; GFX11-CU-NEXT:    s_endpgm
9318    i32* %out, i32 %in) {
9319entry:
9320  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") acq_rel
9321  ret void
9322}
9323
9324define amdgpu_kernel void @flat_system_one_as_seq_cst_atomicrmw(
9325; GFX7-LABEL: flat_system_one_as_seq_cst_atomicrmw:
9326; GFX7:       ; %bb.0: ; %entry
9327; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9328; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
9329; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9330; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9331; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9332; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9333; GFX7-NEXT:    s_waitcnt vmcnt(0)
9334; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
9335; GFX7-NEXT:    s_waitcnt vmcnt(0)
9336; GFX7-NEXT:    buffer_wbinvl1_vol
9337; GFX7-NEXT:    s_endpgm
9338;
9339; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_atomicrmw:
9340; GFX10-WGP:       ; %bb.0: ; %entry
9341; GFX10-WGP-NEXT:    s_clause 0x1
9342; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9343; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
9344; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9345; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9346; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9347; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9348; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9349; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9350; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
9351; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9352; GFX10-WGP-NEXT:    buffer_gl0_inv
9353; GFX10-WGP-NEXT:    buffer_gl1_inv
9354; GFX10-WGP-NEXT:    s_endpgm
9355;
9356; GFX10-CU-LABEL: flat_system_one_as_seq_cst_atomicrmw:
9357; GFX10-CU:       ; %bb.0: ; %entry
9358; GFX10-CU-NEXT:    s_clause 0x1
9359; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9360; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
9361; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9362; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9363; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9364; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9365; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9366; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9367; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
9368; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9369; GFX10-CU-NEXT:    buffer_gl0_inv
9370; GFX10-CU-NEXT:    buffer_gl1_inv
9371; GFX10-CU-NEXT:    s_endpgm
9372;
9373; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_atomicrmw:
9374; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9375; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9376; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
9377; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9378; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9379; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9380; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9381; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9382; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
9383; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9384; SKIP-CACHE-INV-NEXT:    s_endpgm
9385;
9386; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_atomicrmw:
9387; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9388; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9389; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9390; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9391; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9392; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9393; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
9394; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9395; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
9396; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9397; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
9398; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9399; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9400;
9401; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_atomicrmw:
9402; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9403; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9404; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9405; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9406; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9407; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9408; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
9409; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9410; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
9411; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9412; GFX90A-TGSPLIT-NEXT:    buffer_invl2
9413; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9414; GFX90A-TGSPLIT-NEXT:    s_endpgm
9415;
9416; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_atomicrmw:
9417; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9418; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9419; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9420; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9421; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9422; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9423; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
9424; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9425; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
9426; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9427; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
9428; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9429;
9430; GFX940-TGSPLIT-LABEL: flat_system_one_as_seq_cst_atomicrmw:
9431; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9432; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9433; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9434; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9435; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9436; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9437; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
9438; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9439; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
9440; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9441; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
9442; GFX940-TGSPLIT-NEXT:    s_endpgm
9443;
9444; GFX11-WGP-LABEL: flat_system_one_as_seq_cst_atomicrmw:
9445; GFX11-WGP:       ; %bb.0: ; %entry
9446; GFX11-WGP-NEXT:    s_clause 0x1
9447; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9448; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
9449; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9450; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9451; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9452; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9453; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9454; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9455; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
9456; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9457; GFX11-WGP-NEXT:    buffer_gl0_inv
9458; GFX11-WGP-NEXT:    buffer_gl1_inv
9459; GFX11-WGP-NEXT:    s_endpgm
9460;
9461; GFX11-CU-LABEL: flat_system_one_as_seq_cst_atomicrmw:
9462; GFX11-CU:       ; %bb.0: ; %entry
9463; GFX11-CU-NEXT:    s_clause 0x1
9464; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9465; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
9466; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9467; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9468; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9469; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9470; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9471; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9472; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
9473; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9474; GFX11-CU-NEXT:    buffer_gl0_inv
9475; GFX11-CU-NEXT:    buffer_gl1_inv
9476; GFX11-CU-NEXT:    s_endpgm
9477    i32* %out, i32 %in) {
9478entry:
9479  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") seq_cst
9480  ret void
9481}
9482
9483define amdgpu_kernel void @flat_system_one_as_acquire_ret_atomicrmw(
9484; GFX7-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
9485; GFX7:       ; %bb.0: ; %entry
9486; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9487; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
9488; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9489; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9490; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9491; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9492; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9493; GFX7-NEXT:    s_waitcnt vmcnt(0)
9494; GFX7-NEXT:    buffer_wbinvl1_vol
9495; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9496; GFX7-NEXT:    flat_store_dword v[0:1], v2
9497; GFX7-NEXT:    s_endpgm
9498;
9499; GFX10-WGP-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
9500; GFX10-WGP:       ; %bb.0: ; %entry
9501; GFX10-WGP-NEXT:    s_clause 0x1
9502; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9503; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
9504; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9505; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9506; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9507; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9508; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9509; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9510; GFX10-WGP-NEXT:    buffer_gl0_inv
9511; GFX10-WGP-NEXT:    buffer_gl1_inv
9512; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9513; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
9514; GFX10-WGP-NEXT:    s_endpgm
9515;
9516; GFX10-CU-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
9517; GFX10-CU:       ; %bb.0: ; %entry
9518; GFX10-CU-NEXT:    s_clause 0x1
9519; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9520; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
9521; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9522; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9523; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9524; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9525; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9526; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9527; GFX10-CU-NEXT:    buffer_gl0_inv
9528; GFX10-CU-NEXT:    buffer_gl1_inv
9529; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9530; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
9531; GFX10-CU-NEXT:    s_endpgm
9532;
9533; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
9534; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9535; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9536; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
9537; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9538; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9539; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9540; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9541; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9542; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9543; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
9544; SKIP-CACHE-INV-NEXT:    s_endpgm
9545;
9546; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
9547; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9548; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9549; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9550; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9551; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9552; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9553; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9554; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9555; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
9556; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9557; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9558; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9559; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9560;
9561; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
9562; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9563; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9564; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9565; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9566; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9567; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9568; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9569; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9570; GFX90A-TGSPLIT-NEXT:    buffer_invl2
9571; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9572; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9573; GFX90A-TGSPLIT-NEXT:    s_endpgm
9574;
9575; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
9576; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9577; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9578; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9579; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9580; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9581; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9582; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0 sc1
9583; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9584; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
9585; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9586; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9587; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9588;
9589; GFX940-TGSPLIT-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
9590; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9591; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9592; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9593; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9594; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9595; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9596; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0 sc1
9597; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9598; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
9599; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9600; GFX940-TGSPLIT-NEXT:    s_endpgm
9601;
9602; GFX11-WGP-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
9603; GFX11-WGP:       ; %bb.0: ; %entry
9604; GFX11-WGP-NEXT:    s_clause 0x1
9605; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9606; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
9607; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9608; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9609; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9610; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9611; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
9612; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9613; GFX11-WGP-NEXT:    buffer_gl0_inv
9614; GFX11-WGP-NEXT:    buffer_gl1_inv
9615; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9616; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
9617; GFX11-WGP-NEXT:    s_endpgm
9618;
9619; GFX11-CU-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
9620; GFX11-CU:       ; %bb.0: ; %entry
9621; GFX11-CU-NEXT:    s_clause 0x1
9622; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9623; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
9624; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9625; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9626; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9627; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9628; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
9629; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9630; GFX11-CU-NEXT:    buffer_gl0_inv
9631; GFX11-CU-NEXT:    buffer_gl1_inv
9632; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9633; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
9634; GFX11-CU-NEXT:    s_endpgm
9635    i32* %out, i32 %in) {
9636entry:
9637  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") acquire
9638  store i32 %val, i32* %out, align 4
9639  ret void
9640}
9641
9642define amdgpu_kernel void @flat_system_one_as_acq_rel_ret_atomicrmw(
9643; GFX7-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
9644; GFX7:       ; %bb.0: ; %entry
9645; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9646; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
9647; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9648; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9649; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9650; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9651; GFX7-NEXT:    s_waitcnt vmcnt(0)
9652; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9653; GFX7-NEXT:    s_waitcnt vmcnt(0)
9654; GFX7-NEXT:    buffer_wbinvl1_vol
9655; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9656; GFX7-NEXT:    flat_store_dword v[0:1], v2
9657; GFX7-NEXT:    s_endpgm
9658;
9659; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
9660; GFX10-WGP:       ; %bb.0: ; %entry
9661; GFX10-WGP-NEXT:    s_clause 0x1
9662; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9663; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
9664; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9665; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9666; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9667; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9668; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9669; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9670; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9671; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9672; GFX10-WGP-NEXT:    buffer_gl0_inv
9673; GFX10-WGP-NEXT:    buffer_gl1_inv
9674; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9675; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
9676; GFX10-WGP-NEXT:    s_endpgm
9677;
9678; GFX10-CU-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
9679; GFX10-CU:       ; %bb.0: ; %entry
9680; GFX10-CU-NEXT:    s_clause 0x1
9681; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9682; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
9683; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9684; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9685; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9686; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9687; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9688; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9689; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9690; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9691; GFX10-CU-NEXT:    buffer_gl0_inv
9692; GFX10-CU-NEXT:    buffer_gl1_inv
9693; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9694; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
9695; GFX10-CU-NEXT:    s_endpgm
9696;
9697; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
9698; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9699; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9700; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
9701; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9702; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9703; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9704; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9705; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9706; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9707; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9708; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
9709; SKIP-CACHE-INV-NEXT:    s_endpgm
9710;
9711; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
9712; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9713; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9714; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9715; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9716; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9717; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9718; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
9719; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9720; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9721; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9722; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
9723; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9724; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9725; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9726; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9727;
9728; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
9729; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9730; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9731; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9732; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9733; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9734; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9735; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
9736; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9737; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9738; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9739; GFX90A-TGSPLIT-NEXT:    buffer_invl2
9740; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9741; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9742; GFX90A-TGSPLIT-NEXT:    s_endpgm
9743;
9744; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
9745; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9746; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9747; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9748; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9749; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9750; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9751; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
9752; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9753; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0 sc1
9754; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9755; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
9756; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9757; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9758; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9759;
9760; GFX940-TGSPLIT-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
9761; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9762; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9763; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9764; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9765; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9766; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9767; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
9768; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9769; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0 sc1
9770; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9771; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
9772; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9773; GFX940-TGSPLIT-NEXT:    s_endpgm
9774;
9775; GFX11-WGP-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
9776; GFX11-WGP:       ; %bb.0: ; %entry
9777; GFX11-WGP-NEXT:    s_clause 0x1
9778; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9779; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
9780; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9781; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9782; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9783; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9784; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9785; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9786; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
9787; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9788; GFX11-WGP-NEXT:    buffer_gl0_inv
9789; GFX11-WGP-NEXT:    buffer_gl1_inv
9790; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9791; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
9792; GFX11-WGP-NEXT:    s_endpgm
9793;
9794; GFX11-CU-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
9795; GFX11-CU:       ; %bb.0: ; %entry
9796; GFX11-CU-NEXT:    s_clause 0x1
9797; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9798; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
9799; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9800; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9801; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9802; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9803; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9804; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9805; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
9806; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9807; GFX11-CU-NEXT:    buffer_gl0_inv
9808; GFX11-CU-NEXT:    buffer_gl1_inv
9809; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9810; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
9811; GFX11-CU-NEXT:    s_endpgm
9812    i32* %out, i32 %in) {
9813entry:
9814  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") acq_rel
9815  store i32 %val, i32* %out, align 4
9816  ret void
9817}
9818
9819define amdgpu_kernel void @flat_system_one_as_seq_cst_ret_atomicrmw(
9820; GFX7-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
9821; GFX7:       ; %bb.0: ; %entry
9822; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9823; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
9824; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9825; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9826; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9827; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9828; GFX7-NEXT:    s_waitcnt vmcnt(0)
9829; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9830; GFX7-NEXT:    s_waitcnt vmcnt(0)
9831; GFX7-NEXT:    buffer_wbinvl1_vol
9832; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9833; GFX7-NEXT:    flat_store_dword v[0:1], v2
9834; GFX7-NEXT:    s_endpgm
9835;
9836; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
9837; GFX10-WGP:       ; %bb.0: ; %entry
9838; GFX10-WGP-NEXT:    s_clause 0x1
9839; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9840; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
9841; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9842; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9843; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9844; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9845; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9846; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9847; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9848; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9849; GFX10-WGP-NEXT:    buffer_gl0_inv
9850; GFX10-WGP-NEXT:    buffer_gl1_inv
9851; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9852; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
9853; GFX10-WGP-NEXT:    s_endpgm
9854;
9855; GFX10-CU-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
9856; GFX10-CU:       ; %bb.0: ; %entry
9857; GFX10-CU-NEXT:    s_clause 0x1
9858; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9859; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
9860; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9861; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9862; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9863; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9864; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9865; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9866; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9867; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9868; GFX10-CU-NEXT:    buffer_gl0_inv
9869; GFX10-CU-NEXT:    buffer_gl1_inv
9870; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9871; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
9872; GFX10-CU-NEXT:    s_endpgm
9873;
9874; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
9875; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9876; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9877; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
9878; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9879; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9880; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9881; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9882; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9883; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9884; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9885; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
9886; SKIP-CACHE-INV-NEXT:    s_endpgm
9887;
9888; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
9889; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9890; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9891; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9892; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9893; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9894; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9895; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
9896; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9897; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9898; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9899; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
9900; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9901; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9902; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9903; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9904;
9905; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
9906; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9907; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9908; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9909; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9910; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9911; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9912; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
9913; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9914; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9915; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9916; GFX90A-TGSPLIT-NEXT:    buffer_invl2
9917; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9918; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9919; GFX90A-TGSPLIT-NEXT:    s_endpgm
9920;
9921; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
9922; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9923; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9924; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9925; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9926; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9927; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9928; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
9929; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9930; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0 sc1
9931; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9932; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
9933; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9934; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9935; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9936;
9937; GFX940-TGSPLIT-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
9938; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9939; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9940; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9941; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9942; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9943; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9944; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
9945; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9946; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0 sc1
9947; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9948; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
9949; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9950; GFX940-TGSPLIT-NEXT:    s_endpgm
9951;
9952; GFX11-WGP-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
9953; GFX11-WGP:       ; %bb.0: ; %entry
9954; GFX11-WGP-NEXT:    s_clause 0x1
9955; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9956; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
9957; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9958; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9959; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9960; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9961; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9962; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9963; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
9964; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9965; GFX11-WGP-NEXT:    buffer_gl0_inv
9966; GFX11-WGP-NEXT:    buffer_gl1_inv
9967; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9968; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
9969; GFX11-WGP-NEXT:    s_endpgm
9970;
9971; GFX11-CU-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
9972; GFX11-CU:       ; %bb.0: ; %entry
9973; GFX11-CU-NEXT:    s_clause 0x1
9974; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9975; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
9976; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9977; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9978; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9979; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9980; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9981; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9982; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
9983; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9984; GFX11-CU-NEXT:    buffer_gl0_inv
9985; GFX11-CU-NEXT:    buffer_gl1_inv
9986; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9987; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
9988; GFX11-CU-NEXT:    s_endpgm
9989    i32* %out, i32 %in) {
9990entry:
9991  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") seq_cst
9992  store i32 %val, i32* %out, align 4
9993  ret void
9994}
9995
9996define amdgpu_kernel void @flat_system_one_as_monotonic_monotonic_cmpxchg(
9997; GFX7-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
9998; GFX7:       ; %bb.0: ; %entry
9999; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10000; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10001; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10002; GFX7-NEXT:    s_add_u32 s0, s0, 16
10003; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10004; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10005; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10006; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10007; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10008; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10009; GFX7-NEXT:    s_endpgm
10010;
10011; GFX10-WGP-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
10012; GFX10-WGP:       ; %bb.0: ; %entry
10013; GFX10-WGP-NEXT:    s_clause 0x1
10014; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10015; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10016; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10017; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10018; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10019; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10020; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10021; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10022; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10023; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10024; GFX10-WGP-NEXT:    s_endpgm
10025;
10026; GFX10-CU-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
10027; GFX10-CU:       ; %bb.0: ; %entry
10028; GFX10-CU-NEXT:    s_clause 0x1
10029; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10030; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10031; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10032; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10033; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10034; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10035; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10036; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10037; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10038; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10039; GFX10-CU-NEXT:    s_endpgm
10040;
10041; SKIP-CACHE-INV-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
10042; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10043; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10044; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10045; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10046; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10047; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10048; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10049; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10050; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10051; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10052; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10053; SKIP-CACHE-INV-NEXT:    s_endpgm
10054;
10055; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
10056; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10057; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10058; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10059; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10060; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10061; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10062; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10063; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10064;
10065; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
10066; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10067; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10068; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10069; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10070; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10071; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10072; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10073; GFX90A-TGSPLIT-NEXT:    s_endpgm
10074;
10075; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
10076; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10077; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10078; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10079; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10080; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10081; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10082; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
10083; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10084;
10085; GFX940-TGSPLIT-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
10086; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10087; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10088; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10089; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10090; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10091; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10092; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
10093; GFX940-TGSPLIT-NEXT:    s_endpgm
10094;
10095; GFX11-WGP-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
10096; GFX11-WGP:       ; %bb.0: ; %entry
10097; GFX11-WGP-NEXT:    s_clause 0x1
10098; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10099; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10100; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10101; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10102; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
10103; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10104; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
10105; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10106; GFX11-WGP-NEXT:    s_endpgm
10107;
10108; GFX11-CU-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
10109; GFX11-CU:       ; %bb.0: ; %entry
10110; GFX11-CU-NEXT:    s_clause 0x1
10111; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10112; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10113; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10114; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10115; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
10116; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10117; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
10118; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10119; GFX11-CU-NEXT:    s_endpgm
10120    i32* %out, i32 %in, i32 %old) {
10121entry:
10122  %gep = getelementptr i32, i32* %out, i32 4
10123  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") monotonic monotonic
10124  ret void
10125}
10126
10127define amdgpu_kernel void @flat_system_one_as_acquire_monotonic_cmpxchg(
10128; GFX7-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
10129; GFX7:       ; %bb.0: ; %entry
10130; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10131; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10132; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10133; GFX7-NEXT:    s_add_u32 s0, s0, 16
10134; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10135; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10136; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10137; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10138; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10139; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10140; GFX7-NEXT:    s_waitcnt vmcnt(0)
10141; GFX7-NEXT:    buffer_wbinvl1_vol
10142; GFX7-NEXT:    s_endpgm
10143;
10144; GFX10-WGP-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
10145; GFX10-WGP:       ; %bb.0: ; %entry
10146; GFX10-WGP-NEXT:    s_clause 0x1
10147; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10148; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10149; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10150; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10151; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10152; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10153; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10154; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10155; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10156; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10157; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10158; GFX10-WGP-NEXT:    buffer_gl0_inv
10159; GFX10-WGP-NEXT:    buffer_gl1_inv
10160; GFX10-WGP-NEXT:    s_endpgm
10161;
10162; GFX10-CU-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
10163; GFX10-CU:       ; %bb.0: ; %entry
10164; GFX10-CU-NEXT:    s_clause 0x1
10165; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10166; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10167; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10168; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10169; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10170; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10171; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10172; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10173; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10174; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10175; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10176; GFX10-CU-NEXT:    buffer_gl0_inv
10177; GFX10-CU-NEXT:    buffer_gl1_inv
10178; GFX10-CU-NEXT:    s_endpgm
10179;
10180; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
10181; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10182; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10183; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10184; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10185; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10186; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10187; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10188; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10189; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10190; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10191; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10192; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10193; SKIP-CACHE-INV-NEXT:    s_endpgm
10194;
10195; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
10196; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10197; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10198; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10199; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10200; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10201; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10202; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10203; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10204; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
10205; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
10206; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10207;
10208; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
10209; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10210; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10211; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10212; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10213; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10214; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10215; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10216; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10217; GFX90A-TGSPLIT-NEXT:    buffer_invl2
10218; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10219; GFX90A-TGSPLIT-NEXT:    s_endpgm
10220;
10221; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
10222; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10223; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10224; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10225; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10226; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10227; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10228; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
10229; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10230; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
10231; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10232;
10233; GFX940-TGSPLIT-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
10234; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10235; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10236; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10237; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10238; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10239; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10240; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
10241; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10242; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
10243; GFX940-TGSPLIT-NEXT:    s_endpgm
10244;
10245; GFX11-WGP-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
10246; GFX11-WGP:       ; %bb.0: ; %entry
10247; GFX11-WGP-NEXT:    s_clause 0x1
10248; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10249; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10250; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10251; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10252; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
10253; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10254; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
10255; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10256; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10257; GFX11-WGP-NEXT:    buffer_gl0_inv
10258; GFX11-WGP-NEXT:    buffer_gl1_inv
10259; GFX11-WGP-NEXT:    s_endpgm
10260;
10261; GFX11-CU-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
10262; GFX11-CU:       ; %bb.0: ; %entry
10263; GFX11-CU-NEXT:    s_clause 0x1
10264; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10265; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10266; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10267; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10268; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
10269; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10270; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
10271; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10272; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10273; GFX11-CU-NEXT:    buffer_gl0_inv
10274; GFX11-CU-NEXT:    buffer_gl1_inv
10275; GFX11-CU-NEXT:    s_endpgm
10276    i32* %out, i32 %in, i32 %old) {
10277entry:
10278  %gep = getelementptr i32, i32* %out, i32 4
10279  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acquire monotonic
10280  ret void
10281}
10282
10283define amdgpu_kernel void @flat_system_one_as_release_monotonic_cmpxchg(
10284; GFX7-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
10285; GFX7:       ; %bb.0: ; %entry
10286; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10287; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10288; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10289; GFX7-NEXT:    s_add_u32 s0, s0, 16
10290; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10291; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10292; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10293; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10294; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10295; GFX7-NEXT:    s_waitcnt vmcnt(0)
10296; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10297; GFX7-NEXT:    s_endpgm
10298;
10299; GFX10-WGP-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
10300; GFX10-WGP:       ; %bb.0: ; %entry
10301; GFX10-WGP-NEXT:    s_clause 0x1
10302; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10303; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10304; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10305; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10306; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10307; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10308; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10309; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10310; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10311; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10312; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10313; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10314; GFX10-WGP-NEXT:    s_endpgm
10315;
10316; GFX10-CU-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
10317; GFX10-CU:       ; %bb.0: ; %entry
10318; GFX10-CU-NEXT:    s_clause 0x1
10319; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10320; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10321; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10322; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10323; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10324; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10325; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10326; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10327; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10328; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10329; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10330; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10331; GFX10-CU-NEXT:    s_endpgm
10332;
10333; SKIP-CACHE-INV-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
10334; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10335; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10336; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10337; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10338; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10339; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10340; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10341; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10342; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10343; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10344; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10345; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10346; SKIP-CACHE-INV-NEXT:    s_endpgm
10347;
10348; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
10349; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10350; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10351; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10352; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10353; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10354; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10355; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
10356; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10357; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10358; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10359;
10360; GFX90A-TGSPLIT-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
10361; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10362; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10363; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10364; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10365; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10366; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10367; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
10368; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10369; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10370; GFX90A-TGSPLIT-NEXT:    s_endpgm
10371;
10372; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
10373; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10374; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10375; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10376; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10377; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10378; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10379; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
10380; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10381; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
10382; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10383;
10384; GFX940-TGSPLIT-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
10385; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10386; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10387; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10388; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10389; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10390; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10391; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
10392; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10393; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
10394; GFX940-TGSPLIT-NEXT:    s_endpgm
10395;
10396; GFX11-WGP-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
10397; GFX11-WGP:       ; %bb.0: ; %entry
10398; GFX11-WGP-NEXT:    s_clause 0x1
10399; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10400; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10401; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10402; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10403; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
10404; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10405; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
10406; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
10407; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10408; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10409; GFX11-WGP-NEXT:    s_endpgm
10410;
10411; GFX11-CU-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
10412; GFX11-CU:       ; %bb.0: ; %entry
10413; GFX11-CU-NEXT:    s_clause 0x1
10414; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10415; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10416; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10417; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10418; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
10419; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10420; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
10421; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
10422; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10423; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10424; GFX11-CU-NEXT:    s_endpgm
10425    i32* %out, i32 %in, i32 %old) {
10426entry:
10427  %gep = getelementptr i32, i32* %out, i32 4
10428  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") release monotonic
10429  ret void
10430}
10431
10432define amdgpu_kernel void @flat_system_one_as_acq_rel_monotonic_cmpxchg(
10433; GFX7-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
10434; GFX7:       ; %bb.0: ; %entry
10435; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10436; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10437; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10438; GFX7-NEXT:    s_add_u32 s0, s0, 16
10439; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10440; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10441; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10442; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10443; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10444; GFX7-NEXT:    s_waitcnt vmcnt(0)
10445; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10446; GFX7-NEXT:    s_waitcnt vmcnt(0)
10447; GFX7-NEXT:    buffer_wbinvl1_vol
10448; GFX7-NEXT:    s_endpgm
10449;
10450; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
10451; GFX10-WGP:       ; %bb.0: ; %entry
10452; GFX10-WGP-NEXT:    s_clause 0x1
10453; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10454; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10455; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10456; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10457; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10458; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10459; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10460; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10461; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10462; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10463; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10464; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10465; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10466; GFX10-WGP-NEXT:    buffer_gl0_inv
10467; GFX10-WGP-NEXT:    buffer_gl1_inv
10468; GFX10-WGP-NEXT:    s_endpgm
10469;
10470; GFX10-CU-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
10471; GFX10-CU:       ; %bb.0: ; %entry
10472; GFX10-CU-NEXT:    s_clause 0x1
10473; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10474; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10475; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10476; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10477; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10478; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10479; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10480; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10481; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10482; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10483; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10484; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10485; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10486; GFX10-CU-NEXT:    buffer_gl0_inv
10487; GFX10-CU-NEXT:    buffer_gl1_inv
10488; GFX10-CU-NEXT:    s_endpgm
10489;
10490; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
10491; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10492; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10493; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10494; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10495; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10496; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10497; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10498; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10499; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10500; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10501; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10502; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10503; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10504; SKIP-CACHE-INV-NEXT:    s_endpgm
10505;
10506; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
10507; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10508; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10509; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10510; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10511; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10512; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10513; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
10514; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10515; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10516; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10517; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
10518; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
10519; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10520;
10521; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
10522; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10523; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10524; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10525; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10526; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10527; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10528; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
10529; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10530; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10531; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10532; GFX90A-TGSPLIT-NEXT:    buffer_invl2
10533; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10534; GFX90A-TGSPLIT-NEXT:    s_endpgm
10535;
10536; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
10537; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10538; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10539; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10540; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10541; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10542; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10543; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
10544; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10545; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
10546; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10547; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
10548; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10549;
10550; GFX940-TGSPLIT-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
10551; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10552; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10553; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10554; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10555; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10556; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10557; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
10558; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10559; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
10560; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10561; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
10562; GFX940-TGSPLIT-NEXT:    s_endpgm
10563;
10564; GFX11-WGP-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
10565; GFX11-WGP:       ; %bb.0: ; %entry
10566; GFX11-WGP-NEXT:    s_clause 0x1
10567; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10568; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10569; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10570; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10571; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
10572; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10573; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
10574; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
10575; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10576; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10577; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10578; GFX11-WGP-NEXT:    buffer_gl0_inv
10579; GFX11-WGP-NEXT:    buffer_gl1_inv
10580; GFX11-WGP-NEXT:    s_endpgm
10581;
10582; GFX11-CU-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
10583; GFX11-CU:       ; %bb.0: ; %entry
10584; GFX11-CU-NEXT:    s_clause 0x1
10585; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10586; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10587; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10588; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10589; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
10590; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10591; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
10592; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
10593; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10594; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10595; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10596; GFX11-CU-NEXT:    buffer_gl0_inv
10597; GFX11-CU-NEXT:    buffer_gl1_inv
10598; GFX11-CU-NEXT:    s_endpgm
10599    i32* %out, i32 %in, i32 %old) {
10600entry:
10601  %gep = getelementptr i32, i32* %out, i32 4
10602  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel monotonic
10603  ret void
10604}
10605
10606define amdgpu_kernel void @flat_system_one_as_seq_cst_monotonic_cmpxchg(
10607; GFX7-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
10608; GFX7:       ; %bb.0: ; %entry
10609; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10610; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10611; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10612; GFX7-NEXT:    s_add_u32 s0, s0, 16
10613; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10614; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10615; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10616; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10617; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10618; GFX7-NEXT:    s_waitcnt vmcnt(0)
10619; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10620; GFX7-NEXT:    s_waitcnt vmcnt(0)
10621; GFX7-NEXT:    buffer_wbinvl1_vol
10622; GFX7-NEXT:    s_endpgm
10623;
10624; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
10625; GFX10-WGP:       ; %bb.0: ; %entry
10626; GFX10-WGP-NEXT:    s_clause 0x1
10627; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10628; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10629; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10630; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10631; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10632; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10633; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10634; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10635; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10636; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10637; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10638; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10639; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10640; GFX10-WGP-NEXT:    buffer_gl0_inv
10641; GFX10-WGP-NEXT:    buffer_gl1_inv
10642; GFX10-WGP-NEXT:    s_endpgm
10643;
10644; GFX10-CU-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
10645; GFX10-CU:       ; %bb.0: ; %entry
10646; GFX10-CU-NEXT:    s_clause 0x1
10647; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10648; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10649; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10650; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10651; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10652; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10653; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10654; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10655; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10656; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10657; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10658; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10659; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10660; GFX10-CU-NEXT:    buffer_gl0_inv
10661; GFX10-CU-NEXT:    buffer_gl1_inv
10662; GFX10-CU-NEXT:    s_endpgm
10663;
10664; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
10665; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10666; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10667; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10668; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10669; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10670; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10671; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10672; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10673; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10674; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10675; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10676; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10677; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10678; SKIP-CACHE-INV-NEXT:    s_endpgm
10679;
10680; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
10681; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10682; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10683; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10684; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10685; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10686; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10687; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
10688; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10689; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10690; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10691; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
10692; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
10693; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10694;
10695; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
10696; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10697; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10698; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10699; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10700; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10701; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10702; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
10703; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10704; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10705; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10706; GFX90A-TGSPLIT-NEXT:    buffer_invl2
10707; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10708; GFX90A-TGSPLIT-NEXT:    s_endpgm
10709;
10710; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
10711; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10712; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10713; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10714; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10715; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10716; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10717; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
10718; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10719; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
10720; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10721; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
10722; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10723;
10724; GFX940-TGSPLIT-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
10725; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10726; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10727; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10728; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10729; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10730; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10731; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
10732; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10733; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
10734; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10735; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
10736; GFX940-TGSPLIT-NEXT:    s_endpgm
10737;
10738; GFX11-WGP-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
10739; GFX11-WGP:       ; %bb.0: ; %entry
10740; GFX11-WGP-NEXT:    s_clause 0x1
10741; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10742; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10743; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10744; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10745; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
10746; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10747; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
10748; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
10749; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10750; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10751; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10752; GFX11-WGP-NEXT:    buffer_gl0_inv
10753; GFX11-WGP-NEXT:    buffer_gl1_inv
10754; GFX11-WGP-NEXT:    s_endpgm
10755;
10756; GFX11-CU-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
10757; GFX11-CU:       ; %bb.0: ; %entry
10758; GFX11-CU-NEXT:    s_clause 0x1
10759; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10760; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10761; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10762; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10763; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
10764; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10765; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
10766; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
10767; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10768; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10769; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10770; GFX11-CU-NEXT:    buffer_gl0_inv
10771; GFX11-CU-NEXT:    buffer_gl1_inv
10772; GFX11-CU-NEXT:    s_endpgm
10773    i32* %out, i32 %in, i32 %old) {
10774entry:
10775  %gep = getelementptr i32, i32* %out, i32 4
10776  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst monotonic
10777  ret void
10778}
10779
10780define amdgpu_kernel void @flat_system_one_as_monotonic_acquire_cmpxchg(
10781; GFX7-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
10782; GFX7:       ; %bb.0: ; %entry
10783; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10784; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10785; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10786; GFX7-NEXT:    s_add_u32 s0, s0, 16
10787; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10788; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10789; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10790; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10791; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10792; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10793; GFX7-NEXT:    s_waitcnt vmcnt(0)
10794; GFX7-NEXT:    buffer_wbinvl1_vol
10795; GFX7-NEXT:    s_endpgm
10796;
10797; GFX10-WGP-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
10798; GFX10-WGP:       ; %bb.0: ; %entry
10799; GFX10-WGP-NEXT:    s_clause 0x1
10800; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10801; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10802; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10803; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10804; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10805; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10806; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10807; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10808; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10809; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10810; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10811; GFX10-WGP-NEXT:    buffer_gl0_inv
10812; GFX10-WGP-NEXT:    buffer_gl1_inv
10813; GFX10-WGP-NEXT:    s_endpgm
10814;
10815; GFX10-CU-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
10816; GFX10-CU:       ; %bb.0: ; %entry
10817; GFX10-CU-NEXT:    s_clause 0x1
10818; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10819; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10820; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10821; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10822; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10823; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10824; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10825; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10826; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10827; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10828; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10829; GFX10-CU-NEXT:    buffer_gl0_inv
10830; GFX10-CU-NEXT:    buffer_gl1_inv
10831; GFX10-CU-NEXT:    s_endpgm
10832;
10833; SKIP-CACHE-INV-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
10834; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10835; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10836; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10837; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10838; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10839; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10840; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10841; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10842; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10843; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10844; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10845; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10846; SKIP-CACHE-INV-NEXT:    s_endpgm
10847;
10848; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
10849; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10850; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10851; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10852; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10853; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10854; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10855; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10856; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10857; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
10858; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
10859; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10860;
10861; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
10862; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10863; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10864; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10865; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10866; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10867; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10868; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10869; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10870; GFX90A-TGSPLIT-NEXT:    buffer_invl2
10871; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10872; GFX90A-TGSPLIT-NEXT:    s_endpgm
10873;
10874; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
10875; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10876; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10877; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10878; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10879; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10880; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10881; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
10882; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10883; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
10884; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10885;
10886; GFX940-TGSPLIT-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
10887; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10888; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10889; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10890; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10891; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10892; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10893; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
10894; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10895; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
10896; GFX940-TGSPLIT-NEXT:    s_endpgm
10897;
10898; GFX11-WGP-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
10899; GFX11-WGP:       ; %bb.0: ; %entry
10900; GFX11-WGP-NEXT:    s_clause 0x1
10901; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10902; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10903; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10904; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10905; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
10906; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10907; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
10908; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10909; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10910; GFX11-WGP-NEXT:    buffer_gl0_inv
10911; GFX11-WGP-NEXT:    buffer_gl1_inv
10912; GFX11-WGP-NEXT:    s_endpgm
10913;
10914; GFX11-CU-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
10915; GFX11-CU:       ; %bb.0: ; %entry
10916; GFX11-CU-NEXT:    s_clause 0x1
10917; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10918; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10919; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10920; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10921; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
10922; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10923; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
10924; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10925; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10926; GFX11-CU-NEXT:    buffer_gl0_inv
10927; GFX11-CU-NEXT:    buffer_gl1_inv
10928; GFX11-CU-NEXT:    s_endpgm
10929    i32* %out, i32 %in, i32 %old) {
10930entry:
10931  %gep = getelementptr i32, i32* %out, i32 4
10932  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") monotonic acquire
10933  ret void
10934}
10935
10936define amdgpu_kernel void @flat_system_one_as_acquire_acquire_cmpxchg(
10937; GFX7-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
10938; GFX7:       ; %bb.0: ; %entry
10939; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10940; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10941; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10942; GFX7-NEXT:    s_add_u32 s0, s0, 16
10943; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10944; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10945; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10946; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10947; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10948; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10949; GFX7-NEXT:    s_waitcnt vmcnt(0)
10950; GFX7-NEXT:    buffer_wbinvl1_vol
10951; GFX7-NEXT:    s_endpgm
10952;
10953; GFX10-WGP-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
10954; GFX10-WGP:       ; %bb.0: ; %entry
10955; GFX10-WGP-NEXT:    s_clause 0x1
10956; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10957; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10958; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10959; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10960; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10961; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10962; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10963; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10964; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10965; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10966; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10967; GFX10-WGP-NEXT:    buffer_gl0_inv
10968; GFX10-WGP-NEXT:    buffer_gl1_inv
10969; GFX10-WGP-NEXT:    s_endpgm
10970;
10971; GFX10-CU-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
10972; GFX10-CU:       ; %bb.0: ; %entry
10973; GFX10-CU-NEXT:    s_clause 0x1
10974; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10975; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10976; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10977; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10978; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10979; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10980; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10981; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10982; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10983; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10984; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10985; GFX10-CU-NEXT:    buffer_gl0_inv
10986; GFX10-CU-NEXT:    buffer_gl1_inv
10987; GFX10-CU-NEXT:    s_endpgm
10988;
10989; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
10990; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10991; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10992; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10993; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10994; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10995; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10996; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10997; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10998; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10999; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11000; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11001; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11002; SKIP-CACHE-INV-NEXT:    s_endpgm
11003;
11004; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
11005; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11006; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11007; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11008; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11009; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11010; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11011; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11012; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11013; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
11014; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
11015; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11016;
11017; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
11018; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11019; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11020; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11021; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11022; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11023; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11024; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11025; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11026; GFX90A-TGSPLIT-NEXT:    buffer_invl2
11027; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11028; GFX90A-TGSPLIT-NEXT:    s_endpgm
11029;
11030; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
11031; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11032; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11033; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11034; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11035; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11036; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11037; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
11038; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11039; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
11040; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11041;
11042; GFX940-TGSPLIT-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
11043; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11044; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11045; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11046; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11047; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11048; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11049; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
11050; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11051; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
11052; GFX940-TGSPLIT-NEXT:    s_endpgm
11053;
11054; GFX11-WGP-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
11055; GFX11-WGP:       ; %bb.0: ; %entry
11056; GFX11-WGP-NEXT:    s_clause 0x1
11057; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11058; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11059; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11060; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
11061; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
11062; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
11063; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
11064; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11065; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11066; GFX11-WGP-NEXT:    buffer_gl0_inv
11067; GFX11-WGP-NEXT:    buffer_gl1_inv
11068; GFX11-WGP-NEXT:    s_endpgm
11069;
11070; GFX11-CU-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
11071; GFX11-CU:       ; %bb.0: ; %entry
11072; GFX11-CU-NEXT:    s_clause 0x1
11073; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11074; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11075; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11076; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
11077; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
11078; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
11079; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
11080; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11081; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11082; GFX11-CU-NEXT:    buffer_gl0_inv
11083; GFX11-CU-NEXT:    buffer_gl1_inv
11084; GFX11-CU-NEXT:    s_endpgm
11085    i32* %out, i32 %in, i32 %old) {
11086entry:
11087  %gep = getelementptr i32, i32* %out, i32 4
11088  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acquire acquire
11089  ret void
11090}
11091
11092define amdgpu_kernel void @flat_system_one_as_release_acquire_cmpxchg(
11093; GFX7-LABEL: flat_system_one_as_release_acquire_cmpxchg:
11094; GFX7:       ; %bb.0: ; %entry
11095; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11096; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11097; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11098; GFX7-NEXT:    s_add_u32 s0, s0, 16
11099; GFX7-NEXT:    s_addc_u32 s1, s1, 0
11100; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11101; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11102; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11103; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11104; GFX7-NEXT:    s_waitcnt vmcnt(0)
11105; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11106; GFX7-NEXT:    s_waitcnt vmcnt(0)
11107; GFX7-NEXT:    buffer_wbinvl1_vol
11108; GFX7-NEXT:    s_endpgm
11109;
11110; GFX10-WGP-LABEL: flat_system_one_as_release_acquire_cmpxchg:
11111; GFX10-WGP:       ; %bb.0: ; %entry
11112; GFX10-WGP-NEXT:    s_clause 0x1
11113; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11114; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11115; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11116; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
11117; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
11118; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11119; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11120; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11121; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11122; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11123; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11124; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11125; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11126; GFX10-WGP-NEXT:    buffer_gl0_inv
11127; GFX10-WGP-NEXT:    buffer_gl1_inv
11128; GFX10-WGP-NEXT:    s_endpgm
11129;
11130; GFX10-CU-LABEL: flat_system_one_as_release_acquire_cmpxchg:
11131; GFX10-CU:       ; %bb.0: ; %entry
11132; GFX10-CU-NEXT:    s_clause 0x1
11133; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11134; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11135; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11136; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
11137; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
11138; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11139; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11140; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11141; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11142; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11143; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11144; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11145; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11146; GFX10-CU-NEXT:    buffer_gl0_inv
11147; GFX10-CU-NEXT:    buffer_gl1_inv
11148; GFX10-CU-NEXT:    s_endpgm
11149;
11150; SKIP-CACHE-INV-LABEL: flat_system_one_as_release_acquire_cmpxchg:
11151; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11152; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11153; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11154; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11155; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
11156; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
11157; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11158; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11159; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11160; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11161; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11162; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11163; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11164; SKIP-CACHE-INV-NEXT:    s_endpgm
11165;
11166; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_release_acquire_cmpxchg:
11167; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11168; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11169; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11170; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11171; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11172; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11173; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
11174; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11175; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11176; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11177; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
11178; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
11179; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11180;
11181; GFX90A-TGSPLIT-LABEL: flat_system_one_as_release_acquire_cmpxchg:
11182; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11183; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11184; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11185; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11186; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11187; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11188; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
11189; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11190; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11191; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11192; GFX90A-TGSPLIT-NEXT:    buffer_invl2
11193; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11194; GFX90A-TGSPLIT-NEXT:    s_endpgm
11195;
11196; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_release_acquire_cmpxchg:
11197; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11198; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11199; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11200; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11201; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11202; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11203; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
11204; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11205; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
11206; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11207; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
11208; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11209;
11210; GFX940-TGSPLIT-LABEL: flat_system_one_as_release_acquire_cmpxchg:
11211; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11212; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11213; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11214; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11215; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11216; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11217; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
11218; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11219; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
11220; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11221; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
11222; GFX940-TGSPLIT-NEXT:    s_endpgm
11223;
11224; GFX11-WGP-LABEL: flat_system_one_as_release_acquire_cmpxchg:
11225; GFX11-WGP:       ; %bb.0: ; %entry
11226; GFX11-WGP-NEXT:    s_clause 0x1
11227; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11228; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11229; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11230; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
11231; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
11232; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
11233; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
11234; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11235; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11236; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11237; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11238; GFX11-WGP-NEXT:    buffer_gl0_inv
11239; GFX11-WGP-NEXT:    buffer_gl1_inv
11240; GFX11-WGP-NEXT:    s_endpgm
11241;
11242; GFX11-CU-LABEL: flat_system_one_as_release_acquire_cmpxchg:
11243; GFX11-CU:       ; %bb.0: ; %entry
11244; GFX11-CU-NEXT:    s_clause 0x1
11245; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11246; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11247; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11248; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
11249; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
11250; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
11251; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
11252; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11253; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11254; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11255; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11256; GFX11-CU-NEXT:    buffer_gl0_inv
11257; GFX11-CU-NEXT:    buffer_gl1_inv
11258; GFX11-CU-NEXT:    s_endpgm
11259    i32* %out, i32 %in, i32 %old) {
11260entry:
11261  %gep = getelementptr i32, i32* %out, i32 4
11262  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") release acquire
11263  ret void
11264}
11265
11266define amdgpu_kernel void @flat_system_one_as_acq_rel_acquire_cmpxchg(
11267; GFX7-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
11268; GFX7:       ; %bb.0: ; %entry
11269; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11270; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11271; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11272; GFX7-NEXT:    s_add_u32 s0, s0, 16
11273; GFX7-NEXT:    s_addc_u32 s1, s1, 0
11274; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11275; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11276; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11277; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11278; GFX7-NEXT:    s_waitcnt vmcnt(0)
11279; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11280; GFX7-NEXT:    s_waitcnt vmcnt(0)
11281; GFX7-NEXT:    buffer_wbinvl1_vol
11282; GFX7-NEXT:    s_endpgm
11283;
11284; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
11285; GFX10-WGP:       ; %bb.0: ; %entry
11286; GFX10-WGP-NEXT:    s_clause 0x1
11287; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11288; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11289; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11290; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
11291; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
11292; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11293; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11294; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11295; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11296; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11297; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11298; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11299; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11300; GFX10-WGP-NEXT:    buffer_gl0_inv
11301; GFX10-WGP-NEXT:    buffer_gl1_inv
11302; GFX10-WGP-NEXT:    s_endpgm
11303;
11304; GFX10-CU-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
11305; GFX10-CU:       ; %bb.0: ; %entry
11306; GFX10-CU-NEXT:    s_clause 0x1
11307; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11308; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11309; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11310; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
11311; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
11312; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11313; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11314; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11315; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11316; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11317; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11318; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11319; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11320; GFX10-CU-NEXT:    buffer_gl0_inv
11321; GFX10-CU-NEXT:    buffer_gl1_inv
11322; GFX10-CU-NEXT:    s_endpgm
11323;
11324; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
11325; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11326; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11327; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11328; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11329; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
11330; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
11331; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11332; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11333; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11334; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11335; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11336; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11337; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11338; SKIP-CACHE-INV-NEXT:    s_endpgm
11339;
11340; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
11341; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11342; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11343; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11344; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11345; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11346; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11347; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
11348; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11349; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11350; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11351; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
11352; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
11353; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11354;
11355; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
11356; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11357; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11358; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11359; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11360; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11361; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11362; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
11363; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11364; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11365; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11366; GFX90A-TGSPLIT-NEXT:    buffer_invl2
11367; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11368; GFX90A-TGSPLIT-NEXT:    s_endpgm
11369;
11370; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
11371; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11372; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11373; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11374; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11375; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11376; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11377; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
11378; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11379; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
11380; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11381; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
11382; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11383;
11384; GFX940-TGSPLIT-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
11385; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11386; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11387; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11388; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11389; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11390; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11391; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
11392; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11393; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
11394; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11395; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
11396; GFX940-TGSPLIT-NEXT:    s_endpgm
11397;
11398; GFX11-WGP-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
11399; GFX11-WGP:       ; %bb.0: ; %entry
11400; GFX11-WGP-NEXT:    s_clause 0x1
11401; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11402; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11403; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11404; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
11405; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
11406; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
11407; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
11408; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11409; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11410; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11411; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11412; GFX11-WGP-NEXT:    buffer_gl0_inv
11413; GFX11-WGP-NEXT:    buffer_gl1_inv
11414; GFX11-WGP-NEXT:    s_endpgm
11415;
11416; GFX11-CU-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
11417; GFX11-CU:       ; %bb.0: ; %entry
11418; GFX11-CU-NEXT:    s_clause 0x1
11419; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11420; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11421; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11422; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
11423; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
11424; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
11425; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
11426; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11427; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11428; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11429; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11430; GFX11-CU-NEXT:    buffer_gl0_inv
11431; GFX11-CU-NEXT:    buffer_gl1_inv
11432; GFX11-CU-NEXT:    s_endpgm
11433    i32* %out, i32 %in, i32 %old) {
11434entry:
11435  %gep = getelementptr i32, i32* %out, i32 4
11436  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel acquire
11437  ret void
11438}
11439
11440define amdgpu_kernel void @flat_system_one_as_seq_cst_acquire_cmpxchg(
11441; GFX7-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
11442; GFX7:       ; %bb.0: ; %entry
11443; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11444; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11445; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11446; GFX7-NEXT:    s_add_u32 s0, s0, 16
11447; GFX7-NEXT:    s_addc_u32 s1, s1, 0
11448; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11449; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11450; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11451; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11452; GFX7-NEXT:    s_waitcnt vmcnt(0)
11453; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11454; GFX7-NEXT:    s_waitcnt vmcnt(0)
11455; GFX7-NEXT:    buffer_wbinvl1_vol
11456; GFX7-NEXT:    s_endpgm
11457;
11458; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
11459; GFX10-WGP:       ; %bb.0: ; %entry
11460; GFX10-WGP-NEXT:    s_clause 0x1
11461; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11462; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11463; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11464; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
11465; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
11466; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11467; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11468; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11469; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11470; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11471; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11472; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11473; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11474; GFX10-WGP-NEXT:    buffer_gl0_inv
11475; GFX10-WGP-NEXT:    buffer_gl1_inv
11476; GFX10-WGP-NEXT:    s_endpgm
11477;
11478; GFX10-CU-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
11479; GFX10-CU:       ; %bb.0: ; %entry
11480; GFX10-CU-NEXT:    s_clause 0x1
11481; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11482; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11483; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11484; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
11485; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
11486; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11487; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11488; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11489; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11490; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11491; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11492; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11493; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11494; GFX10-CU-NEXT:    buffer_gl0_inv
11495; GFX10-CU-NEXT:    buffer_gl1_inv
11496; GFX10-CU-NEXT:    s_endpgm
11497;
11498; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
11499; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11500; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11501; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11502; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11503; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
11504; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
11505; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11506; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11507; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11508; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11509; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11510; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11511; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11512; SKIP-CACHE-INV-NEXT:    s_endpgm
11513;
11514; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
11515; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11516; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11517; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11518; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11519; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11520; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11521; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
11522; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11523; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11524; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11525; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
11526; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
11527; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11528;
11529; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
11530; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11531; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11532; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11533; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11534; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11535; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11536; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
11537; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11538; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11539; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11540; GFX90A-TGSPLIT-NEXT:    buffer_invl2
11541; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11542; GFX90A-TGSPLIT-NEXT:    s_endpgm
11543;
11544; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
11545; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11546; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11547; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11548; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11549; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11550; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11551; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
11552; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11553; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
11554; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11555; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
11556; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11557;
11558; GFX940-TGSPLIT-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
11559; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11560; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11561; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11562; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11563; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11564; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11565; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
11566; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11567; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
11568; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11569; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
11570; GFX940-TGSPLIT-NEXT:    s_endpgm
11571;
11572; GFX11-WGP-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
11573; GFX11-WGP:       ; %bb.0: ; %entry
11574; GFX11-WGP-NEXT:    s_clause 0x1
11575; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11576; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11577; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11578; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
11579; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
11580; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
11581; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
11582; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11583; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11584; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11585; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11586; GFX11-WGP-NEXT:    buffer_gl0_inv
11587; GFX11-WGP-NEXT:    buffer_gl1_inv
11588; GFX11-WGP-NEXT:    s_endpgm
11589;
11590; GFX11-CU-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
11591; GFX11-CU:       ; %bb.0: ; %entry
11592; GFX11-CU-NEXT:    s_clause 0x1
11593; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11594; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11595; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11596; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
11597; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
11598; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
11599; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
11600; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11601; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11602; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11603; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11604; GFX11-CU-NEXT:    buffer_gl0_inv
11605; GFX11-CU-NEXT:    buffer_gl1_inv
11606; GFX11-CU-NEXT:    s_endpgm
11607    i32* %out, i32 %in, i32 %old) {
11608entry:
11609  %gep = getelementptr i32, i32* %out, i32 4
11610  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst acquire
11611  ret void
11612}
11613
11614define amdgpu_kernel void @flat_system_one_as_monotonic_seq_cst_cmpxchg(
11615; GFX7-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
11616; GFX7:       ; %bb.0: ; %entry
11617; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11618; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11619; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11620; GFX7-NEXT:    s_add_u32 s0, s0, 16
11621; GFX7-NEXT:    s_addc_u32 s1, s1, 0
11622; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11623; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11624; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11625; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11626; GFX7-NEXT:    s_waitcnt vmcnt(0)
11627; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11628; GFX7-NEXT:    s_waitcnt vmcnt(0)
11629; GFX7-NEXT:    buffer_wbinvl1_vol
11630; GFX7-NEXT:    s_endpgm
11631;
11632; GFX10-WGP-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
11633; GFX10-WGP:       ; %bb.0: ; %entry
11634; GFX10-WGP-NEXT:    s_clause 0x1
11635; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11636; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11637; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11638; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
11639; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
11640; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11641; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11642; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11643; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11644; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11645; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11646; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11647; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11648; GFX10-WGP-NEXT:    buffer_gl0_inv
11649; GFX10-WGP-NEXT:    buffer_gl1_inv
11650; GFX10-WGP-NEXT:    s_endpgm
11651;
11652; GFX10-CU-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
11653; GFX10-CU:       ; %bb.0: ; %entry
11654; GFX10-CU-NEXT:    s_clause 0x1
11655; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11656; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11657; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11658; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
11659; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
11660; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11661; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11662; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11663; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11664; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11665; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11666; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11667; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11668; GFX10-CU-NEXT:    buffer_gl0_inv
11669; GFX10-CU-NEXT:    buffer_gl1_inv
11670; GFX10-CU-NEXT:    s_endpgm
11671;
11672; SKIP-CACHE-INV-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
11673; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11674; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11675; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11676; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11677; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
11678; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
11679; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11680; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11681; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11682; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11683; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11684; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11685; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11686; SKIP-CACHE-INV-NEXT:    s_endpgm
11687;
11688; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
11689; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11690; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11691; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11692; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11693; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11694; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11695; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
11696; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11697; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11698; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11699; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
11700; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
11701; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11702;
11703; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
11704; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11705; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11706; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11707; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11708; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11709; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11710; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
11711; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11712; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11713; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11714; GFX90A-TGSPLIT-NEXT:    buffer_invl2
11715; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11716; GFX90A-TGSPLIT-NEXT:    s_endpgm
11717;
11718; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
11719; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11720; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11721; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11722; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11723; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11724; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11725; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
11726; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11727; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
11728; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11729; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
11730; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11731;
11732; GFX940-TGSPLIT-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
11733; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11734; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11735; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11736; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11737; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11738; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11739; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
11740; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11741; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
11742; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11743; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
11744; GFX940-TGSPLIT-NEXT:    s_endpgm
11745;
11746; GFX11-WGP-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
11747; GFX11-WGP:       ; %bb.0: ; %entry
11748; GFX11-WGP-NEXT:    s_clause 0x1
11749; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11750; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11751; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11752; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
11753; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
11754; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
11755; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
11756; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11757; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11758; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11759; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11760; GFX11-WGP-NEXT:    buffer_gl0_inv
11761; GFX11-WGP-NEXT:    buffer_gl1_inv
11762; GFX11-WGP-NEXT:    s_endpgm
11763;
11764; GFX11-CU-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
11765; GFX11-CU:       ; %bb.0: ; %entry
11766; GFX11-CU-NEXT:    s_clause 0x1
11767; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11768; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11769; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11770; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
11771; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
11772; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
11773; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
11774; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11775; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11776; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11777; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11778; GFX11-CU-NEXT:    buffer_gl0_inv
11779; GFX11-CU-NEXT:    buffer_gl1_inv
11780; GFX11-CU-NEXT:    s_endpgm
11781    i32* %out, i32 %in, i32 %old) {
11782entry:
11783  %gep = getelementptr i32, i32* %out, i32 4
11784  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") monotonic seq_cst
11785  ret void
11786}
11787
11788define amdgpu_kernel void @flat_system_one_as_acquire_seq_cst_cmpxchg(
11789; GFX7-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
11790; GFX7:       ; %bb.0: ; %entry
11791; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11792; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11793; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11794; GFX7-NEXT:    s_add_u32 s0, s0, 16
11795; GFX7-NEXT:    s_addc_u32 s1, s1, 0
11796; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11797; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11798; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11799; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11800; GFX7-NEXT:    s_waitcnt vmcnt(0)
11801; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11802; GFX7-NEXT:    s_waitcnt vmcnt(0)
11803; GFX7-NEXT:    buffer_wbinvl1_vol
11804; GFX7-NEXT:    s_endpgm
11805;
11806; GFX10-WGP-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
11807; GFX10-WGP:       ; %bb.0: ; %entry
11808; GFX10-WGP-NEXT:    s_clause 0x1
11809; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11810; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11811; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11812; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
11813; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
11814; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11815; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11816; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11817; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11818; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11819; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11820; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11821; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11822; GFX10-WGP-NEXT:    buffer_gl0_inv
11823; GFX10-WGP-NEXT:    buffer_gl1_inv
11824; GFX10-WGP-NEXT:    s_endpgm
11825;
11826; GFX10-CU-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
11827; GFX10-CU:       ; %bb.0: ; %entry
11828; GFX10-CU-NEXT:    s_clause 0x1
11829; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11830; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11831; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11832; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
11833; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
11834; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11835; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11836; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11837; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11838; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11839; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11840; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11841; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11842; GFX10-CU-NEXT:    buffer_gl0_inv
11843; GFX10-CU-NEXT:    buffer_gl1_inv
11844; GFX10-CU-NEXT:    s_endpgm
11845;
11846; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
11847; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11848; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11849; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11850; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11851; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
11852; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
11853; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11854; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11855; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11856; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11857; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11858; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11859; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11860; SKIP-CACHE-INV-NEXT:    s_endpgm
11861;
11862; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
11863; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11864; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11865; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11866; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11867; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11868; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11869; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
11870; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11871; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11872; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11873; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
11874; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
11875; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11876;
11877; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
11878; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11879; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11880; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11881; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11882; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11883; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11884; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
11885; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11886; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11887; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11888; GFX90A-TGSPLIT-NEXT:    buffer_invl2
11889; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11890; GFX90A-TGSPLIT-NEXT:    s_endpgm
11891;
11892; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
11893; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11894; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11895; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11896; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11897; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11898; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11899; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
11900; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11901; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
11902; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11903; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
11904; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11905;
11906; GFX940-TGSPLIT-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
11907; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11908; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11909; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11910; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11911; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11912; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11913; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
11914; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11915; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
11916; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11917; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
11918; GFX940-TGSPLIT-NEXT:    s_endpgm
11919;
11920; GFX11-WGP-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
11921; GFX11-WGP:       ; %bb.0: ; %entry
11922; GFX11-WGP-NEXT:    s_clause 0x1
11923; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11924; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11925; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11926; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
11927; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
11928; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
11929; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
11930; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11931; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11932; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11933; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11934; GFX11-WGP-NEXT:    buffer_gl0_inv
11935; GFX11-WGP-NEXT:    buffer_gl1_inv
11936; GFX11-WGP-NEXT:    s_endpgm
11937;
11938; GFX11-CU-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
11939; GFX11-CU:       ; %bb.0: ; %entry
11940; GFX11-CU-NEXT:    s_clause 0x1
11941; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11942; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11943; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11944; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
11945; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
11946; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
11947; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
11948; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11949; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11950; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11951; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11952; GFX11-CU-NEXT:    buffer_gl0_inv
11953; GFX11-CU-NEXT:    buffer_gl1_inv
11954; GFX11-CU-NEXT:    s_endpgm
11955    i32* %out, i32 %in, i32 %old) {
11956entry:
11957  %gep = getelementptr i32, i32* %out, i32 4
11958  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acquire seq_cst
11959  ret void
11960}
11961
11962define amdgpu_kernel void @flat_system_one_as_release_seq_cst_cmpxchg(
11963; GFX7-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
11964; GFX7:       ; %bb.0: ; %entry
11965; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11966; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11967; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11968; GFX7-NEXT:    s_add_u32 s0, s0, 16
11969; GFX7-NEXT:    s_addc_u32 s1, s1, 0
11970; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11971; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11972; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11973; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11974; GFX7-NEXT:    s_waitcnt vmcnt(0)
11975; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11976; GFX7-NEXT:    s_waitcnt vmcnt(0)
11977; GFX7-NEXT:    buffer_wbinvl1_vol
11978; GFX7-NEXT:    s_endpgm
11979;
11980; GFX10-WGP-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
11981; GFX10-WGP:       ; %bb.0: ; %entry
11982; GFX10-WGP-NEXT:    s_clause 0x1
11983; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11984; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11985; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11986; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
11987; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
11988; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11989; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11990; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11991; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11992; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11993; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11994; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11995; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11996; GFX10-WGP-NEXT:    buffer_gl0_inv
11997; GFX10-WGP-NEXT:    buffer_gl1_inv
11998; GFX10-WGP-NEXT:    s_endpgm
11999;
12000; GFX10-CU-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
12001; GFX10-CU:       ; %bb.0: ; %entry
12002; GFX10-CU-NEXT:    s_clause 0x1
12003; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12004; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12005; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12006; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
12007; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
12008; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12009; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12010; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12011; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12012; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12013; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12014; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
12015; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12016; GFX10-CU-NEXT:    buffer_gl0_inv
12017; GFX10-CU-NEXT:    buffer_gl1_inv
12018; GFX10-CU-NEXT:    s_endpgm
12019;
12020; SKIP-CACHE-INV-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
12021; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12022; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12023; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12024; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12025; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
12026; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
12027; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12028; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12029; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12030; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12031; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12032; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
12033; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12034; SKIP-CACHE-INV-NEXT:    s_endpgm
12035;
12036; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
12037; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12038; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12039; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12040; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12041; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12042; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12043; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
12044; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12045; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
12046; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12047; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
12048; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
12049; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12050;
12051; GFX90A-TGSPLIT-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
12052; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12053; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12054; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12055; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12056; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12057; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12058; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
12059; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12060; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
12061; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12062; GFX90A-TGSPLIT-NEXT:    buffer_invl2
12063; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
12064; GFX90A-TGSPLIT-NEXT:    s_endpgm
12065;
12066; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
12067; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12068; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12069; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12070; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12071; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12072; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12073; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
12074; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12075; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
12076; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12077; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
12078; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12079;
12080; GFX940-TGSPLIT-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
12081; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12082; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12083; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12084; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12085; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12086; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12087; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
12088; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12089; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
12090; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12091; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
12092; GFX940-TGSPLIT-NEXT:    s_endpgm
12093;
12094; GFX11-WGP-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
12095; GFX11-WGP:       ; %bb.0: ; %entry
12096; GFX11-WGP-NEXT:    s_clause 0x1
12097; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12098; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12099; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12100; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
12101; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
12102; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
12103; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
12104; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12105; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12106; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
12107; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12108; GFX11-WGP-NEXT:    buffer_gl0_inv
12109; GFX11-WGP-NEXT:    buffer_gl1_inv
12110; GFX11-WGP-NEXT:    s_endpgm
12111;
12112; GFX11-CU-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
12113; GFX11-CU:       ; %bb.0: ; %entry
12114; GFX11-CU-NEXT:    s_clause 0x1
12115; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12116; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12117; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12118; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
12119; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
12120; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
12121; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
12122; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
12123; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12124; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
12125; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12126; GFX11-CU-NEXT:    buffer_gl0_inv
12127; GFX11-CU-NEXT:    buffer_gl1_inv
12128; GFX11-CU-NEXT:    s_endpgm
12129    i32* %out, i32 %in, i32 %old) {
12130entry:
12131  %gep = getelementptr i32, i32* %out, i32 4
12132  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") release seq_cst
12133  ret void
12134}
12135
12136define amdgpu_kernel void @flat_system_one_as_acq_rel_seq_cst_cmpxchg(
12137; GFX7-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
12138; GFX7:       ; %bb.0: ; %entry
12139; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12140; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12141; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12142; GFX7-NEXT:    s_add_u32 s0, s0, 16
12143; GFX7-NEXT:    s_addc_u32 s1, s1, 0
12144; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12145; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12146; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12147; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12148; GFX7-NEXT:    s_waitcnt vmcnt(0)
12149; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
12150; GFX7-NEXT:    s_waitcnt vmcnt(0)
12151; GFX7-NEXT:    buffer_wbinvl1_vol
12152; GFX7-NEXT:    s_endpgm
12153;
12154; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
12155; GFX10-WGP:       ; %bb.0: ; %entry
12156; GFX10-WGP-NEXT:    s_clause 0x1
12157; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12158; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12159; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12160; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
12161; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
12162; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12163; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12164; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12165; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12166; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12167; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12168; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
12169; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12170; GFX10-WGP-NEXT:    buffer_gl0_inv
12171; GFX10-WGP-NEXT:    buffer_gl1_inv
12172; GFX10-WGP-NEXT:    s_endpgm
12173;
12174; GFX10-CU-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
12175; GFX10-CU:       ; %bb.0: ; %entry
12176; GFX10-CU-NEXT:    s_clause 0x1
12177; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12178; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12179; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12180; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
12181; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
12182; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12183; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12184; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12185; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12186; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12187; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12188; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
12189; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12190; GFX10-CU-NEXT:    buffer_gl0_inv
12191; GFX10-CU-NEXT:    buffer_gl1_inv
12192; GFX10-CU-NEXT:    s_endpgm
12193;
12194; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
12195; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12196; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12197; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12198; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12199; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
12200; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
12201; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12202; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12203; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12204; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12205; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12206; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
12207; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12208; SKIP-CACHE-INV-NEXT:    s_endpgm
12209;
12210; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
12211; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12212; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12213; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12214; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12215; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12216; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12217; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
12218; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12219; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
12220; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12221; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
12222; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
12223; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12224;
12225; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
12226; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12227; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12228; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12229; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12230; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12231; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12232; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
12233; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12234; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
12235; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12236; GFX90A-TGSPLIT-NEXT:    buffer_invl2
12237; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
12238; GFX90A-TGSPLIT-NEXT:    s_endpgm
12239;
12240; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
12241; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12242; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12243; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12244; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12245; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12246; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12247; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
12248; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12249; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
12250; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12251; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
12252; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12253;
12254; GFX940-TGSPLIT-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
12255; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12256; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12257; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12258; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12259; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12260; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12261; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
12262; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12263; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
12264; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12265; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
12266; GFX940-TGSPLIT-NEXT:    s_endpgm
12267;
12268; GFX11-WGP-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
12269; GFX11-WGP:       ; %bb.0: ; %entry
12270; GFX11-WGP-NEXT:    s_clause 0x1
12271; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12272; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12273; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12274; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
12275; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
12276; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
12277; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
12278; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12279; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12280; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
12281; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12282; GFX11-WGP-NEXT:    buffer_gl0_inv
12283; GFX11-WGP-NEXT:    buffer_gl1_inv
12284; GFX11-WGP-NEXT:    s_endpgm
12285;
12286; GFX11-CU-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
12287; GFX11-CU:       ; %bb.0: ; %entry
12288; GFX11-CU-NEXT:    s_clause 0x1
12289; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12290; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12291; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12292; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
12293; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
12294; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
12295; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
12296; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
12297; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12298; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
12299; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12300; GFX11-CU-NEXT:    buffer_gl0_inv
12301; GFX11-CU-NEXT:    buffer_gl1_inv
12302; GFX11-CU-NEXT:    s_endpgm
12303    i32* %out, i32 %in, i32 %old) {
12304entry:
12305  %gep = getelementptr i32, i32* %out, i32 4
12306  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel seq_cst
12307  ret void
12308}
12309
12310define amdgpu_kernel void @flat_system_one_as_seq_cst_seq_cst_cmpxchg(
12311; GFX7-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
12312; GFX7:       ; %bb.0: ; %entry
12313; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12314; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12315; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12316; GFX7-NEXT:    s_add_u32 s0, s0, 16
12317; GFX7-NEXT:    s_addc_u32 s1, s1, 0
12318; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12319; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12320; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12321; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12322; GFX7-NEXT:    s_waitcnt vmcnt(0)
12323; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
12324; GFX7-NEXT:    s_waitcnt vmcnt(0)
12325; GFX7-NEXT:    buffer_wbinvl1_vol
12326; GFX7-NEXT:    s_endpgm
12327;
12328; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
12329; GFX10-WGP:       ; %bb.0: ; %entry
12330; GFX10-WGP-NEXT:    s_clause 0x1
12331; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12332; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12333; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12334; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
12335; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
12336; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12337; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12338; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12339; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12340; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12341; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12342; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
12343; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12344; GFX10-WGP-NEXT:    buffer_gl0_inv
12345; GFX10-WGP-NEXT:    buffer_gl1_inv
12346; GFX10-WGP-NEXT:    s_endpgm
12347;
12348; GFX10-CU-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
12349; GFX10-CU:       ; %bb.0: ; %entry
12350; GFX10-CU-NEXT:    s_clause 0x1
12351; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12352; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12353; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12354; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
12355; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
12356; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12357; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12358; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12359; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12360; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12361; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12362; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
12363; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12364; GFX10-CU-NEXT:    buffer_gl0_inv
12365; GFX10-CU-NEXT:    buffer_gl1_inv
12366; GFX10-CU-NEXT:    s_endpgm
12367;
12368; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
12369; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12370; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12371; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12372; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12373; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
12374; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
12375; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12376; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12377; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12378; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12379; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12380; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
12381; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12382; SKIP-CACHE-INV-NEXT:    s_endpgm
12383;
12384; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
12385; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12386; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12387; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12388; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12389; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12390; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12391; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
12392; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12393; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
12394; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12395; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
12396; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
12397; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12398;
12399; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
12400; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12401; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12402; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12403; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12404; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12405; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12406; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
12407; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12408; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
12409; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12410; GFX90A-TGSPLIT-NEXT:    buffer_invl2
12411; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
12412; GFX90A-TGSPLIT-NEXT:    s_endpgm
12413;
12414; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
12415; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12416; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12417; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12418; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12419; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12420; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12421; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
12422; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12423; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
12424; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12425; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
12426; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12427;
12428; GFX940-TGSPLIT-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
12429; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12430; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12431; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12432; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12433; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12434; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12435; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
12436; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12437; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
12438; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12439; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
12440; GFX940-TGSPLIT-NEXT:    s_endpgm
12441;
12442; GFX11-WGP-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
12443; GFX11-WGP:       ; %bb.0: ; %entry
12444; GFX11-WGP-NEXT:    s_clause 0x1
12445; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12446; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12447; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12448; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
12449; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
12450; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
12451; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
12452; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12453; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12454; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
12455; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12456; GFX11-WGP-NEXT:    buffer_gl0_inv
12457; GFX11-WGP-NEXT:    buffer_gl1_inv
12458; GFX11-WGP-NEXT:    s_endpgm
12459;
12460; GFX11-CU-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
12461; GFX11-CU:       ; %bb.0: ; %entry
12462; GFX11-CU-NEXT:    s_clause 0x1
12463; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12464; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12465; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12466; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
12467; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
12468; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
12469; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
12470; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
12471; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12472; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
12473; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12474; GFX11-CU-NEXT:    buffer_gl0_inv
12475; GFX11-CU-NEXT:    buffer_gl1_inv
12476; GFX11-CU-NEXT:    s_endpgm
12477    i32* %out, i32 %in, i32 %old) {
12478entry:
12479  %gep = getelementptr i32, i32* %out, i32 4
12480  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst seq_cst
12481  ret void
12482}
12483
12484define amdgpu_kernel void @flat_system_one_as_monotonic_monotonic_ret_cmpxchg(
12485; GFX7-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
12486; GFX7:       ; %bb.0: ; %entry
12487; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12488; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12489; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12490; GFX7-NEXT:    s_add_u32 s4, s0, 16
12491; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12492; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12493; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12494; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12495; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12496; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12497; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12498; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12499; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12500; GFX7-NEXT:    flat_store_dword v[0:1], v2
12501; GFX7-NEXT:    s_endpgm
12502;
12503; GFX10-WGP-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
12504; GFX10-WGP:       ; %bb.0: ; %entry
12505; GFX10-WGP-NEXT:    s_clause 0x1
12506; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12507; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12508; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12509; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12510; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12511; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12512; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12513; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12514; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12515; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12516; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12517; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12518; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12519; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12520; GFX10-WGP-NEXT:    s_endpgm
12521;
12522; GFX10-CU-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
12523; GFX10-CU:       ; %bb.0: ; %entry
12524; GFX10-CU-NEXT:    s_clause 0x1
12525; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12526; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12527; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12528; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12529; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12530; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12531; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12532; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12533; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12534; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12535; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12536; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12537; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12538; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12539; GFX10-CU-NEXT:    s_endpgm
12540;
12541; SKIP-CACHE-INV-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
12542; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12543; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12544; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12545; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12546; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12547; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12548; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12549; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12550; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12551; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12552; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12553; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12554; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12555; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12556; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12557; SKIP-CACHE-INV-NEXT:    s_endpgm
12558;
12559; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
12560; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12561; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12562; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12563; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12564; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12565; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12566; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12567; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12568; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12569; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12570;
12571; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
12572; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12573; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12574; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12575; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12576; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12577; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12578; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12579; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12580; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12581; GFX90A-TGSPLIT-NEXT:    s_endpgm
12582;
12583; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
12584; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12585; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12586; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12587; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12588; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12589; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12590; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
12591; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12592; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12593; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12594;
12595; GFX940-TGSPLIT-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
12596; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12597; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12598; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12599; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12600; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12601; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12602; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
12603; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12604; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12605; GFX940-TGSPLIT-NEXT:    s_endpgm
12606;
12607; GFX11-WGP-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
12608; GFX11-WGP:       ; %bb.0: ; %entry
12609; GFX11-WGP-NEXT:    s_clause 0x1
12610; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12611; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12612; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12613; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
12614; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
12615; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
12616; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
12617; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12618; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12619; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12620; GFX11-WGP-NEXT:    s_endpgm
12621;
12622; GFX11-CU-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
12623; GFX11-CU:       ; %bb.0: ; %entry
12624; GFX11-CU-NEXT:    s_clause 0x1
12625; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12626; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12627; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12628; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
12629; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
12630; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
12631; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
12632; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12633; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12634; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12635; GFX11-CU-NEXT:    s_endpgm
12636    i32* %out, i32 %in, i32 %old) {
12637entry:
12638  %gep = getelementptr i32, i32* %out, i32 4
12639  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") monotonic monotonic
12640  %val0 = extractvalue { i32, i1 } %val, 0
12641  store i32 %val0, i32* %out, align 4
12642  ret void
12643}
12644
12645define amdgpu_kernel void @flat_system_one_as_acquire_monotonic_ret_cmpxchg(
12646; GFX7-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
12647; GFX7:       ; %bb.0: ; %entry
12648; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12649; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12650; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12651; GFX7-NEXT:    s_add_u32 s4, s0, 16
12652; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12653; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12654; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12655; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12656; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12657; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12658; GFX7-NEXT:    s_waitcnt vmcnt(0)
12659; GFX7-NEXT:    buffer_wbinvl1_vol
12660; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12661; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12662; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12663; GFX7-NEXT:    flat_store_dword v[0:1], v2
12664; GFX7-NEXT:    s_endpgm
12665;
12666; GFX10-WGP-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
12667; GFX10-WGP:       ; %bb.0: ; %entry
12668; GFX10-WGP-NEXT:    s_clause 0x1
12669; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12670; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12671; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12672; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12673; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12674; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12675; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12676; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12677; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12678; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12679; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12680; GFX10-WGP-NEXT:    buffer_gl0_inv
12681; GFX10-WGP-NEXT:    buffer_gl1_inv
12682; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12683; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12684; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12685; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12686; GFX10-WGP-NEXT:    s_endpgm
12687;
12688; GFX10-CU-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
12689; GFX10-CU:       ; %bb.0: ; %entry
12690; GFX10-CU-NEXT:    s_clause 0x1
12691; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12692; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12693; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12694; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12695; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12696; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12697; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12698; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12699; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12700; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12701; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12702; GFX10-CU-NEXT:    buffer_gl0_inv
12703; GFX10-CU-NEXT:    buffer_gl1_inv
12704; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12705; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12706; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12707; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12708; GFX10-CU-NEXT:    s_endpgm
12709;
12710; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
12711; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12712; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12713; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12714; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12715; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12716; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12717; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12718; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12719; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12720; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12721; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12722; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12723; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12724; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12725; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12726; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12727; SKIP-CACHE-INV-NEXT:    s_endpgm
12728;
12729; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
12730; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12731; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12732; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12733; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12734; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12735; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12736; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12737; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12738; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
12739; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
12740; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12741; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12742; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12743;
12744; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
12745; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12746; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12747; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12748; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12749; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12750; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12751; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12752; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12753; GFX90A-TGSPLIT-NEXT:    buffer_invl2
12754; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
12755; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12756; GFX90A-TGSPLIT-NEXT:    s_endpgm
12757;
12758; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
12759; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12760; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12761; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12762; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12763; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12764; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12765; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
12766; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12767; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
12768; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12769; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12770; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12771;
12772; GFX940-TGSPLIT-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
12773; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12774; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12775; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12776; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12777; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12778; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12779; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
12780; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12781; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
12782; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12783; GFX940-TGSPLIT-NEXT:    s_endpgm
12784;
12785; GFX11-WGP-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
12786; GFX11-WGP:       ; %bb.0: ; %entry
12787; GFX11-WGP-NEXT:    s_clause 0x1
12788; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12789; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12790; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12791; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
12792; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
12793; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
12794; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
12795; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12796; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12797; GFX11-WGP-NEXT:    buffer_gl0_inv
12798; GFX11-WGP-NEXT:    buffer_gl1_inv
12799; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12800; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12801; GFX11-WGP-NEXT:    s_endpgm
12802;
12803; GFX11-CU-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
12804; GFX11-CU:       ; %bb.0: ; %entry
12805; GFX11-CU-NEXT:    s_clause 0x1
12806; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12807; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12808; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12809; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
12810; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
12811; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
12812; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
12813; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12814; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
12815; GFX11-CU-NEXT:    buffer_gl0_inv
12816; GFX11-CU-NEXT:    buffer_gl1_inv
12817; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12818; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12819; GFX11-CU-NEXT:    s_endpgm
12820    i32* %out, i32 %in, i32 %old) {
12821entry:
12822  %gep = getelementptr i32, i32* %out, i32 4
12823  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acquire monotonic
12824  %val0 = extractvalue { i32, i1 } %val, 0
12825  store i32 %val0, i32* %out, align 4
12826  ret void
12827}
12828
12829define amdgpu_kernel void @flat_system_one_as_release_monotonic_ret_cmpxchg(
12830; GFX7-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
12831; GFX7:       ; %bb.0: ; %entry
12832; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12833; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12834; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12835; GFX7-NEXT:    s_add_u32 s4, s0, 16
12836; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12837; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12838; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12839; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12840; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12841; GFX7-NEXT:    s_waitcnt vmcnt(0)
12842; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12843; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12844; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12845; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12846; GFX7-NEXT:    flat_store_dword v[0:1], v2
12847; GFX7-NEXT:    s_endpgm
12848;
12849; GFX10-WGP-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
12850; GFX10-WGP:       ; %bb.0: ; %entry
12851; GFX10-WGP-NEXT:    s_clause 0x1
12852; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12853; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12854; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12855; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12856; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12857; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12858; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12859; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12860; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12861; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12862; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12863; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12864; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12865; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12866; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12867; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12868; GFX10-WGP-NEXT:    s_endpgm
12869;
12870; GFX10-CU-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
12871; GFX10-CU:       ; %bb.0: ; %entry
12872; GFX10-CU-NEXT:    s_clause 0x1
12873; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12874; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12875; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12876; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12877; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12878; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12879; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12880; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12881; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12882; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12883; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12884; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12885; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12886; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12887; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12888; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12889; GFX10-CU-NEXT:    s_endpgm
12890;
12891; SKIP-CACHE-INV-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
12892; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12893; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12894; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12895; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12896; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12897; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12898; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12899; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12900; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12901; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12902; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12903; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12904; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12905; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12906; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12907; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12908; SKIP-CACHE-INV-NEXT:    s_endpgm
12909;
12910; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
12911; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12912; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12913; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12914; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12915; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12916; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12917; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
12918; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12919; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12920; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12921; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12922; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12923;
12924; GFX90A-TGSPLIT-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
12925; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12926; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12927; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12928; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12929; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12930; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12931; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
12932; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12933; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12934; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12935; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12936; GFX90A-TGSPLIT-NEXT:    s_endpgm
12937;
12938; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
12939; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12940; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12941; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12942; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12943; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12944; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12945; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
12946; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12947; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
12948; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12949; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12950; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12951;
12952; GFX940-TGSPLIT-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
12953; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12954; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12955; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12956; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12957; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12958; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12959; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
12960; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12961; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
12962; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12963; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12964; GFX940-TGSPLIT-NEXT:    s_endpgm
12965;
12966; GFX11-WGP-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
12967; GFX11-WGP:       ; %bb.0: ; %entry
12968; GFX11-WGP-NEXT:    s_clause 0x1
12969; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12970; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12971; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12972; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
12973; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
12974; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
12975; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
12976; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12977; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12978; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12979; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12980; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12981; GFX11-WGP-NEXT:    s_endpgm
12982;
12983; GFX11-CU-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
12984; GFX11-CU:       ; %bb.0: ; %entry
12985; GFX11-CU-NEXT:    s_clause 0x1
12986; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12987; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12988; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12989; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
12990; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
12991; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
12992; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
12993; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
12994; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12995; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12996; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12997; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12998; GFX11-CU-NEXT:    s_endpgm
12999    i32* %out, i32 %in, i32 %old) {
13000entry:
13001  %gep = getelementptr i32, i32* %out, i32 4
13002  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") release monotonic
13003  %val0 = extractvalue { i32, i1 } %val, 0
13004  store i32 %val0, i32* %out, align 4
13005  ret void
13006}
13007
13008define amdgpu_kernel void @flat_system_one_as_acq_rel_monotonic_ret_cmpxchg(
13009; GFX7-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
13010; GFX7:       ; %bb.0: ; %entry
13011; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13012; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
13013; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13014; GFX7-NEXT:    s_add_u32 s4, s0, 16
13015; GFX7-NEXT:    s_addc_u32 s5, s1, 0
13016; GFX7-NEXT:    v_mov_b32_e32 v0, s4
13017; GFX7-NEXT:    v_mov_b32_e32 v2, s2
13018; GFX7-NEXT:    v_mov_b32_e32 v1, s5
13019; GFX7-NEXT:    v_mov_b32_e32 v3, s3
13020; GFX7-NEXT:    s_waitcnt vmcnt(0)
13021; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13022; GFX7-NEXT:    s_waitcnt vmcnt(0)
13023; GFX7-NEXT:    buffer_wbinvl1_vol
13024; GFX7-NEXT:    v_mov_b32_e32 v0, s0
13025; GFX7-NEXT:    v_mov_b32_e32 v1, s1
13026; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13027; GFX7-NEXT:    flat_store_dword v[0:1], v2
13028; GFX7-NEXT:    s_endpgm
13029;
13030; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
13031; GFX10-WGP:       ; %bb.0: ; %entry
13032; GFX10-WGP-NEXT:    s_clause 0x1
13033; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13034; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13035; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13036; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
13037; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
13038; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
13039; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
13040; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
13041; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
13042; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13043; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
13044; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13045; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13046; GFX10-WGP-NEXT:    buffer_gl0_inv
13047; GFX10-WGP-NEXT:    buffer_gl1_inv
13048; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
13049; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
13050; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13051; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
13052; GFX10-WGP-NEXT:    s_endpgm
13053;
13054; GFX10-CU-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
13055; GFX10-CU:       ; %bb.0: ; %entry
13056; GFX10-CU-NEXT:    s_clause 0x1
13057; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13058; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13059; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13060; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
13061; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
13062; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
13063; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
13064; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
13065; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
13066; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13067; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
13068; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13069; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13070; GFX10-CU-NEXT:    buffer_gl0_inv
13071; GFX10-CU-NEXT:    buffer_gl1_inv
13072; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
13073; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
13074; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13075; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
13076; GFX10-CU-NEXT:    s_endpgm
13077;
13078; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
13079; SKIP-CACHE-INV:       ; %bb.0: ; %entry
13080; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13081; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
13082; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13083; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
13084; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
13085; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
13086; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
13087; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
13088; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
13089; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
13090; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13091; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
13092; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
13093; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
13094; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13095; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
13096; SKIP-CACHE-INV-NEXT:    s_endpgm
13097;
13098; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
13099; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
13100; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13101; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13102; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13103; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13104; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13105; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
13106; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13107; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13108; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13109; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
13110; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
13111; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13112; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13113; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
13114;
13115; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
13116; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
13117; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13118; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13119; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13120; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13121; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13122; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
13123; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13124; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13125; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13126; GFX90A-TGSPLIT-NEXT:    buffer_invl2
13127; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
13128; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13129; GFX90A-TGSPLIT-NEXT:    s_endpgm
13130;
13131; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
13132; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
13133; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13134; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13135; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13136; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13137; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13138; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
13139; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13140; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
13141; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13142; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
13143; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13144; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13145; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
13146;
13147; GFX940-TGSPLIT-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
13148; GFX940-TGSPLIT:       ; %bb.0: ; %entry
13149; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13150; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13151; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13152; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13153; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13154; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
13155; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13156; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
13157; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13158; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
13159; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13160; GFX940-TGSPLIT-NEXT:    s_endpgm
13161;
13162; GFX11-WGP-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
13163; GFX11-WGP:       ; %bb.0: ; %entry
13164; GFX11-WGP-NEXT:    s_clause 0x1
13165; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13166; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13167; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13168; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
13169; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
13170; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
13171; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
13172; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13173; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
13174; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13175; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13176; GFX11-WGP-NEXT:    buffer_gl0_inv
13177; GFX11-WGP-NEXT:    buffer_gl1_inv
13178; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13179; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
13180; GFX11-WGP-NEXT:    s_endpgm
13181;
13182; GFX11-CU-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
13183; GFX11-CU:       ; %bb.0: ; %entry
13184; GFX11-CU-NEXT:    s_clause 0x1
13185; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13186; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13187; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13188; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
13189; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
13190; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
13191; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
13192; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13193; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
13194; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13195; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13196; GFX11-CU-NEXT:    buffer_gl0_inv
13197; GFX11-CU-NEXT:    buffer_gl1_inv
13198; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13199; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
13200; GFX11-CU-NEXT:    s_endpgm
13201    i32* %out, i32 %in, i32 %old) {
13202entry:
13203  %gep = getelementptr i32, i32* %out, i32 4
13204  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel monotonic
13205  %val0 = extractvalue { i32, i1 } %val, 0
13206  store i32 %val0, i32* %out, align 4
13207  ret void
13208}
13209
13210define amdgpu_kernel void @flat_system_one_as_seq_cst_monotonic_ret_cmpxchg(
13211; GFX7-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
13212; GFX7:       ; %bb.0: ; %entry
13213; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13214; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
13215; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13216; GFX7-NEXT:    s_add_u32 s4, s0, 16
13217; GFX7-NEXT:    s_addc_u32 s5, s1, 0
13218; GFX7-NEXT:    v_mov_b32_e32 v0, s4
13219; GFX7-NEXT:    v_mov_b32_e32 v2, s2
13220; GFX7-NEXT:    v_mov_b32_e32 v1, s5
13221; GFX7-NEXT:    v_mov_b32_e32 v3, s3
13222; GFX7-NEXT:    s_waitcnt vmcnt(0)
13223; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13224; GFX7-NEXT:    s_waitcnt vmcnt(0)
13225; GFX7-NEXT:    buffer_wbinvl1_vol
13226; GFX7-NEXT:    v_mov_b32_e32 v0, s0
13227; GFX7-NEXT:    v_mov_b32_e32 v1, s1
13228; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13229; GFX7-NEXT:    flat_store_dword v[0:1], v2
13230; GFX7-NEXT:    s_endpgm
13231;
13232; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
13233; GFX10-WGP:       ; %bb.0: ; %entry
13234; GFX10-WGP-NEXT:    s_clause 0x1
13235; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13236; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13237; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13238; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
13239; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
13240; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
13241; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
13242; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
13243; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
13244; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13245; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
13246; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13247; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13248; GFX10-WGP-NEXT:    buffer_gl0_inv
13249; GFX10-WGP-NEXT:    buffer_gl1_inv
13250; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
13251; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
13252; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13253; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
13254; GFX10-WGP-NEXT:    s_endpgm
13255;
13256; GFX10-CU-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
13257; GFX10-CU:       ; %bb.0: ; %entry
13258; GFX10-CU-NEXT:    s_clause 0x1
13259; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13260; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13261; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13262; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
13263; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
13264; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
13265; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
13266; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
13267; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
13268; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13269; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
13270; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13271; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13272; GFX10-CU-NEXT:    buffer_gl0_inv
13273; GFX10-CU-NEXT:    buffer_gl1_inv
13274; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
13275; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
13276; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13277; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
13278; GFX10-CU-NEXT:    s_endpgm
13279;
13280; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
13281; SKIP-CACHE-INV:       ; %bb.0: ; %entry
13282; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13283; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
13284; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13285; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
13286; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
13287; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
13288; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
13289; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
13290; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
13291; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
13292; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13293; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
13294; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
13295; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
13296; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13297; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
13298; SKIP-CACHE-INV-NEXT:    s_endpgm
13299;
13300; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
13301; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
13302; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13303; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13304; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13305; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13306; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13307; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
13308; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13309; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13310; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13311; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
13312; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
13313; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13314; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13315; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
13316;
13317; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
13318; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
13319; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13320; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13321; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13322; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13323; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13324; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
13325; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13326; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13327; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13328; GFX90A-TGSPLIT-NEXT:    buffer_invl2
13329; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
13330; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13331; GFX90A-TGSPLIT-NEXT:    s_endpgm
13332;
13333; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
13334; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
13335; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13336; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13337; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13338; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13339; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13340; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
13341; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13342; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
13343; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13344; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
13345; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13346; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13347; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
13348;
13349; GFX940-TGSPLIT-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
13350; GFX940-TGSPLIT:       ; %bb.0: ; %entry
13351; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13352; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13353; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13354; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13355; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13356; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
13357; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13358; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
13359; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13360; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
13361; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13362; GFX940-TGSPLIT-NEXT:    s_endpgm
13363;
13364; GFX11-WGP-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
13365; GFX11-WGP:       ; %bb.0: ; %entry
13366; GFX11-WGP-NEXT:    s_clause 0x1
13367; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13368; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13369; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13370; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
13371; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
13372; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
13373; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
13374; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13375; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
13376; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13377; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13378; GFX11-WGP-NEXT:    buffer_gl0_inv
13379; GFX11-WGP-NEXT:    buffer_gl1_inv
13380; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13381; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
13382; GFX11-WGP-NEXT:    s_endpgm
13383;
13384; GFX11-CU-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
13385; GFX11-CU:       ; %bb.0: ; %entry
13386; GFX11-CU-NEXT:    s_clause 0x1
13387; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13388; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13389; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13390; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
13391; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
13392; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
13393; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
13394; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13395; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
13396; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13397; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13398; GFX11-CU-NEXT:    buffer_gl0_inv
13399; GFX11-CU-NEXT:    buffer_gl1_inv
13400; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13401; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
13402; GFX11-CU-NEXT:    s_endpgm
13403    i32* %out, i32 %in, i32 %old) {
13404entry:
13405  %gep = getelementptr i32, i32* %out, i32 4
13406  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst monotonic
13407  %val0 = extractvalue { i32, i1 } %val, 0
13408  store i32 %val0, i32* %out, align 4
13409  ret void
13410}
13411
13412define amdgpu_kernel void @flat_system_one_as_monotonic_acquire_ret_cmpxchg(
13413; GFX7-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
13414; GFX7:       ; %bb.0: ; %entry
13415; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13416; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
13417; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13418; GFX7-NEXT:    s_add_u32 s4, s0, 16
13419; GFX7-NEXT:    s_addc_u32 s5, s1, 0
13420; GFX7-NEXT:    v_mov_b32_e32 v0, s4
13421; GFX7-NEXT:    v_mov_b32_e32 v2, s2
13422; GFX7-NEXT:    v_mov_b32_e32 v1, s5
13423; GFX7-NEXT:    v_mov_b32_e32 v3, s3
13424; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13425; GFX7-NEXT:    s_waitcnt vmcnt(0)
13426; GFX7-NEXT:    buffer_wbinvl1_vol
13427; GFX7-NEXT:    v_mov_b32_e32 v0, s0
13428; GFX7-NEXT:    v_mov_b32_e32 v1, s1
13429; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13430; GFX7-NEXT:    flat_store_dword v[0:1], v2
13431; GFX7-NEXT:    s_endpgm
13432;
13433; GFX10-WGP-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
13434; GFX10-WGP:       ; %bb.0: ; %entry
13435; GFX10-WGP-NEXT:    s_clause 0x1
13436; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13437; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13438; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13439; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
13440; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
13441; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
13442; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
13443; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
13444; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
13445; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13446; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13447; GFX10-WGP-NEXT:    buffer_gl0_inv
13448; GFX10-WGP-NEXT:    buffer_gl1_inv
13449; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
13450; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
13451; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13452; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
13453; GFX10-WGP-NEXT:    s_endpgm
13454;
13455; GFX10-CU-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
13456; GFX10-CU:       ; %bb.0: ; %entry
13457; GFX10-CU-NEXT:    s_clause 0x1
13458; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13459; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13460; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13461; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
13462; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
13463; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
13464; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
13465; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
13466; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
13467; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13468; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13469; GFX10-CU-NEXT:    buffer_gl0_inv
13470; GFX10-CU-NEXT:    buffer_gl1_inv
13471; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
13472; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
13473; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13474; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
13475; GFX10-CU-NEXT:    s_endpgm
13476;
13477; SKIP-CACHE-INV-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
13478; SKIP-CACHE-INV:       ; %bb.0: ; %entry
13479; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13480; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
13481; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13482; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
13483; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
13484; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
13485; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
13486; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
13487; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
13488; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13489; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
13490; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
13491; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
13492; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13493; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
13494; SKIP-CACHE-INV-NEXT:    s_endpgm
13495;
13496; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
13497; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
13498; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13499; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13500; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13501; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13502; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13503; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13504; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13505; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
13506; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
13507; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13508; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13509; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
13510;
13511; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
13512; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
13513; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13514; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13515; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13516; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13517; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13518; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13519; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13520; GFX90A-TGSPLIT-NEXT:    buffer_invl2
13521; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
13522; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13523; GFX90A-TGSPLIT-NEXT:    s_endpgm
13524;
13525; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
13526; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
13527; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13528; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13529; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13530; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13531; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13532; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
13533; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13534; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
13535; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13536; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13537; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
13538;
13539; GFX940-TGSPLIT-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
13540; GFX940-TGSPLIT:       ; %bb.0: ; %entry
13541; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13542; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13543; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13544; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13545; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13546; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
13547; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13548; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
13549; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13550; GFX940-TGSPLIT-NEXT:    s_endpgm
13551;
13552; GFX11-WGP-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
13553; GFX11-WGP:       ; %bb.0: ; %entry
13554; GFX11-WGP-NEXT:    s_clause 0x1
13555; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13556; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13557; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13558; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
13559; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
13560; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
13561; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
13562; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13563; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13564; GFX11-WGP-NEXT:    buffer_gl0_inv
13565; GFX11-WGP-NEXT:    buffer_gl1_inv
13566; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13567; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
13568; GFX11-WGP-NEXT:    s_endpgm
13569;
13570; GFX11-CU-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
13571; GFX11-CU:       ; %bb.0: ; %entry
13572; GFX11-CU-NEXT:    s_clause 0x1
13573; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13574; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13575; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13576; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
13577; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
13578; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
13579; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
13580; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13581; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13582; GFX11-CU-NEXT:    buffer_gl0_inv
13583; GFX11-CU-NEXT:    buffer_gl1_inv
13584; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13585; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
13586; GFX11-CU-NEXT:    s_endpgm
13587    i32* %out, i32 %in, i32 %old) {
13588entry:
13589  %gep = getelementptr i32, i32* %out, i32 4
13590  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") monotonic acquire
13591  %val0 = extractvalue { i32, i1 } %val, 0
13592  store i32 %val0, i32* %out, align 4
13593  ret void
13594}
13595
13596define amdgpu_kernel void @flat_system_one_as_acquire_acquire_ret_cmpxchg(
13597; GFX7-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
13598; GFX7:       ; %bb.0: ; %entry
13599; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13600; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
13601; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13602; GFX7-NEXT:    s_add_u32 s4, s0, 16
13603; GFX7-NEXT:    s_addc_u32 s5, s1, 0
13604; GFX7-NEXT:    v_mov_b32_e32 v0, s4
13605; GFX7-NEXT:    v_mov_b32_e32 v2, s2
13606; GFX7-NEXT:    v_mov_b32_e32 v1, s5
13607; GFX7-NEXT:    v_mov_b32_e32 v3, s3
13608; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13609; GFX7-NEXT:    s_waitcnt vmcnt(0)
13610; GFX7-NEXT:    buffer_wbinvl1_vol
13611; GFX7-NEXT:    v_mov_b32_e32 v0, s0
13612; GFX7-NEXT:    v_mov_b32_e32 v1, s1
13613; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13614; GFX7-NEXT:    flat_store_dword v[0:1], v2
13615; GFX7-NEXT:    s_endpgm
13616;
13617; GFX10-WGP-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
13618; GFX10-WGP:       ; %bb.0: ; %entry
13619; GFX10-WGP-NEXT:    s_clause 0x1
13620; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13621; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13622; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13623; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
13624; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
13625; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
13626; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
13627; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
13628; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
13629; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13630; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13631; GFX10-WGP-NEXT:    buffer_gl0_inv
13632; GFX10-WGP-NEXT:    buffer_gl1_inv
13633; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
13634; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
13635; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13636; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
13637; GFX10-WGP-NEXT:    s_endpgm
13638;
13639; GFX10-CU-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
13640; GFX10-CU:       ; %bb.0: ; %entry
13641; GFX10-CU-NEXT:    s_clause 0x1
13642; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13643; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13644; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13645; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
13646; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
13647; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
13648; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
13649; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
13650; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
13651; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13652; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13653; GFX10-CU-NEXT:    buffer_gl0_inv
13654; GFX10-CU-NEXT:    buffer_gl1_inv
13655; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
13656; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
13657; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13658; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
13659; GFX10-CU-NEXT:    s_endpgm
13660;
13661; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
13662; SKIP-CACHE-INV:       ; %bb.0: ; %entry
13663; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13664; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
13665; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13666; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
13667; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
13668; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
13669; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
13670; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
13671; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
13672; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13673; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
13674; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
13675; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
13676; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13677; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
13678; SKIP-CACHE-INV-NEXT:    s_endpgm
13679;
13680; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
13681; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
13682; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13683; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13684; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13685; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13686; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13687; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13688; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13689; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
13690; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
13691; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13692; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13693; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
13694;
13695; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
13696; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
13697; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13698; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13699; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13700; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13701; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13702; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13703; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13704; GFX90A-TGSPLIT-NEXT:    buffer_invl2
13705; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
13706; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13707; GFX90A-TGSPLIT-NEXT:    s_endpgm
13708;
13709; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
13710; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
13711; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13712; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13713; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13714; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13715; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13716; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
13717; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13718; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
13719; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13720; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13721; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
13722;
13723; GFX940-TGSPLIT-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
13724; GFX940-TGSPLIT:       ; %bb.0: ; %entry
13725; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13726; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13727; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13728; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13729; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13730; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
13731; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13732; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
13733; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13734; GFX940-TGSPLIT-NEXT:    s_endpgm
13735;
13736; GFX11-WGP-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
13737; GFX11-WGP:       ; %bb.0: ; %entry
13738; GFX11-WGP-NEXT:    s_clause 0x1
13739; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13740; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13741; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13742; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
13743; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
13744; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
13745; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
13746; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13747; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13748; GFX11-WGP-NEXT:    buffer_gl0_inv
13749; GFX11-WGP-NEXT:    buffer_gl1_inv
13750; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13751; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
13752; GFX11-WGP-NEXT:    s_endpgm
13753;
13754; GFX11-CU-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
13755; GFX11-CU:       ; %bb.0: ; %entry
13756; GFX11-CU-NEXT:    s_clause 0x1
13757; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13758; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13759; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13760; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
13761; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
13762; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
13763; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
13764; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13765; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13766; GFX11-CU-NEXT:    buffer_gl0_inv
13767; GFX11-CU-NEXT:    buffer_gl1_inv
13768; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13769; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
13770; GFX11-CU-NEXT:    s_endpgm
13771    i32* %out, i32 %in, i32 %old) {
13772entry:
13773  %gep = getelementptr i32, i32* %out, i32 4
13774  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acquire acquire
13775  %val0 = extractvalue { i32, i1 } %val, 0
13776  store i32 %val0, i32* %out, align 4
13777  ret void
13778}
13779
13780define amdgpu_kernel void @flat_system_one_as_release_acquire_ret_cmpxchg(
13781; GFX7-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
13782; GFX7:       ; %bb.0: ; %entry
13783; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13784; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
13785; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13786; GFX7-NEXT:    s_add_u32 s4, s0, 16
13787; GFX7-NEXT:    s_addc_u32 s5, s1, 0
13788; GFX7-NEXT:    v_mov_b32_e32 v0, s4
13789; GFX7-NEXT:    v_mov_b32_e32 v2, s2
13790; GFX7-NEXT:    v_mov_b32_e32 v1, s5
13791; GFX7-NEXT:    v_mov_b32_e32 v3, s3
13792; GFX7-NEXT:    s_waitcnt vmcnt(0)
13793; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13794; GFX7-NEXT:    s_waitcnt vmcnt(0)
13795; GFX7-NEXT:    buffer_wbinvl1_vol
13796; GFX7-NEXT:    v_mov_b32_e32 v0, s0
13797; GFX7-NEXT:    v_mov_b32_e32 v1, s1
13798; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13799; GFX7-NEXT:    flat_store_dword v[0:1], v2
13800; GFX7-NEXT:    s_endpgm
13801;
13802; GFX10-WGP-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
13803; GFX10-WGP:       ; %bb.0: ; %entry
13804; GFX10-WGP-NEXT:    s_clause 0x1
13805; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13806; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13807; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13808; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
13809; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
13810; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
13811; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
13812; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
13813; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
13814; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13815; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
13816; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13817; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13818; GFX10-WGP-NEXT:    buffer_gl0_inv
13819; GFX10-WGP-NEXT:    buffer_gl1_inv
13820; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
13821; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
13822; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13823; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
13824; GFX10-WGP-NEXT:    s_endpgm
13825;
13826; GFX10-CU-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
13827; GFX10-CU:       ; %bb.0: ; %entry
13828; GFX10-CU-NEXT:    s_clause 0x1
13829; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13830; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13831; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13832; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
13833; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
13834; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
13835; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
13836; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
13837; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
13838; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13839; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
13840; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13841; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13842; GFX10-CU-NEXT:    buffer_gl0_inv
13843; GFX10-CU-NEXT:    buffer_gl1_inv
13844; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
13845; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
13846; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13847; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
13848; GFX10-CU-NEXT:    s_endpgm
13849;
13850; SKIP-CACHE-INV-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
13851; SKIP-CACHE-INV:       ; %bb.0: ; %entry
13852; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13853; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
13854; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13855; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
13856; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
13857; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
13858; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
13859; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
13860; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
13861; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
13862; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13863; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
13864; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
13865; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
13866; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13867; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
13868; SKIP-CACHE-INV-NEXT:    s_endpgm
13869;
13870; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
13871; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
13872; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13873; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13874; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13875; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13876; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13877; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
13878; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13879; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13880; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13881; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
13882; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
13883; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13884; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13885; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
13886;
13887; GFX90A-TGSPLIT-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
13888; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
13889; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13890; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13891; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13892; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13893; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13894; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
13895; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13896; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13897; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13898; GFX90A-TGSPLIT-NEXT:    buffer_invl2
13899; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
13900; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13901; GFX90A-TGSPLIT-NEXT:    s_endpgm
13902;
13903; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
13904; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
13905; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13906; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13907; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13908; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13909; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13910; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
13911; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13912; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
13913; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13914; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
13915; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13916; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13917; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
13918;
13919; GFX940-TGSPLIT-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
13920; GFX940-TGSPLIT:       ; %bb.0: ; %entry
13921; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13922; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13923; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13924; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13925; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13926; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
13927; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13928; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
13929; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13930; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
13931; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13932; GFX940-TGSPLIT-NEXT:    s_endpgm
13933;
13934; GFX11-WGP-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
13935; GFX11-WGP:       ; %bb.0: ; %entry
13936; GFX11-WGP-NEXT:    s_clause 0x1
13937; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13938; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13939; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13940; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
13941; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
13942; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
13943; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
13944; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13945; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
13946; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13947; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13948; GFX11-WGP-NEXT:    buffer_gl0_inv
13949; GFX11-WGP-NEXT:    buffer_gl1_inv
13950; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13951; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
13952; GFX11-WGP-NEXT:    s_endpgm
13953;
13954; GFX11-CU-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
13955; GFX11-CU:       ; %bb.0: ; %entry
13956; GFX11-CU-NEXT:    s_clause 0x1
13957; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13958; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13959; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13960; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
13961; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
13962; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
13963; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
13964; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13965; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
13966; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13967; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13968; GFX11-CU-NEXT:    buffer_gl0_inv
13969; GFX11-CU-NEXT:    buffer_gl1_inv
13970; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13971; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
13972; GFX11-CU-NEXT:    s_endpgm
13973    i32* %out, i32 %in, i32 %old) {
13974entry:
13975  %gep = getelementptr i32, i32* %out, i32 4
13976  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") release acquire
13977  %val0 = extractvalue { i32, i1 } %val, 0
13978  store i32 %val0, i32* %out, align 4
13979  ret void
13980}
13981
13982define amdgpu_kernel void @flat_system_one_as_acq_rel_acquire_ret_cmpxchg(
13983; GFX7-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
13984; GFX7:       ; %bb.0: ; %entry
13985; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13986; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
13987; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13988; GFX7-NEXT:    s_add_u32 s4, s0, 16
13989; GFX7-NEXT:    s_addc_u32 s5, s1, 0
13990; GFX7-NEXT:    v_mov_b32_e32 v0, s4
13991; GFX7-NEXT:    v_mov_b32_e32 v2, s2
13992; GFX7-NEXT:    v_mov_b32_e32 v1, s5
13993; GFX7-NEXT:    v_mov_b32_e32 v3, s3
13994; GFX7-NEXT:    s_waitcnt vmcnt(0)
13995; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13996; GFX7-NEXT:    s_waitcnt vmcnt(0)
13997; GFX7-NEXT:    buffer_wbinvl1_vol
13998; GFX7-NEXT:    v_mov_b32_e32 v0, s0
13999; GFX7-NEXT:    v_mov_b32_e32 v1, s1
14000; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14001; GFX7-NEXT:    flat_store_dword v[0:1], v2
14002; GFX7-NEXT:    s_endpgm
14003;
14004; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
14005; GFX10-WGP:       ; %bb.0: ; %entry
14006; GFX10-WGP-NEXT:    s_clause 0x1
14007; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14008; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14009; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14010; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
14011; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
14012; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
14013; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
14014; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
14015; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
14016; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14017; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14018; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14019; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14020; GFX10-WGP-NEXT:    buffer_gl0_inv
14021; GFX10-WGP-NEXT:    buffer_gl1_inv
14022; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
14023; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
14024; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14025; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
14026; GFX10-WGP-NEXT:    s_endpgm
14027;
14028; GFX10-CU-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
14029; GFX10-CU:       ; %bb.0: ; %entry
14030; GFX10-CU-NEXT:    s_clause 0x1
14031; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14032; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14033; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14034; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
14035; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
14036; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
14037; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
14038; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
14039; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
14040; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14041; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14042; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14043; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14044; GFX10-CU-NEXT:    buffer_gl0_inv
14045; GFX10-CU-NEXT:    buffer_gl1_inv
14046; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
14047; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
14048; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14049; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
14050; GFX10-CU-NEXT:    s_endpgm
14051;
14052; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
14053; SKIP-CACHE-INV:       ; %bb.0: ; %entry
14054; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14055; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
14056; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14057; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
14058; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
14059; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
14060; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
14061; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
14062; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
14063; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14064; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14065; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14066; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
14067; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
14068; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14069; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
14070; SKIP-CACHE-INV-NEXT:    s_endpgm
14071;
14072; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
14073; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
14074; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14075; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14076; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14077; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14078; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14079; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
14080; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14081; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14082; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14083; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
14084; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
14085; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14086; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14087; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
14088;
14089; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
14090; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
14091; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14092; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14093; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14094; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14095; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14096; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
14097; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14098; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14099; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14100; GFX90A-TGSPLIT-NEXT:    buffer_invl2
14101; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
14102; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14103; GFX90A-TGSPLIT-NEXT:    s_endpgm
14104;
14105; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
14106; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
14107; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14108; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14109; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14110; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14111; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14112; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
14113; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14114; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
14115; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14116; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
14117; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14118; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14119; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
14120;
14121; GFX940-TGSPLIT-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
14122; GFX940-TGSPLIT:       ; %bb.0: ; %entry
14123; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14124; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14125; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14126; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14127; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14128; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
14129; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14130; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
14131; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14132; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
14133; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14134; GFX940-TGSPLIT-NEXT:    s_endpgm
14135;
14136; GFX11-WGP-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
14137; GFX11-WGP:       ; %bb.0: ; %entry
14138; GFX11-WGP-NEXT:    s_clause 0x1
14139; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14140; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14141; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14142; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
14143; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
14144; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
14145; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
14146; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14147; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14148; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14149; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14150; GFX11-WGP-NEXT:    buffer_gl0_inv
14151; GFX11-WGP-NEXT:    buffer_gl1_inv
14152; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14153; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
14154; GFX11-WGP-NEXT:    s_endpgm
14155;
14156; GFX11-CU-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
14157; GFX11-CU:       ; %bb.0: ; %entry
14158; GFX11-CU-NEXT:    s_clause 0x1
14159; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14160; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14161; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14162; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
14163; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
14164; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
14165; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
14166; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14167; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14168; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14169; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14170; GFX11-CU-NEXT:    buffer_gl0_inv
14171; GFX11-CU-NEXT:    buffer_gl1_inv
14172; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14173; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
14174; GFX11-CU-NEXT:    s_endpgm
14175    i32* %out, i32 %in, i32 %old) {
14176entry:
14177  %gep = getelementptr i32, i32* %out, i32 4
14178  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel acquire
14179  %val0 = extractvalue { i32, i1 } %val, 0
14180  store i32 %val0, i32* %out, align 4
14181  ret void
14182}
14183
14184define amdgpu_kernel void @flat_system_one_as_seq_cst_acquire_ret_cmpxchg(
14185; GFX7-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
14186; GFX7:       ; %bb.0: ; %entry
14187; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14188; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
14189; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14190; GFX7-NEXT:    s_add_u32 s4, s0, 16
14191; GFX7-NEXT:    s_addc_u32 s5, s1, 0
14192; GFX7-NEXT:    v_mov_b32_e32 v0, s4
14193; GFX7-NEXT:    v_mov_b32_e32 v2, s2
14194; GFX7-NEXT:    v_mov_b32_e32 v1, s5
14195; GFX7-NEXT:    v_mov_b32_e32 v3, s3
14196; GFX7-NEXT:    s_waitcnt vmcnt(0)
14197; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14198; GFX7-NEXT:    s_waitcnt vmcnt(0)
14199; GFX7-NEXT:    buffer_wbinvl1_vol
14200; GFX7-NEXT:    v_mov_b32_e32 v0, s0
14201; GFX7-NEXT:    v_mov_b32_e32 v1, s1
14202; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14203; GFX7-NEXT:    flat_store_dword v[0:1], v2
14204; GFX7-NEXT:    s_endpgm
14205;
14206; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
14207; GFX10-WGP:       ; %bb.0: ; %entry
14208; GFX10-WGP-NEXT:    s_clause 0x1
14209; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14210; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14211; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14212; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
14213; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
14214; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
14215; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
14216; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
14217; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
14218; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14219; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14220; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14221; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14222; GFX10-WGP-NEXT:    buffer_gl0_inv
14223; GFX10-WGP-NEXT:    buffer_gl1_inv
14224; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
14225; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
14226; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14227; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
14228; GFX10-WGP-NEXT:    s_endpgm
14229;
14230; GFX10-CU-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
14231; GFX10-CU:       ; %bb.0: ; %entry
14232; GFX10-CU-NEXT:    s_clause 0x1
14233; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14234; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14235; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14236; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
14237; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
14238; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
14239; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
14240; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
14241; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
14242; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14243; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14244; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14245; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14246; GFX10-CU-NEXT:    buffer_gl0_inv
14247; GFX10-CU-NEXT:    buffer_gl1_inv
14248; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
14249; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
14250; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14251; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
14252; GFX10-CU-NEXT:    s_endpgm
14253;
14254; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
14255; SKIP-CACHE-INV:       ; %bb.0: ; %entry
14256; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14257; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
14258; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14259; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
14260; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
14261; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
14262; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
14263; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
14264; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
14265; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14266; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14267; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14268; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
14269; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
14270; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14271; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
14272; SKIP-CACHE-INV-NEXT:    s_endpgm
14273;
14274; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
14275; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
14276; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14277; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14278; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14279; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14280; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14281; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
14282; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14283; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14284; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14285; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
14286; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
14287; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14288; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14289; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
14290;
14291; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
14292; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
14293; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14294; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14295; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14296; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14297; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14298; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
14299; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14300; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14301; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14302; GFX90A-TGSPLIT-NEXT:    buffer_invl2
14303; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
14304; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14305; GFX90A-TGSPLIT-NEXT:    s_endpgm
14306;
14307; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
14308; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
14309; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14310; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14311; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14312; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14313; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14314; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
14315; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14316; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
14317; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14318; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
14319; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14320; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14321; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
14322;
14323; GFX940-TGSPLIT-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
14324; GFX940-TGSPLIT:       ; %bb.0: ; %entry
14325; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14326; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14327; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14328; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14329; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14330; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
14331; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14332; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
14333; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14334; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
14335; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14336; GFX940-TGSPLIT-NEXT:    s_endpgm
14337;
14338; GFX11-WGP-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
14339; GFX11-WGP:       ; %bb.0: ; %entry
14340; GFX11-WGP-NEXT:    s_clause 0x1
14341; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14342; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14343; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14344; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
14345; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
14346; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
14347; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
14348; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14349; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14350; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14351; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14352; GFX11-WGP-NEXT:    buffer_gl0_inv
14353; GFX11-WGP-NEXT:    buffer_gl1_inv
14354; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14355; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
14356; GFX11-WGP-NEXT:    s_endpgm
14357;
14358; GFX11-CU-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
14359; GFX11-CU:       ; %bb.0: ; %entry
14360; GFX11-CU-NEXT:    s_clause 0x1
14361; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14362; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14363; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14364; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
14365; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
14366; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
14367; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
14368; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14369; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14370; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14371; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14372; GFX11-CU-NEXT:    buffer_gl0_inv
14373; GFX11-CU-NEXT:    buffer_gl1_inv
14374; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14375; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
14376; GFX11-CU-NEXT:    s_endpgm
14377    i32* %out, i32 %in, i32 %old) {
14378entry:
14379  %gep = getelementptr i32, i32* %out, i32 4
14380  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst acquire
14381  %val0 = extractvalue { i32, i1 } %val, 0
14382  store i32 %val0, i32* %out, align 4
14383  ret void
14384}
14385
14386define amdgpu_kernel void @flat_system_one_as_monotonic_seq_cst_ret_cmpxchg(
14387; GFX7-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
14388; GFX7:       ; %bb.0: ; %entry
14389; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14390; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
14391; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14392; GFX7-NEXT:    s_add_u32 s4, s0, 16
14393; GFX7-NEXT:    s_addc_u32 s5, s1, 0
14394; GFX7-NEXT:    v_mov_b32_e32 v0, s4
14395; GFX7-NEXT:    v_mov_b32_e32 v2, s2
14396; GFX7-NEXT:    v_mov_b32_e32 v1, s5
14397; GFX7-NEXT:    v_mov_b32_e32 v3, s3
14398; GFX7-NEXT:    s_waitcnt vmcnt(0)
14399; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14400; GFX7-NEXT:    s_waitcnt vmcnt(0)
14401; GFX7-NEXT:    buffer_wbinvl1_vol
14402; GFX7-NEXT:    v_mov_b32_e32 v0, s0
14403; GFX7-NEXT:    v_mov_b32_e32 v1, s1
14404; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14405; GFX7-NEXT:    flat_store_dword v[0:1], v2
14406; GFX7-NEXT:    s_endpgm
14407;
14408; GFX10-WGP-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
14409; GFX10-WGP:       ; %bb.0: ; %entry
14410; GFX10-WGP-NEXT:    s_clause 0x1
14411; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14412; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14413; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14414; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
14415; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
14416; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
14417; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
14418; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
14419; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
14420; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14421; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14422; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14423; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14424; GFX10-WGP-NEXT:    buffer_gl0_inv
14425; GFX10-WGP-NEXT:    buffer_gl1_inv
14426; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
14427; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
14428; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14429; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
14430; GFX10-WGP-NEXT:    s_endpgm
14431;
14432; GFX10-CU-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
14433; GFX10-CU:       ; %bb.0: ; %entry
14434; GFX10-CU-NEXT:    s_clause 0x1
14435; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14436; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14437; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14438; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
14439; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
14440; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
14441; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
14442; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
14443; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
14444; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14445; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14446; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14447; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14448; GFX10-CU-NEXT:    buffer_gl0_inv
14449; GFX10-CU-NEXT:    buffer_gl1_inv
14450; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
14451; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
14452; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14453; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
14454; GFX10-CU-NEXT:    s_endpgm
14455;
14456; SKIP-CACHE-INV-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
14457; SKIP-CACHE-INV:       ; %bb.0: ; %entry
14458; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14459; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
14460; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14461; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
14462; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
14463; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
14464; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
14465; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
14466; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
14467; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14468; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14469; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14470; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
14471; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
14472; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14473; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
14474; SKIP-CACHE-INV-NEXT:    s_endpgm
14475;
14476; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
14477; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
14478; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14479; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14480; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14481; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14482; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14483; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
14484; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14485; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14486; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14487; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
14488; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
14489; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14490; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14491; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
14492;
14493; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
14494; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
14495; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14496; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14497; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14498; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14499; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14500; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
14501; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14502; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14503; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14504; GFX90A-TGSPLIT-NEXT:    buffer_invl2
14505; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
14506; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14507; GFX90A-TGSPLIT-NEXT:    s_endpgm
14508;
14509; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
14510; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
14511; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14512; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14513; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14514; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14515; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14516; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
14517; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14518; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
14519; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14520; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
14521; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14522; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14523; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
14524;
14525; GFX940-TGSPLIT-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
14526; GFX940-TGSPLIT:       ; %bb.0: ; %entry
14527; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14528; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14529; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14530; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14531; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14532; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
14533; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14534; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
14535; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14536; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
14537; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14538; GFX940-TGSPLIT-NEXT:    s_endpgm
14539;
14540; GFX11-WGP-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
14541; GFX11-WGP:       ; %bb.0: ; %entry
14542; GFX11-WGP-NEXT:    s_clause 0x1
14543; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14544; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14545; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14546; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
14547; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
14548; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
14549; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
14550; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14551; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14552; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14553; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14554; GFX11-WGP-NEXT:    buffer_gl0_inv
14555; GFX11-WGP-NEXT:    buffer_gl1_inv
14556; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14557; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
14558; GFX11-WGP-NEXT:    s_endpgm
14559;
14560; GFX11-CU-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
14561; GFX11-CU:       ; %bb.0: ; %entry
14562; GFX11-CU-NEXT:    s_clause 0x1
14563; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14564; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14565; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14566; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
14567; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
14568; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
14569; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
14570; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14571; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14572; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14573; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14574; GFX11-CU-NEXT:    buffer_gl0_inv
14575; GFX11-CU-NEXT:    buffer_gl1_inv
14576; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14577; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
14578; GFX11-CU-NEXT:    s_endpgm
14579    i32* %out, i32 %in, i32 %old) {
14580entry:
14581  %gep = getelementptr i32, i32* %out, i32 4
14582  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") monotonic seq_cst
14583  %val0 = extractvalue { i32, i1 } %val, 0
14584  store i32 %val0, i32* %out, align 4
14585  ret void
14586}
14587
14588define amdgpu_kernel void @flat_system_one_as_acquire_seq_cst_ret_cmpxchg(
14589; GFX7-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
14590; GFX7:       ; %bb.0: ; %entry
14591; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14592; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
14593; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14594; GFX7-NEXT:    s_add_u32 s4, s0, 16
14595; GFX7-NEXT:    s_addc_u32 s5, s1, 0
14596; GFX7-NEXT:    v_mov_b32_e32 v0, s4
14597; GFX7-NEXT:    v_mov_b32_e32 v2, s2
14598; GFX7-NEXT:    v_mov_b32_e32 v1, s5
14599; GFX7-NEXT:    v_mov_b32_e32 v3, s3
14600; GFX7-NEXT:    s_waitcnt vmcnt(0)
14601; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14602; GFX7-NEXT:    s_waitcnt vmcnt(0)
14603; GFX7-NEXT:    buffer_wbinvl1_vol
14604; GFX7-NEXT:    v_mov_b32_e32 v0, s0
14605; GFX7-NEXT:    v_mov_b32_e32 v1, s1
14606; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14607; GFX7-NEXT:    flat_store_dword v[0:1], v2
14608; GFX7-NEXT:    s_endpgm
14609;
14610; GFX10-WGP-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
14611; GFX10-WGP:       ; %bb.0: ; %entry
14612; GFX10-WGP-NEXT:    s_clause 0x1
14613; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14614; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14615; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14616; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
14617; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
14618; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
14619; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
14620; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
14621; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
14622; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14623; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14624; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14625; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14626; GFX10-WGP-NEXT:    buffer_gl0_inv
14627; GFX10-WGP-NEXT:    buffer_gl1_inv
14628; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
14629; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
14630; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14631; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
14632; GFX10-WGP-NEXT:    s_endpgm
14633;
14634; GFX10-CU-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
14635; GFX10-CU:       ; %bb.0: ; %entry
14636; GFX10-CU-NEXT:    s_clause 0x1
14637; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14638; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14639; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14640; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
14641; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
14642; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
14643; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
14644; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
14645; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
14646; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14647; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14648; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14649; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14650; GFX10-CU-NEXT:    buffer_gl0_inv
14651; GFX10-CU-NEXT:    buffer_gl1_inv
14652; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
14653; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
14654; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14655; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
14656; GFX10-CU-NEXT:    s_endpgm
14657;
14658; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
14659; SKIP-CACHE-INV:       ; %bb.0: ; %entry
14660; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14661; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
14662; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14663; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
14664; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
14665; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
14666; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
14667; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
14668; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
14669; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14670; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14671; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14672; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
14673; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
14674; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14675; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
14676; SKIP-CACHE-INV-NEXT:    s_endpgm
14677;
14678; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
14679; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
14680; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14681; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14682; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14683; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14684; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14685; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
14686; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14687; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14688; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14689; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
14690; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
14691; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14692; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14693; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
14694;
14695; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
14696; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
14697; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14698; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14699; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14700; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14701; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14702; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
14703; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14704; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14705; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14706; GFX90A-TGSPLIT-NEXT:    buffer_invl2
14707; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
14708; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14709; GFX90A-TGSPLIT-NEXT:    s_endpgm
14710;
14711; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
14712; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
14713; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14714; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14715; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14716; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14717; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14718; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
14719; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14720; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
14721; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14722; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
14723; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14724; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14725; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
14726;
14727; GFX940-TGSPLIT-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
14728; GFX940-TGSPLIT:       ; %bb.0: ; %entry
14729; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14730; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14731; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14732; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14733; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14734; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
14735; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14736; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
14737; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14738; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
14739; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14740; GFX940-TGSPLIT-NEXT:    s_endpgm
14741;
14742; GFX11-WGP-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
14743; GFX11-WGP:       ; %bb.0: ; %entry
14744; GFX11-WGP-NEXT:    s_clause 0x1
14745; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14746; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14747; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14748; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
14749; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
14750; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
14751; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
14752; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14753; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14754; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14755; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14756; GFX11-WGP-NEXT:    buffer_gl0_inv
14757; GFX11-WGP-NEXT:    buffer_gl1_inv
14758; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14759; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
14760; GFX11-WGP-NEXT:    s_endpgm
14761;
14762; GFX11-CU-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
14763; GFX11-CU:       ; %bb.0: ; %entry
14764; GFX11-CU-NEXT:    s_clause 0x1
14765; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14766; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14767; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14768; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
14769; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
14770; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
14771; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
14772; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14773; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14774; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14775; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14776; GFX11-CU-NEXT:    buffer_gl0_inv
14777; GFX11-CU-NEXT:    buffer_gl1_inv
14778; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14779; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
14780; GFX11-CU-NEXT:    s_endpgm
14781    i32* %out, i32 %in, i32 %old) {
14782entry:
14783  %gep = getelementptr i32, i32* %out, i32 4
14784  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acquire seq_cst
14785  %val0 = extractvalue { i32, i1 } %val, 0
14786  store i32 %val0, i32* %out, align 4
14787  ret void
14788}
14789
14790define amdgpu_kernel void @flat_system_one_as_release_seq_cst_ret_cmpxchg(
14791; GFX7-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
14792; GFX7:       ; %bb.0: ; %entry
14793; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14794; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
14795; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14796; GFX7-NEXT:    s_add_u32 s4, s0, 16
14797; GFX7-NEXT:    s_addc_u32 s5, s1, 0
14798; GFX7-NEXT:    v_mov_b32_e32 v0, s4
14799; GFX7-NEXT:    v_mov_b32_e32 v2, s2
14800; GFX7-NEXT:    v_mov_b32_e32 v1, s5
14801; GFX7-NEXT:    v_mov_b32_e32 v3, s3
14802; GFX7-NEXT:    s_waitcnt vmcnt(0)
14803; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14804; GFX7-NEXT:    s_waitcnt vmcnt(0)
14805; GFX7-NEXT:    buffer_wbinvl1_vol
14806; GFX7-NEXT:    v_mov_b32_e32 v0, s0
14807; GFX7-NEXT:    v_mov_b32_e32 v1, s1
14808; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14809; GFX7-NEXT:    flat_store_dword v[0:1], v2
14810; GFX7-NEXT:    s_endpgm
14811;
14812; GFX10-WGP-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
14813; GFX10-WGP:       ; %bb.0: ; %entry
14814; GFX10-WGP-NEXT:    s_clause 0x1
14815; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14816; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14817; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14818; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
14819; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
14820; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
14821; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
14822; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
14823; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
14824; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14825; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14826; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14827; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14828; GFX10-WGP-NEXT:    buffer_gl0_inv
14829; GFX10-WGP-NEXT:    buffer_gl1_inv
14830; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
14831; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
14832; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14833; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
14834; GFX10-WGP-NEXT:    s_endpgm
14835;
14836; GFX10-CU-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
14837; GFX10-CU:       ; %bb.0: ; %entry
14838; GFX10-CU-NEXT:    s_clause 0x1
14839; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14840; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14841; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14842; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
14843; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
14844; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
14845; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
14846; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
14847; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
14848; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14849; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14850; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14851; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14852; GFX10-CU-NEXT:    buffer_gl0_inv
14853; GFX10-CU-NEXT:    buffer_gl1_inv
14854; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
14855; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
14856; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14857; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
14858; GFX10-CU-NEXT:    s_endpgm
14859;
14860; SKIP-CACHE-INV-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
14861; SKIP-CACHE-INV:       ; %bb.0: ; %entry
14862; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14863; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
14864; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14865; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
14866; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
14867; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
14868; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
14869; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
14870; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
14871; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14872; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14873; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14874; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
14875; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
14876; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14877; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
14878; SKIP-CACHE-INV-NEXT:    s_endpgm
14879;
14880; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
14881; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
14882; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14883; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14884; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14885; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14886; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14887; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
14888; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14889; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14890; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14891; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
14892; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
14893; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14894; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14895; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
14896;
14897; GFX90A-TGSPLIT-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
14898; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
14899; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14900; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14901; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14902; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14903; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14904; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
14905; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14906; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14907; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14908; GFX90A-TGSPLIT-NEXT:    buffer_invl2
14909; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
14910; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14911; GFX90A-TGSPLIT-NEXT:    s_endpgm
14912;
14913; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
14914; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
14915; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14916; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14917; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14918; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14919; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14920; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
14921; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14922; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
14923; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14924; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
14925; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14926; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14927; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
14928;
14929; GFX940-TGSPLIT-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
14930; GFX940-TGSPLIT:       ; %bb.0: ; %entry
14931; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14932; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14933; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14934; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14935; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14936; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
14937; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14938; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
14939; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14940; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
14941; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14942; GFX940-TGSPLIT-NEXT:    s_endpgm
14943;
14944; GFX11-WGP-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
14945; GFX11-WGP:       ; %bb.0: ; %entry
14946; GFX11-WGP-NEXT:    s_clause 0x1
14947; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14948; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14949; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14950; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
14951; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
14952; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
14953; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
14954; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14955; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14956; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14957; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14958; GFX11-WGP-NEXT:    buffer_gl0_inv
14959; GFX11-WGP-NEXT:    buffer_gl1_inv
14960; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14961; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
14962; GFX11-WGP-NEXT:    s_endpgm
14963;
14964; GFX11-CU-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
14965; GFX11-CU:       ; %bb.0: ; %entry
14966; GFX11-CU-NEXT:    s_clause 0x1
14967; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14968; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14969; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14970; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
14971; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
14972; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
14973; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
14974; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14975; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14976; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14977; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14978; GFX11-CU-NEXT:    buffer_gl0_inv
14979; GFX11-CU-NEXT:    buffer_gl1_inv
14980; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14981; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
14982; GFX11-CU-NEXT:    s_endpgm
14983    i32* %out, i32 %in, i32 %old) {
14984entry:
14985  %gep = getelementptr i32, i32* %out, i32 4
14986  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") release seq_cst
14987  %val0 = extractvalue { i32, i1 } %val, 0
14988  store i32 %val0, i32* %out, align 4
14989  ret void
14990}
14991
14992define amdgpu_kernel void @flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg(
14993; GFX7-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
14994; GFX7:       ; %bb.0: ; %entry
14995; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14996; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
14997; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14998; GFX7-NEXT:    s_add_u32 s4, s0, 16
14999; GFX7-NEXT:    s_addc_u32 s5, s1, 0
15000; GFX7-NEXT:    v_mov_b32_e32 v0, s4
15001; GFX7-NEXT:    v_mov_b32_e32 v2, s2
15002; GFX7-NEXT:    v_mov_b32_e32 v1, s5
15003; GFX7-NEXT:    v_mov_b32_e32 v3, s3
15004; GFX7-NEXT:    s_waitcnt vmcnt(0)
15005; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
15006; GFX7-NEXT:    s_waitcnt vmcnt(0)
15007; GFX7-NEXT:    buffer_wbinvl1_vol
15008; GFX7-NEXT:    v_mov_b32_e32 v0, s0
15009; GFX7-NEXT:    v_mov_b32_e32 v1, s1
15010; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
15011; GFX7-NEXT:    flat_store_dword v[0:1], v2
15012; GFX7-NEXT:    s_endpgm
15013;
15014; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
15015; GFX10-WGP:       ; %bb.0: ; %entry
15016; GFX10-WGP-NEXT:    s_clause 0x1
15017; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
15018; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
15019; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
15020; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
15021; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
15022; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
15023; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
15024; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
15025; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
15026; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
15027; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
15028; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
15029; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
15030; GFX10-WGP-NEXT:    buffer_gl0_inv
15031; GFX10-WGP-NEXT:    buffer_gl1_inv
15032; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
15033; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
15034; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
15035; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
15036; GFX10-WGP-NEXT:    s_endpgm
15037;
15038; GFX10-CU-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
15039; GFX10-CU:       ; %bb.0: ; %entry
15040; GFX10-CU-NEXT:    s_clause 0x1
15041; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
15042; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
15043; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
15044; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
15045; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
15046; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
15047; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
15048; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
15049; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
15050; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
15051; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
15052; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
15053; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
15054; GFX10-CU-NEXT:    buffer_gl0_inv
15055; GFX10-CU-NEXT:    buffer_gl1_inv
15056; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
15057; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
15058; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
15059; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
15060; GFX10-CU-NEXT:    s_endpgm
15061;
15062; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
15063; SKIP-CACHE-INV:       ; %bb.0: ; %entry
15064; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
15065; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
15066; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
15067; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
15068; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
15069; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
15070; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
15071; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
15072; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
15073; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
15074; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
15075; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
15076; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
15077; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
15078; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
15079; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
15080; SKIP-CACHE-INV-NEXT:    s_endpgm
15081;
15082; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
15083; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
15084; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
15085; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
15086; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
15087; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
15088; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
15089; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
15090; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
15091; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
15092; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
15093; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
15094; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
15095; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
15096; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
15097; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
15098;
15099; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
15100; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
15101; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
15102; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
15103; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
15104; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
15105; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
15106; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
15107; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
15108; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
15109; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
15110; GFX90A-TGSPLIT-NEXT:    buffer_invl2
15111; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
15112; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
15113; GFX90A-TGSPLIT-NEXT:    s_endpgm
15114;
15115; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
15116; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
15117; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
15118; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
15119; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
15120; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
15121; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
15122; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
15123; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
15124; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
15125; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
15126; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
15127; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
15128; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
15129; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
15130;
15131; GFX940-TGSPLIT-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
15132; GFX940-TGSPLIT:       ; %bb.0: ; %entry
15133; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
15134; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
15135; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
15136; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
15137; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
15138; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
15139; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
15140; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
15141; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
15142; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
15143; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
15144; GFX940-TGSPLIT-NEXT:    s_endpgm
15145;
15146; GFX11-WGP-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
15147; GFX11-WGP:       ; %bb.0: ; %entry
15148; GFX11-WGP-NEXT:    s_clause 0x1
15149; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
15150; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
15151; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
15152; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
15153; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
15154; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
15155; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
15156; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
15157; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
15158; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
15159; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
15160; GFX11-WGP-NEXT:    buffer_gl0_inv
15161; GFX11-WGP-NEXT:    buffer_gl1_inv
15162; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
15163; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
15164; GFX11-WGP-NEXT:    s_endpgm
15165;
15166; GFX11-CU-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
15167; GFX11-CU:       ; %bb.0: ; %entry
15168; GFX11-CU-NEXT:    s_clause 0x1
15169; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
15170; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
15171; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
15172; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
15173; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
15174; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
15175; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
15176; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
15177; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
15178; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
15179; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
15180; GFX11-CU-NEXT:    buffer_gl0_inv
15181; GFX11-CU-NEXT:    buffer_gl1_inv
15182; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
15183; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
15184; GFX11-CU-NEXT:    s_endpgm
15185    i32* %out, i32 %in, i32 %old) {
15186entry:
15187  %gep = getelementptr i32, i32* %out, i32 4
15188  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel seq_cst
15189  %val0 = extractvalue { i32, i1 } %val, 0
15190  store i32 %val0, i32* %out, align 4
15191  ret void
15192}
15193
15194define amdgpu_kernel void @flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg(
15195; GFX7-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
15196; GFX7:       ; %bb.0: ; %entry
15197; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
15198; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
15199; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
15200; GFX7-NEXT:    s_add_u32 s4, s0, 16
15201; GFX7-NEXT:    s_addc_u32 s5, s1, 0
15202; GFX7-NEXT:    v_mov_b32_e32 v0, s4
15203; GFX7-NEXT:    v_mov_b32_e32 v2, s2
15204; GFX7-NEXT:    v_mov_b32_e32 v1, s5
15205; GFX7-NEXT:    v_mov_b32_e32 v3, s3
15206; GFX7-NEXT:    s_waitcnt vmcnt(0)
15207; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
15208; GFX7-NEXT:    s_waitcnt vmcnt(0)
15209; GFX7-NEXT:    buffer_wbinvl1_vol
15210; GFX7-NEXT:    v_mov_b32_e32 v0, s0
15211; GFX7-NEXT:    v_mov_b32_e32 v1, s1
15212; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
15213; GFX7-NEXT:    flat_store_dword v[0:1], v2
15214; GFX7-NEXT:    s_endpgm
15215;
15216; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
15217; GFX10-WGP:       ; %bb.0: ; %entry
15218; GFX10-WGP-NEXT:    s_clause 0x1
15219; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
15220; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
15221; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
15222; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
15223; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
15224; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
15225; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
15226; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
15227; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
15228; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
15229; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
15230; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
15231; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
15232; GFX10-WGP-NEXT:    buffer_gl0_inv
15233; GFX10-WGP-NEXT:    buffer_gl1_inv
15234; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
15235; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
15236; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
15237; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
15238; GFX10-WGP-NEXT:    s_endpgm
15239;
15240; GFX10-CU-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
15241; GFX10-CU:       ; %bb.0: ; %entry
15242; GFX10-CU-NEXT:    s_clause 0x1
15243; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
15244; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
15245; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
15246; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
15247; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
15248; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
15249; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
15250; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
15251; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
15252; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
15253; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
15254; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
15255; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
15256; GFX10-CU-NEXT:    buffer_gl0_inv
15257; GFX10-CU-NEXT:    buffer_gl1_inv
15258; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
15259; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
15260; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
15261; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
15262; GFX10-CU-NEXT:    s_endpgm
15263;
15264; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
15265; SKIP-CACHE-INV:       ; %bb.0: ; %entry
15266; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
15267; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
15268; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
15269; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
15270; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
15271; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
15272; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
15273; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
15274; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
15275; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
15276; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
15277; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
15278; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
15279; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
15280; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
15281; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
15282; SKIP-CACHE-INV-NEXT:    s_endpgm
15283;
15284; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
15285; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
15286; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
15287; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
15288; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
15289; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
15290; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
15291; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
15292; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
15293; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
15294; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
15295; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
15296; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
15297; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
15298; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
15299; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
15300;
15301; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
15302; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
15303; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
15304; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
15305; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
15306; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
15307; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
15308; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
15309; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
15310; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
15311; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
15312; GFX90A-TGSPLIT-NEXT:    buffer_invl2
15313; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
15314; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
15315; GFX90A-TGSPLIT-NEXT:    s_endpgm
15316;
15317; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
15318; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
15319; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
15320; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
15321; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
15322; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
15323; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
15324; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
15325; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
15326; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
15327; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
15328; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
15329; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
15330; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
15331; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
15332;
15333; GFX940-TGSPLIT-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
15334; GFX940-TGSPLIT:       ; %bb.0: ; %entry
15335; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
15336; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
15337; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
15338; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
15339; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
15340; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
15341; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
15342; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
15343; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
15344; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
15345; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
15346; GFX940-TGSPLIT-NEXT:    s_endpgm
15347;
15348; GFX11-WGP-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
15349; GFX11-WGP:       ; %bb.0: ; %entry
15350; GFX11-WGP-NEXT:    s_clause 0x1
15351; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
15352; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
15353; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
15354; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
15355; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
15356; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
15357; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
15358; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
15359; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
15360; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
15361; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
15362; GFX11-WGP-NEXT:    buffer_gl0_inv
15363; GFX11-WGP-NEXT:    buffer_gl1_inv
15364; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
15365; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
15366; GFX11-WGP-NEXT:    s_endpgm
15367;
15368; GFX11-CU-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
15369; GFX11-CU:       ; %bb.0: ; %entry
15370; GFX11-CU-NEXT:    s_clause 0x1
15371; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
15372; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
15373; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
15374; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
15375; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
15376; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
15377; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
15378; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
15379; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
15380; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
15381; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
15382; GFX11-CU-NEXT:    buffer_gl0_inv
15383; GFX11-CU-NEXT:    buffer_gl1_inv
15384; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
15385; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
15386; GFX11-CU-NEXT:    s_endpgm
15387    i32* %out, i32 %in, i32 %old) {
15388entry:
15389  %gep = getelementptr i32, i32* %out, i32 4
15390  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst seq_cst
15391  %val0 = extractvalue { i32, i1 } %val, 0
15392  store i32 %val0, i32* %out, align 4
15393  ret void
15394}
15395
15396