1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
3; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
4; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
5; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
6; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-NOTTGSPLIT %s
7; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-TGSPLIT %s
8; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-NOTTGSPLIT %s
9; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
10; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11-WGP %s
11; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11-CU %s
12
13define amdgpu_kernel void @flat_agent_unordered_load(
14; GFX7-LABEL: flat_agent_unordered_load:
15; GFX7:       ; %bb.0: ; %entry
16; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
17; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
18; GFX7-NEXT:    v_mov_b32_e32 v0, s0
19; GFX7-NEXT:    v_mov_b32_e32 v1, s1
20; GFX7-NEXT:    flat_load_dword v2, v[0:1]
21; GFX7-NEXT:    v_mov_b32_e32 v0, s2
22; GFX7-NEXT:    v_mov_b32_e32 v1, s3
23; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
24; GFX7-NEXT:    flat_store_dword v[0:1], v2
25; GFX7-NEXT:    s_endpgm
26;
27; GFX10-WGP-LABEL: flat_agent_unordered_load:
28; GFX10-WGP:       ; %bb.0: ; %entry
29; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
30; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
31; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
32; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
33; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
34; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
35; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
36; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
37; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
38; GFX10-WGP-NEXT:    s_endpgm
39;
40; GFX10-CU-LABEL: flat_agent_unordered_load:
41; GFX10-CU:       ; %bb.0: ; %entry
42; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
43; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
44; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
45; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
46; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
47; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
48; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
49; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
50; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
51; GFX10-CU-NEXT:    s_endpgm
52;
53; SKIP-CACHE-INV-LABEL: flat_agent_unordered_load:
54; SKIP-CACHE-INV:       ; %bb.0: ; %entry
55; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
56; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
57; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
58; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
59; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
60; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
61; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
62; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
63; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
64; SKIP-CACHE-INV-NEXT:    s_endpgm
65;
66; GFX90A-NOTTGSPLIT-LABEL: flat_agent_unordered_load:
67; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
68; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
69; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
70; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
71; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
72; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
73; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
74; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
75; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
76; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
77; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
78;
79; GFX90A-TGSPLIT-LABEL: flat_agent_unordered_load:
80; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
81; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
82; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
83; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
84; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
85; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
86; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
87; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
88; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
89; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
90; GFX90A-TGSPLIT-NEXT:    s_endpgm
91;
92; GFX940-NOTTGSPLIT-LABEL: flat_agent_unordered_load:
93; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
94; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
95; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
96; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
97; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
98; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
99; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
100; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
101; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
102; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
103; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
104;
105; GFX940-TGSPLIT-LABEL: flat_agent_unordered_load:
106; GFX940-TGSPLIT:       ; %bb.0: ; %entry
107; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
108; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
109; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
110; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
111; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
112; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
113; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
114; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
115; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
116; GFX940-TGSPLIT-NEXT:    s_endpgm
117;
118; GFX11-WGP-LABEL: flat_agent_unordered_load:
119; GFX11-WGP:       ; %bb.0: ; %entry
120; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
121; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
122; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
123; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1]
124; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
125; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
126; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
127; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
128; GFX11-WGP-NEXT:    s_endpgm
129;
130; GFX11-CU-LABEL: flat_agent_unordered_load:
131; GFX11-CU:       ; %bb.0: ; %entry
132; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
133; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
134; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
135; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
136; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
137; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
138; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
139; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
140; GFX11-CU-NEXT:    s_endpgm
141    i32* %in, i32* %out) {
142entry:
143  %val = load atomic i32, i32* %in syncscope("agent") unordered, align 4
144  store i32 %val, i32* %out
145  ret void
146}
147
148define amdgpu_kernel void @flat_agent_monotonic_load(
149; GFX7-LABEL: flat_agent_monotonic_load:
150; GFX7:       ; %bb.0: ; %entry
151; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
152; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
153; GFX7-NEXT:    v_mov_b32_e32 v0, s0
154; GFX7-NEXT:    v_mov_b32_e32 v1, s1
155; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
156; GFX7-NEXT:    v_mov_b32_e32 v0, s2
157; GFX7-NEXT:    v_mov_b32_e32 v1, s3
158; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
159; GFX7-NEXT:    flat_store_dword v[0:1], v2
160; GFX7-NEXT:    s_endpgm
161;
162; GFX10-WGP-LABEL: flat_agent_monotonic_load:
163; GFX10-WGP:       ; %bb.0: ; %entry
164; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
165; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
166; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
167; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
168; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
169; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
170; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
171; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
172; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
173; GFX10-WGP-NEXT:    s_endpgm
174;
175; GFX10-CU-LABEL: flat_agent_monotonic_load:
176; GFX10-CU:       ; %bb.0: ; %entry
177; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
178; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
179; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
180; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
181; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
182; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
183; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
184; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
185; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
186; GFX10-CU-NEXT:    s_endpgm
187;
188; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_load:
189; SKIP-CACHE-INV:       ; %bb.0: ; %entry
190; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
191; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
192; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
193; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
194; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
195; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
196; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
197; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
198; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
199; SKIP-CACHE-INV-NEXT:    s_endpgm
200;
201; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_load:
202; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
203; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
204; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
205; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
206; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
207; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
208; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
209; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
210; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
211; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
212; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
213;
214; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_load:
215; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
216; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
217; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
218; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
219; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
220; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
221; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
222; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
223; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
224; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
225; GFX90A-TGSPLIT-NEXT:    s_endpgm
226;
227; GFX940-NOTTGSPLIT-LABEL: flat_agent_monotonic_load:
228; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
229; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
230; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
231; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
232; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
233; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc1
234; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
235; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
236; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
237; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
238; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
239;
240; GFX940-TGSPLIT-LABEL: flat_agent_monotonic_load:
241; GFX940-TGSPLIT:       ; %bb.0: ; %entry
242; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
243; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
244; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
245; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
246; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc1
247; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
248; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
249; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
250; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
251; GFX940-TGSPLIT-NEXT:    s_endpgm
252;
253; GFX11-WGP-LABEL: flat_agent_monotonic_load:
254; GFX11-WGP:       ; %bb.0: ; %entry
255; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
256; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
257; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
258; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc
259; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
260; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
261; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
262; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
263; GFX11-WGP-NEXT:    s_endpgm
264;
265; GFX11-CU-LABEL: flat_agent_monotonic_load:
266; GFX11-CU:       ; %bb.0: ; %entry
267; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
268; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
269; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
270; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1] glc
271; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
272; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
273; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
274; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
275; GFX11-CU-NEXT:    s_endpgm
276    i32* %in, i32* %out) {
277entry:
278  %val = load atomic i32, i32* %in syncscope("agent") monotonic, align 4
279  store i32 %val, i32* %out
280  ret void
281}
282
283define amdgpu_kernel void @flat_agent_acquire_load(
284; GFX7-LABEL: flat_agent_acquire_load:
285; GFX7:       ; %bb.0: ; %entry
286; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
287; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
288; GFX7-NEXT:    v_mov_b32_e32 v0, s0
289; GFX7-NEXT:    v_mov_b32_e32 v1, s1
290; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
291; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
292; GFX7-NEXT:    buffer_wbinvl1_vol
293; GFX7-NEXT:    v_mov_b32_e32 v0, s2
294; GFX7-NEXT:    v_mov_b32_e32 v1, s3
295; GFX7-NEXT:    flat_store_dword v[0:1], v2
296; GFX7-NEXT:    s_endpgm
297;
298; GFX10-WGP-LABEL: flat_agent_acquire_load:
299; GFX10-WGP:       ; %bb.0: ; %entry
300; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
301; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
302; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
303; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
304; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
305; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
306; GFX10-WGP-NEXT:    buffer_gl0_inv
307; GFX10-WGP-NEXT:    buffer_gl1_inv
308; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
309; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
310; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
311; GFX10-WGP-NEXT:    s_endpgm
312;
313; GFX10-CU-LABEL: flat_agent_acquire_load:
314; GFX10-CU:       ; %bb.0: ; %entry
315; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
316; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
317; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
318; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
319; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
320; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
321; GFX10-CU-NEXT:    buffer_gl0_inv
322; GFX10-CU-NEXT:    buffer_gl1_inv
323; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
324; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
325; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
326; GFX10-CU-NEXT:    s_endpgm
327;
328; SKIP-CACHE-INV-LABEL: flat_agent_acquire_load:
329; SKIP-CACHE-INV:       ; %bb.0: ; %entry
330; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
331; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
332; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
333; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
334; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
335; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
336; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
337; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
338; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
339; SKIP-CACHE-INV-NEXT:    s_endpgm
340;
341; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_load:
342; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
343; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
344; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
345; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
346; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
347; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
348; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
349; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
350; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
351; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
352; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
353; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
354;
355; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_load:
356; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
357; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
358; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
359; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
360; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
361; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
362; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
363; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
364; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
365; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
366; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
367; GFX90A-TGSPLIT-NEXT:    s_endpgm
368;
369; GFX940-NOTTGSPLIT-LABEL: flat_agent_acquire_load:
370; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
371; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
372; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
373; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
374; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
375; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc1
376; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
377; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
378; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
379; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
380; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
381; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
382;
383; GFX940-TGSPLIT-LABEL: flat_agent_acquire_load:
384; GFX940-TGSPLIT:       ; %bb.0: ; %entry
385; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
386; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
387; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
388; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
389; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc1
390; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
391; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
392; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
393; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
394; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
395; GFX940-TGSPLIT-NEXT:    s_endpgm
396;
397; GFX11-WGP-LABEL: flat_agent_acquire_load:
398; GFX11-WGP:       ; %bb.0: ; %entry
399; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
400; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
401; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
402; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc
403; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
404; GFX11-WGP-NEXT:    buffer_gl0_inv
405; GFX11-WGP-NEXT:    buffer_gl1_inv
406; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
407; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
408; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
409; GFX11-WGP-NEXT:    s_endpgm
410;
411; GFX11-CU-LABEL: flat_agent_acquire_load:
412; GFX11-CU:       ; %bb.0: ; %entry
413; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
414; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
415; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
416; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1] glc
417; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
418; GFX11-CU-NEXT:    buffer_gl0_inv
419; GFX11-CU-NEXT:    buffer_gl1_inv
420; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
421; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
422; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
423; GFX11-CU-NEXT:    s_endpgm
424    i32* %in, i32* %out) {
425entry:
426  %val = load atomic i32, i32* %in syncscope("agent") acquire, align 4
427  store i32 %val, i32* %out
428  ret void
429}
430
431define amdgpu_kernel void @flat_agent_seq_cst_load(
432; GFX7-LABEL: flat_agent_seq_cst_load:
433; GFX7:       ; %bb.0: ; %entry
434; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
435; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
436; GFX7-NEXT:    v_mov_b32_e32 v0, s0
437; GFX7-NEXT:    v_mov_b32_e32 v1, s1
438; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
439; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
440; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
441; GFX7-NEXT:    buffer_wbinvl1_vol
442; GFX7-NEXT:    v_mov_b32_e32 v0, s2
443; GFX7-NEXT:    v_mov_b32_e32 v1, s3
444; GFX7-NEXT:    flat_store_dword v[0:1], v2
445; GFX7-NEXT:    s_endpgm
446;
447; GFX10-WGP-LABEL: flat_agent_seq_cst_load:
448; GFX10-WGP:       ; %bb.0: ; %entry
449; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
450; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
451; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
452; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
453; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
454; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
455; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
456; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
457; GFX10-WGP-NEXT:    buffer_gl0_inv
458; GFX10-WGP-NEXT:    buffer_gl1_inv
459; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
460; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
461; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
462; GFX10-WGP-NEXT:    s_endpgm
463;
464; GFX10-CU-LABEL: flat_agent_seq_cst_load:
465; GFX10-CU:       ; %bb.0: ; %entry
466; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
467; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
468; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
469; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
470; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
471; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
472; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
473; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
474; GFX10-CU-NEXT:    buffer_gl0_inv
475; GFX10-CU-NEXT:    buffer_gl1_inv
476; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
477; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
478; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
479; GFX10-CU-NEXT:    s_endpgm
480;
481; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_load:
482; SKIP-CACHE-INV:       ; %bb.0: ; %entry
483; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
484; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
485; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
486; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
487; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
488; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
489; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
490; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
491; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
492; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
493; SKIP-CACHE-INV-NEXT:    s_endpgm
494;
495; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_load:
496; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
497; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
498; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
499; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
500; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
501; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
502; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
503; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
504; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
505; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
506; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
507; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
508; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
509;
510; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_load:
511; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
512; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
513; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
514; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
515; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
516; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
517; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
518; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
519; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
520; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
521; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
522; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
523; GFX90A-TGSPLIT-NEXT:    s_endpgm
524;
525; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_load:
526; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
527; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
528; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
529; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
530; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
531; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
532; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc1
533; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
534; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
535; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
536; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
537; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
538; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
539;
540; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_load:
541; GFX940-TGSPLIT:       ; %bb.0: ; %entry
542; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
543; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
544; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
545; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
546; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
547; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc1
548; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
549; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
550; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
551; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
552; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
553; GFX940-TGSPLIT-NEXT:    s_endpgm
554;
555; GFX11-WGP-LABEL: flat_agent_seq_cst_load:
556; GFX11-WGP:       ; %bb.0: ; %entry
557; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
558; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
559; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
560; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
561; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
562; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc
563; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
564; GFX11-WGP-NEXT:    buffer_gl0_inv
565; GFX11-WGP-NEXT:    buffer_gl1_inv
566; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
567; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
568; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
569; GFX11-WGP-NEXT:    s_endpgm
570;
571; GFX11-CU-LABEL: flat_agent_seq_cst_load:
572; GFX11-CU:       ; %bb.0: ; %entry
573; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
574; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
575; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
576; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
577; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
578; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1] glc
579; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
580; GFX11-CU-NEXT:    buffer_gl0_inv
581; GFX11-CU-NEXT:    buffer_gl1_inv
582; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
583; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
584; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
585; GFX11-CU-NEXT:    s_endpgm
586    i32* %in, i32* %out) {
587entry:
588  %val = load atomic i32, i32* %in syncscope("agent") seq_cst, align 4
589  store i32 %val, i32* %out
590  ret void
591}
592
593define amdgpu_kernel void @flat_agent_unordered_store(
594; GFX7-LABEL: flat_agent_unordered_store:
595; GFX7:       ; %bb.0: ; %entry
596; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
597; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
598; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
599; GFX7-NEXT:    v_mov_b32_e32 v0, s0
600; GFX7-NEXT:    v_mov_b32_e32 v1, s1
601; GFX7-NEXT:    v_mov_b32_e32 v2, s2
602; GFX7-NEXT:    flat_store_dword v[0:1], v2
603; GFX7-NEXT:    s_endpgm
604;
605; GFX10-WGP-LABEL: flat_agent_unordered_store:
606; GFX10-WGP:       ; %bb.0: ; %entry
607; GFX10-WGP-NEXT:    s_clause 0x1
608; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
609; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
610; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
611; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
612; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
613; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
614; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
615; GFX10-WGP-NEXT:    s_endpgm
616;
617; GFX10-CU-LABEL: flat_agent_unordered_store:
618; GFX10-CU:       ; %bb.0: ; %entry
619; GFX10-CU-NEXT:    s_clause 0x1
620; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
621; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
622; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
623; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
624; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
625; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
626; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
627; GFX10-CU-NEXT:    s_endpgm
628;
629; SKIP-CACHE-INV-LABEL: flat_agent_unordered_store:
630; SKIP-CACHE-INV:       ; %bb.0: ; %entry
631; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
632; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
633; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
634; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
635; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
636; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
637; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
638; SKIP-CACHE-INV-NEXT:    s_endpgm
639;
640; GFX90A-NOTTGSPLIT-LABEL: flat_agent_unordered_store:
641; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
642; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
643; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
644; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
645; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
646; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
647; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
648; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
649;
650; GFX90A-TGSPLIT-LABEL: flat_agent_unordered_store:
651; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
652; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
653; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
654; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
655; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
656; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
657; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
658; GFX90A-TGSPLIT-NEXT:    s_endpgm
659;
660; GFX940-NOTTGSPLIT-LABEL: flat_agent_unordered_store:
661; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
662; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
663; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
664; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
665; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
666; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
667; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
668; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
669;
670; GFX940-TGSPLIT-LABEL: flat_agent_unordered_store:
671; GFX940-TGSPLIT:       ; %bb.0: ; %entry
672; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
673; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
674; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
675; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
676; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
677; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
678; GFX940-TGSPLIT-NEXT:    s_endpgm
679;
680; GFX11-WGP-LABEL: flat_agent_unordered_store:
681; GFX11-WGP:       ; %bb.0: ; %entry
682; GFX11-WGP-NEXT:    s_clause 0x1
683; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
684; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
685; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
686; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
687; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
688; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
689; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
690; GFX11-WGP-NEXT:    s_endpgm
691;
692; GFX11-CU-LABEL: flat_agent_unordered_store:
693; GFX11-CU:       ; %bb.0: ; %entry
694; GFX11-CU-NEXT:    s_clause 0x1
695; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
696; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
697; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
698; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
699; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
700; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
701; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
702; GFX11-CU-NEXT:    s_endpgm
703    i32 %in, i32* %out) {
704entry:
705  store atomic i32 %in, i32* %out syncscope("agent") unordered, align 4
706  ret void
707}
708
709define amdgpu_kernel void @flat_agent_monotonic_store(
710; GFX7-LABEL: flat_agent_monotonic_store:
711; GFX7:       ; %bb.0: ; %entry
712; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
713; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
714; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
715; GFX7-NEXT:    v_mov_b32_e32 v0, s0
716; GFX7-NEXT:    v_mov_b32_e32 v1, s1
717; GFX7-NEXT:    v_mov_b32_e32 v2, s2
718; GFX7-NEXT:    flat_store_dword v[0:1], v2
719; GFX7-NEXT:    s_endpgm
720;
721; GFX10-WGP-LABEL: flat_agent_monotonic_store:
722; GFX10-WGP:       ; %bb.0: ; %entry
723; GFX10-WGP-NEXT:    s_clause 0x1
724; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
725; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
726; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
727; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
728; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
729; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
730; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
731; GFX10-WGP-NEXT:    s_endpgm
732;
733; GFX10-CU-LABEL: flat_agent_monotonic_store:
734; GFX10-CU:       ; %bb.0: ; %entry
735; GFX10-CU-NEXT:    s_clause 0x1
736; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
737; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
738; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
739; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
740; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
741; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
742; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
743; GFX10-CU-NEXT:    s_endpgm
744;
745; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_store:
746; SKIP-CACHE-INV:       ; %bb.0: ; %entry
747; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
748; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
749; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
750; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
751; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
752; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
753; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
754; SKIP-CACHE-INV-NEXT:    s_endpgm
755;
756; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_store:
757; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
758; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
759; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
760; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
761; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
762; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
763; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
764; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
765;
766; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_store:
767; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
768; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
769; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
770; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
771; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
772; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
773; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
774; GFX90A-TGSPLIT-NEXT:    s_endpgm
775;
776; GFX940-NOTTGSPLIT-LABEL: flat_agent_monotonic_store:
777; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
778; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
779; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
780; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
781; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
782; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
783; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc1
784; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
785;
786; GFX940-TGSPLIT-LABEL: flat_agent_monotonic_store:
787; GFX940-TGSPLIT:       ; %bb.0: ; %entry
788; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
789; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
790; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
791; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
792; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
793; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc1
794; GFX940-TGSPLIT-NEXT:    s_endpgm
795;
796; GFX11-WGP-LABEL: flat_agent_monotonic_store:
797; GFX11-WGP:       ; %bb.0: ; %entry
798; GFX11-WGP-NEXT:    s_clause 0x1
799; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
800; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
801; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
802; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
803; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
804; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
805; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
806; GFX11-WGP-NEXT:    s_endpgm
807;
808; GFX11-CU-LABEL: flat_agent_monotonic_store:
809; GFX11-CU:       ; %bb.0: ; %entry
810; GFX11-CU-NEXT:    s_clause 0x1
811; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
812; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
813; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
814; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
815; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
816; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
817; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
818; GFX11-CU-NEXT:    s_endpgm
819    i32 %in, i32* %out) {
820entry:
821  store atomic i32 %in, i32* %out syncscope("agent") monotonic, align 4
822  ret void
823}
824
825define amdgpu_kernel void @flat_agent_release_store(
826; GFX7-LABEL: flat_agent_release_store:
827; GFX7:       ; %bb.0: ; %entry
828; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
829; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
830; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
831; GFX7-NEXT:    v_mov_b32_e32 v0, s0
832; GFX7-NEXT:    v_mov_b32_e32 v1, s1
833; GFX7-NEXT:    v_mov_b32_e32 v2, s2
834; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
835; GFX7-NEXT:    flat_store_dword v[0:1], v2
836; GFX7-NEXT:    s_endpgm
837;
838; GFX10-WGP-LABEL: flat_agent_release_store:
839; GFX10-WGP:       ; %bb.0: ; %entry
840; GFX10-WGP-NEXT:    s_clause 0x1
841; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
842; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
843; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
844; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
845; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
846; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
847; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
848; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
849; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
850; GFX10-WGP-NEXT:    s_endpgm
851;
852; GFX10-CU-LABEL: flat_agent_release_store:
853; GFX10-CU:       ; %bb.0: ; %entry
854; GFX10-CU-NEXT:    s_clause 0x1
855; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
856; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
857; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
858; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
859; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
860; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
861; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
862; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
863; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
864; GFX10-CU-NEXT:    s_endpgm
865;
866; SKIP-CACHE-INV-LABEL: flat_agent_release_store:
867; SKIP-CACHE-INV:       ; %bb.0: ; %entry
868; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
869; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
870; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
871; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
872; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
873; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
874; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
875; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
876; SKIP-CACHE-INV-NEXT:    s_endpgm
877;
878; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_store:
879; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
880; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
881; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
882; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
883; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
884; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
885; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
886; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
887; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
888;
889; GFX90A-TGSPLIT-LABEL: flat_agent_release_store:
890; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
891; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
892; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
893; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
894; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
895; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
896; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
897; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
898; GFX90A-TGSPLIT-NEXT:    s_endpgm
899;
900; GFX940-NOTTGSPLIT-LABEL: flat_agent_release_store:
901; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
902; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
903; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
904; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
905; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
906; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
907; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
908; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
909; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc1
910; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
911;
912; GFX940-TGSPLIT-LABEL: flat_agent_release_store:
913; GFX940-TGSPLIT:       ; %bb.0: ; %entry
914; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
915; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
916; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
917; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
918; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
919; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
920; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
921; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc1
922; GFX940-TGSPLIT-NEXT:    s_endpgm
923;
924; GFX11-WGP-LABEL: flat_agent_release_store:
925; GFX11-WGP:       ; %bb.0: ; %entry
926; GFX11-WGP-NEXT:    s_clause 0x1
927; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
928; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
929; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
930; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
931; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
932; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
933; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
934; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
935; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
936; GFX11-WGP-NEXT:    s_endpgm
937;
938; GFX11-CU-LABEL: flat_agent_release_store:
939; GFX11-CU:       ; %bb.0: ; %entry
940; GFX11-CU-NEXT:    s_clause 0x1
941; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
942; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
943; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
944; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
945; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
946; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
947; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
948; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
949; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
950; GFX11-CU-NEXT:    s_endpgm
951    i32 %in, i32* %out) {
952entry:
953  store atomic i32 %in, i32* %out syncscope("agent") release, align 4
954  ret void
955}
956
957define amdgpu_kernel void @flat_agent_seq_cst_store(
958; GFX7-LABEL: flat_agent_seq_cst_store:
959; GFX7:       ; %bb.0: ; %entry
960; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
961; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
962; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
963; GFX7-NEXT:    v_mov_b32_e32 v0, s0
964; GFX7-NEXT:    v_mov_b32_e32 v1, s1
965; GFX7-NEXT:    v_mov_b32_e32 v2, s2
966; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
967; GFX7-NEXT:    flat_store_dword v[0:1], v2
968; GFX7-NEXT:    s_endpgm
969;
970; GFX10-WGP-LABEL: flat_agent_seq_cst_store:
971; GFX10-WGP:       ; %bb.0: ; %entry
972; GFX10-WGP-NEXT:    s_clause 0x1
973; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
974; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
975; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
976; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
977; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
978; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
979; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
980; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
981; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
982; GFX10-WGP-NEXT:    s_endpgm
983;
984; GFX10-CU-LABEL: flat_agent_seq_cst_store:
985; GFX10-CU:       ; %bb.0: ; %entry
986; GFX10-CU-NEXT:    s_clause 0x1
987; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
988; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
989; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
990; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
991; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
992; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
993; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
994; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
995; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
996; GFX10-CU-NEXT:    s_endpgm
997;
998; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_store:
999; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1000; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
1001; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
1002; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1003; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1004; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1005; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1006; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1007; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1008; SKIP-CACHE-INV-NEXT:    s_endpgm
1009;
1010; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_store:
1011; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1012; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1013; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
1014; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1015; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1016; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1017; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1018; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1019; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1020;
1021; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_store:
1022; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1023; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1024; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
1025; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1026; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1027; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1028; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1029; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1030; GFX90A-TGSPLIT-NEXT:    s_endpgm
1031;
1032; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_store:
1033; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1034; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
1035; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
1036; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1037; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1038; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1039; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
1040; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1041; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc1
1042; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1043;
1044; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_store:
1045; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1046; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
1047; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
1048; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1049; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1050; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1051; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
1052; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1053; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc1
1054; GFX940-TGSPLIT-NEXT:    s_endpgm
1055;
1056; GFX11-WGP-LABEL: flat_agent_seq_cst_store:
1057; GFX11-WGP:       ; %bb.0: ; %entry
1058; GFX11-WGP-NEXT:    s_clause 0x1
1059; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
1060; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
1061; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1062; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1063; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1064; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1065; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1066; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
1067; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1068; GFX11-WGP-NEXT:    s_endpgm
1069;
1070; GFX11-CU-LABEL: flat_agent_seq_cst_store:
1071; GFX11-CU:       ; %bb.0: ; %entry
1072; GFX11-CU-NEXT:    s_clause 0x1
1073; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
1074; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
1075; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1076; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1077; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1078; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1079; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1080; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
1081; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1082; GFX11-CU-NEXT:    s_endpgm
1083    i32 %in, i32* %out) {
1084entry:
1085  store atomic i32 %in, i32* %out syncscope("agent") seq_cst, align 4
1086  ret void
1087}
1088
1089define amdgpu_kernel void @flat_agent_monotonic_atomicrmw(
1090; GFX7-LABEL: flat_agent_monotonic_atomicrmw:
1091; GFX7:       ; %bb.0: ; %entry
1092; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1093; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1094; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1095; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1096; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1097; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1098; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1099; GFX7-NEXT:    s_endpgm
1100;
1101; GFX10-WGP-LABEL: flat_agent_monotonic_atomicrmw:
1102; GFX10-WGP:       ; %bb.0: ; %entry
1103; GFX10-WGP-NEXT:    s_clause 0x1
1104; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1105; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1106; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1107; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1108; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1109; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1110; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1111; GFX10-WGP-NEXT:    s_endpgm
1112;
1113; GFX10-CU-LABEL: flat_agent_monotonic_atomicrmw:
1114; GFX10-CU:       ; %bb.0: ; %entry
1115; GFX10-CU-NEXT:    s_clause 0x1
1116; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1117; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1118; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1119; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1120; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1121; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1122; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1123; GFX10-CU-NEXT:    s_endpgm
1124;
1125; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_atomicrmw:
1126; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1127; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1128; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1129; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1130; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1131; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1132; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1133; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1134; SKIP-CACHE-INV-NEXT:    s_endpgm
1135;
1136; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_atomicrmw:
1137; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1138; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1139; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1140; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1141; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1142; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1143; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1144; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1145;
1146; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_atomicrmw:
1147; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1148; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1149; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1150; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1151; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1152; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1153; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1154; GFX90A-TGSPLIT-NEXT:    s_endpgm
1155;
1156; GFX940-NOTTGSPLIT-LABEL: flat_agent_monotonic_atomicrmw:
1157; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1158; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1159; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1160; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1161; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1162; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1163; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1164; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1165;
1166; GFX940-TGSPLIT-LABEL: flat_agent_monotonic_atomicrmw:
1167; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1168; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1169; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1170; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1171; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1172; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1173; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1174; GFX940-TGSPLIT-NEXT:    s_endpgm
1175;
1176; GFX11-WGP-LABEL: flat_agent_monotonic_atomicrmw:
1177; GFX11-WGP:       ; %bb.0: ; %entry
1178; GFX11-WGP-NEXT:    s_clause 0x1
1179; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1180; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1181; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1182; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1183; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1184; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1185; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1186; GFX11-WGP-NEXT:    s_endpgm
1187;
1188; GFX11-CU-LABEL: flat_agent_monotonic_atomicrmw:
1189; GFX11-CU:       ; %bb.0: ; %entry
1190; GFX11-CU-NEXT:    s_clause 0x1
1191; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1192; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1193; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1194; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1195; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1196; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1197; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1198; GFX11-CU-NEXT:    s_endpgm
1199    i32* %out, i32 %in) {
1200entry:
1201  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") monotonic
1202  ret void
1203}
1204
1205define amdgpu_kernel void @flat_agent_acquire_atomicrmw(
1206; GFX7-LABEL: flat_agent_acquire_atomicrmw:
1207; GFX7:       ; %bb.0: ; %entry
1208; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1209; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1210; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1211; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1212; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1213; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1214; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1215; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1216; GFX7-NEXT:    buffer_wbinvl1_vol
1217; GFX7-NEXT:    s_endpgm
1218;
1219; GFX10-WGP-LABEL: flat_agent_acquire_atomicrmw:
1220; GFX10-WGP:       ; %bb.0: ; %entry
1221; GFX10-WGP-NEXT:    s_clause 0x1
1222; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1223; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1224; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1225; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1226; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1227; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1228; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1229; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1230; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1231; GFX10-WGP-NEXT:    buffer_gl0_inv
1232; GFX10-WGP-NEXT:    buffer_gl1_inv
1233; GFX10-WGP-NEXT:    s_endpgm
1234;
1235; GFX10-CU-LABEL: flat_agent_acquire_atomicrmw:
1236; GFX10-CU:       ; %bb.0: ; %entry
1237; GFX10-CU-NEXT:    s_clause 0x1
1238; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1239; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1240; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1241; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1242; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1243; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1244; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1245; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1246; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1247; GFX10-CU-NEXT:    buffer_gl0_inv
1248; GFX10-CU-NEXT:    buffer_gl1_inv
1249; GFX10-CU-NEXT:    s_endpgm
1250;
1251; SKIP-CACHE-INV-LABEL: flat_agent_acquire_atomicrmw:
1252; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1253; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1254; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1255; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1256; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1257; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1258; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1259; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1260; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1261; SKIP-CACHE-INV-NEXT:    s_endpgm
1262;
1263; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_atomicrmw:
1264; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1265; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1266; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1267; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1268; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1269; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1270; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1271; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1272; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1273; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1274;
1275; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_atomicrmw:
1276; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1277; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1278; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1279; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1280; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1281; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1282; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1283; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1284; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1285; GFX90A-TGSPLIT-NEXT:    s_endpgm
1286;
1287; GFX940-NOTTGSPLIT-LABEL: flat_agent_acquire_atomicrmw:
1288; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1289; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1290; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1291; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1292; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1293; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1294; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1295; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1296; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
1297; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1298;
1299; GFX940-TGSPLIT-LABEL: flat_agent_acquire_atomicrmw:
1300; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1301; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1302; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1303; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1304; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1305; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1306; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1307; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1308; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
1309; GFX940-TGSPLIT-NEXT:    s_endpgm
1310;
1311; GFX11-WGP-LABEL: flat_agent_acquire_atomicrmw:
1312; GFX11-WGP:       ; %bb.0: ; %entry
1313; GFX11-WGP-NEXT:    s_clause 0x1
1314; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1315; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1316; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1317; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1318; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1319; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1320; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1321; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1322; GFX11-WGP-NEXT:    buffer_gl0_inv
1323; GFX11-WGP-NEXT:    buffer_gl1_inv
1324; GFX11-WGP-NEXT:    s_endpgm
1325;
1326; GFX11-CU-LABEL: flat_agent_acquire_atomicrmw:
1327; GFX11-CU:       ; %bb.0: ; %entry
1328; GFX11-CU-NEXT:    s_clause 0x1
1329; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1330; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1331; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1332; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1333; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1334; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1335; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1336; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1337; GFX11-CU-NEXT:    buffer_gl0_inv
1338; GFX11-CU-NEXT:    buffer_gl1_inv
1339; GFX11-CU-NEXT:    s_endpgm
1340    i32* %out, i32 %in) {
1341entry:
1342  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") acquire
1343  ret void
1344}
1345
1346define amdgpu_kernel void @flat_agent_release_atomicrmw(
1347; GFX7-LABEL: flat_agent_release_atomicrmw:
1348; GFX7:       ; %bb.0: ; %entry
1349; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1350; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1351; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1352; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1353; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1354; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1355; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1356; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1357; GFX7-NEXT:    s_endpgm
1358;
1359; GFX10-WGP-LABEL: flat_agent_release_atomicrmw:
1360; GFX10-WGP:       ; %bb.0: ; %entry
1361; GFX10-WGP-NEXT:    s_clause 0x1
1362; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1363; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1364; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1365; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1366; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1367; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1368; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1369; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1370; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1371; GFX10-WGP-NEXT:    s_endpgm
1372;
1373; GFX10-CU-LABEL: flat_agent_release_atomicrmw:
1374; GFX10-CU:       ; %bb.0: ; %entry
1375; GFX10-CU-NEXT:    s_clause 0x1
1376; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1377; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1378; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1379; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1380; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1381; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1382; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1383; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1384; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1385; GFX10-CU-NEXT:    s_endpgm
1386;
1387; SKIP-CACHE-INV-LABEL: flat_agent_release_atomicrmw:
1388; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1389; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1390; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1391; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1392; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1393; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1394; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1395; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1396; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1397; SKIP-CACHE-INV-NEXT:    s_endpgm
1398;
1399; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_atomicrmw:
1400; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1401; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1402; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1403; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1404; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1405; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1406; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1407; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1408; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1409;
1410; GFX90A-TGSPLIT-LABEL: flat_agent_release_atomicrmw:
1411; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1412; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1413; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1414; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1415; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1416; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1417; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1418; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1419; GFX90A-TGSPLIT-NEXT:    s_endpgm
1420;
1421; GFX940-NOTTGSPLIT-LABEL: flat_agent_release_atomicrmw:
1422; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1423; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1424; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1425; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1426; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1427; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1428; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
1429; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1430; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1431; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1432;
1433; GFX940-TGSPLIT-LABEL: flat_agent_release_atomicrmw:
1434; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1435; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1436; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1437; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1438; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1439; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1440; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
1441; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1442; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1443; GFX940-TGSPLIT-NEXT:    s_endpgm
1444;
1445; GFX11-WGP-LABEL: flat_agent_release_atomicrmw:
1446; GFX11-WGP:       ; %bb.0: ; %entry
1447; GFX11-WGP-NEXT:    s_clause 0x1
1448; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1449; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1450; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1451; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1452; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1453; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1454; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1455; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1456; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1457; GFX11-WGP-NEXT:    s_endpgm
1458;
1459; GFX11-CU-LABEL: flat_agent_release_atomicrmw:
1460; GFX11-CU:       ; %bb.0: ; %entry
1461; GFX11-CU-NEXT:    s_clause 0x1
1462; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1463; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1464; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1465; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1466; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1467; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1468; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1469; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1470; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1471; GFX11-CU-NEXT:    s_endpgm
1472    i32* %out, i32 %in) {
1473entry:
1474  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") release
1475  ret void
1476}
1477
1478define amdgpu_kernel void @flat_agent_acq_rel_atomicrmw(
1479; GFX7-LABEL: flat_agent_acq_rel_atomicrmw:
1480; GFX7:       ; %bb.0: ; %entry
1481; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1482; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1483; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1484; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1485; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1486; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1487; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1488; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1489; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1490; GFX7-NEXT:    buffer_wbinvl1_vol
1491; GFX7-NEXT:    s_endpgm
1492;
1493; GFX10-WGP-LABEL: flat_agent_acq_rel_atomicrmw:
1494; GFX10-WGP:       ; %bb.0: ; %entry
1495; GFX10-WGP-NEXT:    s_clause 0x1
1496; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1497; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1498; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1499; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1500; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1501; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1502; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1503; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1504; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1505; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1506; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1507; GFX10-WGP-NEXT:    buffer_gl0_inv
1508; GFX10-WGP-NEXT:    buffer_gl1_inv
1509; GFX10-WGP-NEXT:    s_endpgm
1510;
1511; GFX10-CU-LABEL: flat_agent_acq_rel_atomicrmw:
1512; GFX10-CU:       ; %bb.0: ; %entry
1513; GFX10-CU-NEXT:    s_clause 0x1
1514; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1515; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1516; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1517; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1518; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1519; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1520; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1521; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1522; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1523; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1524; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1525; GFX10-CU-NEXT:    buffer_gl0_inv
1526; GFX10-CU-NEXT:    buffer_gl1_inv
1527; GFX10-CU-NEXT:    s_endpgm
1528;
1529; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_atomicrmw:
1530; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1531; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1532; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1533; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1534; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1535; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1536; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1537; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1538; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1539; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1540; SKIP-CACHE-INV-NEXT:    s_endpgm
1541;
1542; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_atomicrmw:
1543; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1544; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1545; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1546; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1547; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1548; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1549; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1550; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1551; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1552; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1553; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1554;
1555; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_atomicrmw:
1556; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1557; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1558; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1559; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1560; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1561; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1562; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1563; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1564; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1565; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1566; GFX90A-TGSPLIT-NEXT:    s_endpgm
1567;
1568; GFX940-NOTTGSPLIT-LABEL: flat_agent_acq_rel_atomicrmw:
1569; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1570; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1571; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1572; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1573; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1574; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1575; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
1576; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1577; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1578; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1579; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
1580; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1581;
1582; GFX940-TGSPLIT-LABEL: flat_agent_acq_rel_atomicrmw:
1583; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1584; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1585; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1586; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1587; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1588; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1589; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
1590; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1591; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1592; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1593; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
1594; GFX940-TGSPLIT-NEXT:    s_endpgm
1595;
1596; GFX11-WGP-LABEL: flat_agent_acq_rel_atomicrmw:
1597; GFX11-WGP:       ; %bb.0: ; %entry
1598; GFX11-WGP-NEXT:    s_clause 0x1
1599; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1600; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1601; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1602; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1603; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1604; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1605; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1606; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1607; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1608; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1609; GFX11-WGP-NEXT:    buffer_gl0_inv
1610; GFX11-WGP-NEXT:    buffer_gl1_inv
1611; GFX11-WGP-NEXT:    s_endpgm
1612;
1613; GFX11-CU-LABEL: flat_agent_acq_rel_atomicrmw:
1614; GFX11-CU:       ; %bb.0: ; %entry
1615; GFX11-CU-NEXT:    s_clause 0x1
1616; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1617; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1618; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1619; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1620; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1621; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1622; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1623; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1624; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1625; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1626; GFX11-CU-NEXT:    buffer_gl0_inv
1627; GFX11-CU-NEXT:    buffer_gl1_inv
1628; GFX11-CU-NEXT:    s_endpgm
1629    i32* %out, i32 %in) {
1630entry:
1631  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") acq_rel
1632  ret void
1633}
1634
1635define amdgpu_kernel void @flat_agent_seq_cst_atomicrmw(
1636; GFX7-LABEL: flat_agent_seq_cst_atomicrmw:
1637; GFX7:       ; %bb.0: ; %entry
1638; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1639; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1640; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1641; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1642; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1643; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1644; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1645; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1646; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1647; GFX7-NEXT:    buffer_wbinvl1_vol
1648; GFX7-NEXT:    s_endpgm
1649;
1650; GFX10-WGP-LABEL: flat_agent_seq_cst_atomicrmw:
1651; GFX10-WGP:       ; %bb.0: ; %entry
1652; GFX10-WGP-NEXT:    s_clause 0x1
1653; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1654; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1655; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1656; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1657; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1658; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1659; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1660; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1661; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1662; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1663; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1664; GFX10-WGP-NEXT:    buffer_gl0_inv
1665; GFX10-WGP-NEXT:    buffer_gl1_inv
1666; GFX10-WGP-NEXT:    s_endpgm
1667;
1668; GFX10-CU-LABEL: flat_agent_seq_cst_atomicrmw:
1669; GFX10-CU:       ; %bb.0: ; %entry
1670; GFX10-CU-NEXT:    s_clause 0x1
1671; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1672; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1673; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1674; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1675; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1676; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1677; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1678; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1679; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1680; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1681; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1682; GFX10-CU-NEXT:    buffer_gl0_inv
1683; GFX10-CU-NEXT:    buffer_gl1_inv
1684; GFX10-CU-NEXT:    s_endpgm
1685;
1686; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_atomicrmw:
1687; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1688; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1689; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1690; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1691; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1692; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1693; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1694; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1695; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1696; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1697; SKIP-CACHE-INV-NEXT:    s_endpgm
1698;
1699; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_atomicrmw:
1700; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1701; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1702; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1703; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1704; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1705; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1706; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1707; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1708; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1709; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1710; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1711;
1712; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_atomicrmw:
1713; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1714; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1715; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1716; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1717; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1718; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1719; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1720; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1721; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1722; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1723; GFX90A-TGSPLIT-NEXT:    s_endpgm
1724;
1725; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_atomicrmw:
1726; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1727; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1728; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1729; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1730; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1731; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1732; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
1733; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1734; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1735; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1736; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
1737; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1738;
1739; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_atomicrmw:
1740; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1741; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1742; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1743; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1744; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1745; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1746; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
1747; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1748; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1749; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1750; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
1751; GFX940-TGSPLIT-NEXT:    s_endpgm
1752;
1753; GFX11-WGP-LABEL: flat_agent_seq_cst_atomicrmw:
1754; GFX11-WGP:       ; %bb.0: ; %entry
1755; GFX11-WGP-NEXT:    s_clause 0x1
1756; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1757; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1758; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1759; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1760; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1761; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1762; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1763; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1764; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1765; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1766; GFX11-WGP-NEXT:    buffer_gl0_inv
1767; GFX11-WGP-NEXT:    buffer_gl1_inv
1768; GFX11-WGP-NEXT:    s_endpgm
1769;
1770; GFX11-CU-LABEL: flat_agent_seq_cst_atomicrmw:
1771; GFX11-CU:       ; %bb.0: ; %entry
1772; GFX11-CU-NEXT:    s_clause 0x1
1773; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1774; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1775; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1776; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1777; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1778; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1779; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1780; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1781; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1782; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1783; GFX11-CU-NEXT:    buffer_gl0_inv
1784; GFX11-CU-NEXT:    buffer_gl1_inv
1785; GFX11-CU-NEXT:    s_endpgm
1786    i32* %out, i32 %in) {
1787entry:
1788  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") seq_cst
1789  ret void
1790}
1791
1792define amdgpu_kernel void @flat_agent_acquire_ret_atomicrmw(
1793; GFX7-LABEL: flat_agent_acquire_ret_atomicrmw:
1794; GFX7:       ; %bb.0: ; %entry
1795; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1796; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1797; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1798; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1799; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1800; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1801; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1802; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1803; GFX7-NEXT:    buffer_wbinvl1_vol
1804; GFX7-NEXT:    flat_store_dword v[0:1], v2
1805; GFX7-NEXT:    s_endpgm
1806;
1807; GFX10-WGP-LABEL: flat_agent_acquire_ret_atomicrmw:
1808; GFX10-WGP:       ; %bb.0: ; %entry
1809; GFX10-WGP-NEXT:    s_clause 0x1
1810; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1811; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1812; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1813; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1814; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1815; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1816; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1817; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1818; GFX10-WGP-NEXT:    buffer_gl0_inv
1819; GFX10-WGP-NEXT:    buffer_gl1_inv
1820; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1821; GFX10-WGP-NEXT:    s_endpgm
1822;
1823; GFX10-CU-LABEL: flat_agent_acquire_ret_atomicrmw:
1824; GFX10-CU:       ; %bb.0: ; %entry
1825; GFX10-CU-NEXT:    s_clause 0x1
1826; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1827; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1828; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1829; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1830; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1831; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1832; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1833; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1834; GFX10-CU-NEXT:    buffer_gl0_inv
1835; GFX10-CU-NEXT:    buffer_gl1_inv
1836; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1837; GFX10-CU-NEXT:    s_endpgm
1838;
1839; SKIP-CACHE-INV-LABEL: flat_agent_acquire_ret_atomicrmw:
1840; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1841; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1842; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1843; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1844; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1845; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1846; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1847; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1848; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1849; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1850; SKIP-CACHE-INV-NEXT:    s_endpgm
1851;
1852; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_ret_atomicrmw:
1853; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1854; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1855; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1856; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1857; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1858; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1859; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1860; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1861; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1862; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1863; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1864;
1865; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_ret_atomicrmw:
1866; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1867; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1868; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1869; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1870; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1871; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1872; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1873; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1874; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1875; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1876; GFX90A-TGSPLIT-NEXT:    s_endpgm
1877;
1878; GFX940-NOTTGSPLIT-LABEL: flat_agent_acquire_ret_atomicrmw:
1879; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1880; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1881; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1882; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1883; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1884; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1885; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
1886; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1887; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
1888; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1889; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1890;
1891; GFX940-TGSPLIT-LABEL: flat_agent_acquire_ret_atomicrmw:
1892; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1893; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1894; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1895; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1896; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1897; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1898; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
1899; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1900; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
1901; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1902; GFX940-TGSPLIT-NEXT:    s_endpgm
1903;
1904; GFX11-WGP-LABEL: flat_agent_acquire_ret_atomicrmw:
1905; GFX11-WGP:       ; %bb.0: ; %entry
1906; GFX11-WGP-NEXT:    s_clause 0x1
1907; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1908; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1909; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1910; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1911; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1912; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
1913; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1914; GFX11-WGP-NEXT:    buffer_gl0_inv
1915; GFX11-WGP-NEXT:    buffer_gl1_inv
1916; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
1917; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1918; GFX11-WGP-NEXT:    s_endpgm
1919;
1920; GFX11-CU-LABEL: flat_agent_acquire_ret_atomicrmw:
1921; GFX11-CU:       ; %bb.0: ; %entry
1922; GFX11-CU-NEXT:    s_clause 0x1
1923; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1924; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1925; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1926; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1927; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1928; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
1929; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1930; GFX11-CU-NEXT:    buffer_gl0_inv
1931; GFX11-CU-NEXT:    buffer_gl1_inv
1932; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
1933; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1934; GFX11-CU-NEXT:    s_endpgm
1935    i32* %out, i32 %in) {
1936entry:
1937  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") acquire
1938  store i32 %val, i32* %out, align 4
1939  ret void
1940}
1941
1942define amdgpu_kernel void @flat_agent_acq_rel_ret_atomicrmw(
1943; GFX7-LABEL: flat_agent_acq_rel_ret_atomicrmw:
1944; GFX7:       ; %bb.0: ; %entry
1945; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1946; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1947; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1948; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1949; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1950; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1951; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1952; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1953; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1954; GFX7-NEXT:    buffer_wbinvl1_vol
1955; GFX7-NEXT:    flat_store_dword v[0:1], v2
1956; GFX7-NEXT:    s_endpgm
1957;
1958; GFX10-WGP-LABEL: flat_agent_acq_rel_ret_atomicrmw:
1959; GFX10-WGP:       ; %bb.0: ; %entry
1960; GFX10-WGP-NEXT:    s_clause 0x1
1961; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1962; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1963; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1964; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1965; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1966; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1967; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1968; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1969; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1970; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1971; GFX10-WGP-NEXT:    buffer_gl0_inv
1972; GFX10-WGP-NEXT:    buffer_gl1_inv
1973; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1974; GFX10-WGP-NEXT:    s_endpgm
1975;
1976; GFX10-CU-LABEL: flat_agent_acq_rel_ret_atomicrmw:
1977; GFX10-CU:       ; %bb.0: ; %entry
1978; GFX10-CU-NEXT:    s_clause 0x1
1979; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1980; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1981; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1982; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1983; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1984; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1985; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1986; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1987; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1988; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1989; GFX10-CU-NEXT:    buffer_gl0_inv
1990; GFX10-CU-NEXT:    buffer_gl1_inv
1991; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1992; GFX10-CU-NEXT:    s_endpgm
1993;
1994; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_ret_atomicrmw:
1995; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1996; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1997; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1998; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1999; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2000; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2001; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2002; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2003; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2004; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2005; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
2006; SKIP-CACHE-INV-NEXT:    s_endpgm
2007;
2008; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_ret_atomicrmw:
2009; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2010; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2011; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
2012; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2013; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2014; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
2015; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2016; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2017; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2018; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2019; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2020; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2021;
2022; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_ret_atomicrmw:
2023; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2024; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2025; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
2026; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2027; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2028; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
2029; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2030; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2031; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2032; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2033; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2034; GFX90A-TGSPLIT-NEXT:    s_endpgm
2035;
2036; GFX940-NOTTGSPLIT-LABEL: flat_agent_acq_rel_ret_atomicrmw:
2037; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2038; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2039; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
2040; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2041; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2042; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
2043; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
2044; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2045; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
2046; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2047; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
2048; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2049; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2050;
2051; GFX940-TGSPLIT-LABEL: flat_agent_acq_rel_ret_atomicrmw:
2052; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2053; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2054; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
2055; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2056; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2057; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
2058; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
2059; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2060; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
2061; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2062; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
2063; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2064; GFX940-TGSPLIT-NEXT:    s_endpgm
2065;
2066; GFX11-WGP-LABEL: flat_agent_acq_rel_ret_atomicrmw:
2067; GFX11-WGP:       ; %bb.0: ; %entry
2068; GFX11-WGP-NEXT:    s_clause 0x1
2069; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2070; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
2071; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2072; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2073; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2074; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2075; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2076; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
2077; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2078; GFX11-WGP-NEXT:    buffer_gl0_inv
2079; GFX11-WGP-NEXT:    buffer_gl1_inv
2080; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
2081; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2082; GFX11-WGP-NEXT:    s_endpgm
2083;
2084; GFX11-CU-LABEL: flat_agent_acq_rel_ret_atomicrmw:
2085; GFX11-CU:       ; %bb.0: ; %entry
2086; GFX11-CU-NEXT:    s_clause 0x1
2087; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2088; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
2089; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2090; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2091; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2092; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2093; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2094; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
2095; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2096; GFX11-CU-NEXT:    buffer_gl0_inv
2097; GFX11-CU-NEXT:    buffer_gl1_inv
2098; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
2099; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2100; GFX11-CU-NEXT:    s_endpgm
2101    i32* %out, i32 %in) {
2102entry:
2103  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") acq_rel
2104  store i32 %val, i32* %out, align 4
2105  ret void
2106}
2107
2108define amdgpu_kernel void @flat_agent_seq_cst_ret_atomicrmw(
2109; GFX7-LABEL: flat_agent_seq_cst_ret_atomicrmw:
2110; GFX7:       ; %bb.0: ; %entry
2111; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2112; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
2113; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2114; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2115; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2116; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2117; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2118; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2119; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2120; GFX7-NEXT:    buffer_wbinvl1_vol
2121; GFX7-NEXT:    flat_store_dword v[0:1], v2
2122; GFX7-NEXT:    s_endpgm
2123;
2124; GFX10-WGP-LABEL: flat_agent_seq_cst_ret_atomicrmw:
2125; GFX10-WGP:       ; %bb.0: ; %entry
2126; GFX10-WGP-NEXT:    s_clause 0x1
2127; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2128; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
2129; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2130; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2131; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2132; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2133; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2134; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2135; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2136; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2137; GFX10-WGP-NEXT:    buffer_gl0_inv
2138; GFX10-WGP-NEXT:    buffer_gl1_inv
2139; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
2140; GFX10-WGP-NEXT:    s_endpgm
2141;
2142; GFX10-CU-LABEL: flat_agent_seq_cst_ret_atomicrmw:
2143; GFX10-CU:       ; %bb.0: ; %entry
2144; GFX10-CU-NEXT:    s_clause 0x1
2145; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2146; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
2147; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2148; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2149; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2150; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2151; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2152; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2153; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2154; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2155; GFX10-CU-NEXT:    buffer_gl0_inv
2156; GFX10-CU-NEXT:    buffer_gl1_inv
2157; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
2158; GFX10-CU-NEXT:    s_endpgm
2159;
2160; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_ret_atomicrmw:
2161; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2162; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2163; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
2164; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2165; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2166; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2167; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2168; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2169; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2170; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2171; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
2172; SKIP-CACHE-INV-NEXT:    s_endpgm
2173;
2174; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_ret_atomicrmw:
2175; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2176; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2177; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
2178; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2179; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2180; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
2181; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2182; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2183; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2184; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2185; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2186; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2187;
2188; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_ret_atomicrmw:
2189; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2190; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2191; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
2192; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2193; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2194; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
2195; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2196; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2197; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2198; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2199; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2200; GFX90A-TGSPLIT-NEXT:    s_endpgm
2201;
2202; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_ret_atomicrmw:
2203; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2204; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2205; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
2206; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2207; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2208; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
2209; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
2210; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2211; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
2212; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2213; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
2214; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2215; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2216;
2217; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_ret_atomicrmw:
2218; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2219; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2220; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
2221; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2222; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2223; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
2224; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
2225; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2226; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
2227; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2228; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
2229; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2230; GFX940-TGSPLIT-NEXT:    s_endpgm
2231;
2232; GFX11-WGP-LABEL: flat_agent_seq_cst_ret_atomicrmw:
2233; GFX11-WGP:       ; %bb.0: ; %entry
2234; GFX11-WGP-NEXT:    s_clause 0x1
2235; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2236; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
2237; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2238; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2239; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2240; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2241; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2242; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
2243; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2244; GFX11-WGP-NEXT:    buffer_gl0_inv
2245; GFX11-WGP-NEXT:    buffer_gl1_inv
2246; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
2247; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2248; GFX11-WGP-NEXT:    s_endpgm
2249;
2250; GFX11-CU-LABEL: flat_agent_seq_cst_ret_atomicrmw:
2251; GFX11-CU:       ; %bb.0: ; %entry
2252; GFX11-CU-NEXT:    s_clause 0x1
2253; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2254; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
2255; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2256; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2257; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2258; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2259; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2260; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
2261; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2262; GFX11-CU-NEXT:    buffer_gl0_inv
2263; GFX11-CU-NEXT:    buffer_gl1_inv
2264; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
2265; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2266; GFX11-CU-NEXT:    s_endpgm
2267    i32* %out, i32 %in) {
2268entry:
2269  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") seq_cst
2270  store i32 %val, i32* %out, align 4
2271  ret void
2272}
2273
2274define amdgpu_kernel void @flat_agent_monotonic_monotonic_cmpxchg(
2275; GFX7-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
2276; GFX7:       ; %bb.0: ; %entry
2277; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2278; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2279; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2280; GFX7-NEXT:    s_add_u32 s0, s0, 16
2281; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2282; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2283; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2284; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2285; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2286; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2287; GFX7-NEXT:    s_endpgm
2288;
2289; GFX10-WGP-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
2290; GFX10-WGP:       ; %bb.0: ; %entry
2291; GFX10-WGP-NEXT:    s_clause 0x1
2292; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2293; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2294; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2295; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2296; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2297; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2298; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2299; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2300; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2301; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2302; GFX10-WGP-NEXT:    s_endpgm
2303;
2304; GFX10-CU-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
2305; GFX10-CU:       ; %bb.0: ; %entry
2306; GFX10-CU-NEXT:    s_clause 0x1
2307; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2308; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2309; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2310; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2311; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2312; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2313; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2314; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2315; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2316; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2317; GFX10-CU-NEXT:    s_endpgm
2318;
2319; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
2320; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2321; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2322; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2323; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2324; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2325; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2326; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2327; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2328; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2329; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2330; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2331; SKIP-CACHE-INV-NEXT:    s_endpgm
2332;
2333; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
2334; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2335; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2336; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2337; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2338; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2339; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2340; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2341; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2342;
2343; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
2344; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2345; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2346; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2347; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2348; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2349; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2350; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2351; GFX90A-TGSPLIT-NEXT:    s_endpgm
2352;
2353; GFX940-NOTTGSPLIT-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
2354; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2355; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2356; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2357; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2358; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2359; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2360; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2361; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2362;
2363; GFX940-TGSPLIT-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
2364; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2365; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2366; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2367; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2368; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2369; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2370; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2371; GFX940-TGSPLIT-NEXT:    s_endpgm
2372;
2373; GFX11-WGP-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
2374; GFX11-WGP:       ; %bb.0: ; %entry
2375; GFX11-WGP-NEXT:    s_clause 0x1
2376; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2377; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2378; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2379; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2380; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
2381; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2382; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2383; GFX11-WGP-NEXT:    s_endpgm
2384;
2385; GFX11-CU-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
2386; GFX11-CU:       ; %bb.0: ; %entry
2387; GFX11-CU-NEXT:    s_clause 0x1
2388; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2389; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2390; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2391; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2392; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
2393; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2394; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2395; GFX11-CU-NEXT:    s_endpgm
2396    i32* %out, i32 %in, i32 %old) {
2397entry:
2398  %gep = getelementptr i32, i32* %out, i32 4
2399  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") monotonic monotonic
2400  ret void
2401}
2402
2403define amdgpu_kernel void @flat_agent_acquire_monotonic_cmpxchg(
2404; GFX7-LABEL: flat_agent_acquire_monotonic_cmpxchg:
2405; GFX7:       ; %bb.0: ; %entry
2406; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2407; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2408; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2409; GFX7-NEXT:    s_add_u32 s0, s0, 16
2410; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2411; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2412; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2413; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2414; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2415; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2416; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2417; GFX7-NEXT:    buffer_wbinvl1_vol
2418; GFX7-NEXT:    s_endpgm
2419;
2420; GFX10-WGP-LABEL: flat_agent_acquire_monotonic_cmpxchg:
2421; GFX10-WGP:       ; %bb.0: ; %entry
2422; GFX10-WGP-NEXT:    s_clause 0x1
2423; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2424; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2425; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2426; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2427; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2428; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2429; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2430; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2431; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2432; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2433; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2434; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2435; GFX10-WGP-NEXT:    buffer_gl0_inv
2436; GFX10-WGP-NEXT:    buffer_gl1_inv
2437; GFX10-WGP-NEXT:    s_endpgm
2438;
2439; GFX10-CU-LABEL: flat_agent_acquire_monotonic_cmpxchg:
2440; GFX10-CU:       ; %bb.0: ; %entry
2441; GFX10-CU-NEXT:    s_clause 0x1
2442; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2443; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2444; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2445; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2446; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2447; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2448; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2449; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2450; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2451; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2452; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2453; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2454; GFX10-CU-NEXT:    buffer_gl0_inv
2455; GFX10-CU-NEXT:    buffer_gl1_inv
2456; GFX10-CU-NEXT:    s_endpgm
2457;
2458; SKIP-CACHE-INV-LABEL: flat_agent_acquire_monotonic_cmpxchg:
2459; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2460; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2461; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2462; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2463; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2464; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2465; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2466; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2467; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2468; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2469; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2470; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2471; SKIP-CACHE-INV-NEXT:    s_endpgm
2472;
2473; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_monotonic_cmpxchg:
2474; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2475; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2476; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2477; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2478; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2479; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2480; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2481; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2482; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2483; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2484;
2485; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_monotonic_cmpxchg:
2486; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2487; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2488; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2489; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2490; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2491; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2492; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2493; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2494; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2495; GFX90A-TGSPLIT-NEXT:    s_endpgm
2496;
2497; GFX940-NOTTGSPLIT-LABEL: flat_agent_acquire_monotonic_cmpxchg:
2498; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2499; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2500; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2501; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2502; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2503; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2504; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2505; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2506; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
2507; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2508;
2509; GFX940-TGSPLIT-LABEL: flat_agent_acquire_monotonic_cmpxchg:
2510; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2511; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2512; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2513; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2514; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2515; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2516; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2517; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2518; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
2519; GFX940-TGSPLIT-NEXT:    s_endpgm
2520;
2521; GFX11-WGP-LABEL: flat_agent_acquire_monotonic_cmpxchg:
2522; GFX11-WGP:       ; %bb.0: ; %entry
2523; GFX11-WGP-NEXT:    s_clause 0x1
2524; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2525; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2526; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2527; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2528; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
2529; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2530; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2531; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2532; GFX11-WGP-NEXT:    buffer_gl0_inv
2533; GFX11-WGP-NEXT:    buffer_gl1_inv
2534; GFX11-WGP-NEXT:    s_endpgm
2535;
2536; GFX11-CU-LABEL: flat_agent_acquire_monotonic_cmpxchg:
2537; GFX11-CU:       ; %bb.0: ; %entry
2538; GFX11-CU-NEXT:    s_clause 0x1
2539; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2540; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2541; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2542; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2543; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
2544; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2545; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2546; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2547; GFX11-CU-NEXT:    buffer_gl0_inv
2548; GFX11-CU-NEXT:    buffer_gl1_inv
2549; GFX11-CU-NEXT:    s_endpgm
2550    i32* %out, i32 %in, i32 %old) {
2551entry:
2552  %gep = getelementptr i32, i32* %out, i32 4
2553  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire monotonic
2554  ret void
2555}
2556
2557define amdgpu_kernel void @flat_agent_release_monotonic_cmpxchg(
2558; GFX7-LABEL: flat_agent_release_monotonic_cmpxchg:
2559; GFX7:       ; %bb.0: ; %entry
2560; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2561; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2562; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2563; GFX7-NEXT:    s_add_u32 s0, s0, 16
2564; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2565; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2566; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2567; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2568; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2569; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2570; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2571; GFX7-NEXT:    s_endpgm
2572;
2573; GFX10-WGP-LABEL: flat_agent_release_monotonic_cmpxchg:
2574; GFX10-WGP:       ; %bb.0: ; %entry
2575; GFX10-WGP-NEXT:    s_clause 0x1
2576; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2577; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2578; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2579; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2580; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2581; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2582; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2583; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2584; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2585; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2586; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2587; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2588; GFX10-WGP-NEXT:    s_endpgm
2589;
2590; GFX10-CU-LABEL: flat_agent_release_monotonic_cmpxchg:
2591; GFX10-CU:       ; %bb.0: ; %entry
2592; GFX10-CU-NEXT:    s_clause 0x1
2593; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2594; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2595; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2596; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2597; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2598; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2599; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2600; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2601; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2602; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2603; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2604; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2605; GFX10-CU-NEXT:    s_endpgm
2606;
2607; SKIP-CACHE-INV-LABEL: flat_agent_release_monotonic_cmpxchg:
2608; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2609; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2610; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2611; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2612; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2613; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2614; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2615; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2616; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2617; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2618; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2619; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2620; SKIP-CACHE-INV-NEXT:    s_endpgm
2621;
2622; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_monotonic_cmpxchg:
2623; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2624; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2625; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2626; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2627; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2628; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2629; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2630; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2631; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2632;
2633; GFX90A-TGSPLIT-LABEL: flat_agent_release_monotonic_cmpxchg:
2634; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2635; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2636; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2637; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2638; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2639; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2640; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2641; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2642; GFX90A-TGSPLIT-NEXT:    s_endpgm
2643;
2644; GFX940-NOTTGSPLIT-LABEL: flat_agent_release_monotonic_cmpxchg:
2645; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2646; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2647; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2648; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2649; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2650; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2651; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
2652; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2653; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2654; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2655;
2656; GFX940-TGSPLIT-LABEL: flat_agent_release_monotonic_cmpxchg:
2657; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2658; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2659; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2660; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2661; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2662; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2663; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
2664; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2665; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2666; GFX940-TGSPLIT-NEXT:    s_endpgm
2667;
2668; GFX11-WGP-LABEL: flat_agent_release_monotonic_cmpxchg:
2669; GFX11-WGP:       ; %bb.0: ; %entry
2670; GFX11-WGP-NEXT:    s_clause 0x1
2671; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2672; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2673; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2674; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2675; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
2676; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2677; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2678; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2679; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2680; GFX11-WGP-NEXT:    s_endpgm
2681;
2682; GFX11-CU-LABEL: flat_agent_release_monotonic_cmpxchg:
2683; GFX11-CU:       ; %bb.0: ; %entry
2684; GFX11-CU-NEXT:    s_clause 0x1
2685; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2686; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2687; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2688; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2689; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
2690; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2691; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2692; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2693; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2694; GFX11-CU-NEXT:    s_endpgm
2695    i32* %out, i32 %in, i32 %old) {
2696entry:
2697  %gep = getelementptr i32, i32* %out, i32 4
2698  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release monotonic
2699  ret void
2700}
2701
2702define amdgpu_kernel void @flat_agent_acq_rel_monotonic_cmpxchg(
2703; GFX7-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
2704; GFX7:       ; %bb.0: ; %entry
2705; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2706; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2707; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2708; GFX7-NEXT:    s_add_u32 s0, s0, 16
2709; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2710; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2711; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2712; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2713; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2714; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2715; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2716; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2717; GFX7-NEXT:    buffer_wbinvl1_vol
2718; GFX7-NEXT:    s_endpgm
2719;
2720; GFX10-WGP-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
2721; GFX10-WGP:       ; %bb.0: ; %entry
2722; GFX10-WGP-NEXT:    s_clause 0x1
2723; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2724; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2725; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2726; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2727; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2728; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2729; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2730; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2731; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2732; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2733; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2734; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2735; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2736; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2737; GFX10-WGP-NEXT:    buffer_gl0_inv
2738; GFX10-WGP-NEXT:    buffer_gl1_inv
2739; GFX10-WGP-NEXT:    s_endpgm
2740;
2741; GFX10-CU-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
2742; GFX10-CU:       ; %bb.0: ; %entry
2743; GFX10-CU-NEXT:    s_clause 0x1
2744; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2745; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2746; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2747; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2748; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2749; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2750; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2751; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2752; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2753; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2754; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2755; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2756; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2757; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2758; GFX10-CU-NEXT:    buffer_gl0_inv
2759; GFX10-CU-NEXT:    buffer_gl1_inv
2760; GFX10-CU-NEXT:    s_endpgm
2761;
2762; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
2763; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2764; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2765; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2766; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2767; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2768; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2769; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2770; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2771; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2772; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2773; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2774; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2775; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2776; SKIP-CACHE-INV-NEXT:    s_endpgm
2777;
2778; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
2779; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2780; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2781; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2782; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2783; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2784; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2785; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2786; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2787; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2788; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2789; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2790;
2791; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
2792; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2793; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2794; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2795; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2796; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2797; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2798; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2799; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2800; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2801; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2802; GFX90A-TGSPLIT-NEXT:    s_endpgm
2803;
2804; GFX940-NOTTGSPLIT-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
2805; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2806; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2807; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2808; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2809; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2810; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2811; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
2812; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2813; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2814; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2815; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
2816; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2817;
2818; GFX940-TGSPLIT-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
2819; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2820; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2821; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2822; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2823; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2824; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2825; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
2826; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2827; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2828; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2829; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
2830; GFX940-TGSPLIT-NEXT:    s_endpgm
2831;
2832; GFX11-WGP-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
2833; GFX11-WGP:       ; %bb.0: ; %entry
2834; GFX11-WGP-NEXT:    s_clause 0x1
2835; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2836; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2837; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2838; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2839; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
2840; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2841; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2842; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2843; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2844; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2845; GFX11-WGP-NEXT:    buffer_gl0_inv
2846; GFX11-WGP-NEXT:    buffer_gl1_inv
2847; GFX11-WGP-NEXT:    s_endpgm
2848;
2849; GFX11-CU-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
2850; GFX11-CU:       ; %bb.0: ; %entry
2851; GFX11-CU-NEXT:    s_clause 0x1
2852; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2853; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2854; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2855; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
2856; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
2857; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2858; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2859; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2860; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2861; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2862; GFX11-CU-NEXT:    buffer_gl0_inv
2863; GFX11-CU-NEXT:    buffer_gl1_inv
2864; GFX11-CU-NEXT:    s_endpgm
2865    i32* %out, i32 %in, i32 %old) {
2866entry:
2867  %gep = getelementptr i32, i32* %out, i32 4
2868  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel monotonic
2869  ret void
2870}
2871
2872define amdgpu_kernel void @flat_agent_seq_cst_monotonic_cmpxchg(
2873; GFX7-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
2874; GFX7:       ; %bb.0: ; %entry
2875; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2876; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2877; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2878; GFX7-NEXT:    s_add_u32 s0, s0, 16
2879; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2880; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2881; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2882; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2883; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2884; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2885; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2886; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2887; GFX7-NEXT:    buffer_wbinvl1_vol
2888; GFX7-NEXT:    s_endpgm
2889;
2890; GFX10-WGP-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
2891; GFX10-WGP:       ; %bb.0: ; %entry
2892; GFX10-WGP-NEXT:    s_clause 0x1
2893; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2894; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2895; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2896; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2897; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2898; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2899; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2900; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2901; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2902; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2903; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2904; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2905; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2906; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2907; GFX10-WGP-NEXT:    buffer_gl0_inv
2908; GFX10-WGP-NEXT:    buffer_gl1_inv
2909; GFX10-WGP-NEXT:    s_endpgm
2910;
2911; GFX10-CU-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
2912; GFX10-CU:       ; %bb.0: ; %entry
2913; GFX10-CU-NEXT:    s_clause 0x1
2914; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2915; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2916; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2917; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2918; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2919; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2920; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2921; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2922; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2923; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2924; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2925; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2926; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2927; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2928; GFX10-CU-NEXT:    buffer_gl0_inv
2929; GFX10-CU-NEXT:    buffer_gl1_inv
2930; GFX10-CU-NEXT:    s_endpgm
2931;
2932; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
2933; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2934; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2935; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2936; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2937; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2938; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2939; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2940; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2941; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2942; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2943; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2944; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2945; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2946; SKIP-CACHE-INV-NEXT:    s_endpgm
2947;
2948; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
2949; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2950; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2951; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2952; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2953; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2954; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2955; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2956; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2957; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2958; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2959; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2960;
2961; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
2962; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2963; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2964; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2965; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2966; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2967; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2968; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2969; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2970; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2971; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2972; GFX90A-TGSPLIT-NEXT:    s_endpgm
2973;
2974; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
2975; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2976; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2977; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2978; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2979; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2980; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2981; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
2982; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2983; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2984; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2985; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
2986; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2987;
2988; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
2989; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2990; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2991; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2992; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2993; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2994; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2995; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
2996; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2997; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2998; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2999; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
3000; GFX940-TGSPLIT-NEXT:    s_endpgm
3001;
3002; GFX11-WGP-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
3003; GFX11-WGP:       ; %bb.0: ; %entry
3004; GFX11-WGP-NEXT:    s_clause 0x1
3005; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3006; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3007; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3008; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3009; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3010; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3011; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3012; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3013; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3014; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3015; GFX11-WGP-NEXT:    buffer_gl0_inv
3016; GFX11-WGP-NEXT:    buffer_gl1_inv
3017; GFX11-WGP-NEXT:    s_endpgm
3018;
3019; GFX11-CU-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
3020; GFX11-CU:       ; %bb.0: ; %entry
3021; GFX11-CU-NEXT:    s_clause 0x1
3022; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3023; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3024; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3025; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3026; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3027; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3028; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3029; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3030; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3031; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3032; GFX11-CU-NEXT:    buffer_gl0_inv
3033; GFX11-CU-NEXT:    buffer_gl1_inv
3034; GFX11-CU-NEXT:    s_endpgm
3035    i32* %out, i32 %in, i32 %old) {
3036entry:
3037  %gep = getelementptr i32, i32* %out, i32 4
3038  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst monotonic
3039  ret void
3040}
3041
3042define amdgpu_kernel void @flat_agent_monotonic_acquire_cmpxchg(
3043; GFX7-LABEL: flat_agent_monotonic_acquire_cmpxchg:
3044; GFX7:       ; %bb.0: ; %entry
3045; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3046; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3047; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3048; GFX7-NEXT:    s_add_u32 s0, s0, 16
3049; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3050; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3051; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3052; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3053; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3054; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3055; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3056; GFX7-NEXT:    buffer_wbinvl1_vol
3057; GFX7-NEXT:    s_endpgm
3058;
3059; GFX10-WGP-LABEL: flat_agent_monotonic_acquire_cmpxchg:
3060; GFX10-WGP:       ; %bb.0: ; %entry
3061; GFX10-WGP-NEXT:    s_clause 0x1
3062; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3063; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3064; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3065; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3066; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3067; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3068; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3069; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3070; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3071; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3072; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3073; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3074; GFX10-WGP-NEXT:    buffer_gl0_inv
3075; GFX10-WGP-NEXT:    buffer_gl1_inv
3076; GFX10-WGP-NEXT:    s_endpgm
3077;
3078; GFX10-CU-LABEL: flat_agent_monotonic_acquire_cmpxchg:
3079; GFX10-CU:       ; %bb.0: ; %entry
3080; GFX10-CU-NEXT:    s_clause 0x1
3081; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3082; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3083; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3084; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3085; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3086; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3087; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3088; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3089; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3090; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3091; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3092; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3093; GFX10-CU-NEXT:    buffer_gl0_inv
3094; GFX10-CU-NEXT:    buffer_gl1_inv
3095; GFX10-CU-NEXT:    s_endpgm
3096;
3097; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_acquire_cmpxchg:
3098; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3099; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3100; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3101; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3102; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3103; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3104; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3105; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3106; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3107; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3108; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3109; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3110; SKIP-CACHE-INV-NEXT:    s_endpgm
3111;
3112; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_acquire_cmpxchg:
3113; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3114; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3115; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3116; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3117; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3118; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3119; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3120; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3121; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3122; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3123;
3124; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_acquire_cmpxchg:
3125; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3126; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3127; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3128; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3129; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3130; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3131; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3132; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3133; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3134; GFX90A-TGSPLIT-NEXT:    s_endpgm
3135;
3136; GFX940-NOTTGSPLIT-LABEL: flat_agent_monotonic_acquire_cmpxchg:
3137; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3138; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3139; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3140; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3141; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3142; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3143; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3144; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3145; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
3146; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3147;
3148; GFX940-TGSPLIT-LABEL: flat_agent_monotonic_acquire_cmpxchg:
3149; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3150; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3151; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3152; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3153; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3154; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3155; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3156; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3157; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
3158; GFX940-TGSPLIT-NEXT:    s_endpgm
3159;
3160; GFX11-WGP-LABEL: flat_agent_monotonic_acquire_cmpxchg:
3161; GFX11-WGP:       ; %bb.0: ; %entry
3162; GFX11-WGP-NEXT:    s_clause 0x1
3163; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3164; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3165; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3166; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3167; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3168; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3169; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3170; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3171; GFX11-WGP-NEXT:    buffer_gl0_inv
3172; GFX11-WGP-NEXT:    buffer_gl1_inv
3173; GFX11-WGP-NEXT:    s_endpgm
3174;
3175; GFX11-CU-LABEL: flat_agent_monotonic_acquire_cmpxchg:
3176; GFX11-CU:       ; %bb.0: ; %entry
3177; GFX11-CU-NEXT:    s_clause 0x1
3178; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3179; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3180; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3181; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3182; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3183; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3184; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3185; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3186; GFX11-CU-NEXT:    buffer_gl0_inv
3187; GFX11-CU-NEXT:    buffer_gl1_inv
3188; GFX11-CU-NEXT:    s_endpgm
3189    i32* %out, i32 %in, i32 %old) {
3190entry:
3191  %gep = getelementptr i32, i32* %out, i32 4
3192  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") monotonic acquire
3193  ret void
3194}
3195
3196define amdgpu_kernel void @flat_agent_acquire_acquire_cmpxchg(
3197; GFX7-LABEL: flat_agent_acquire_acquire_cmpxchg:
3198; GFX7:       ; %bb.0: ; %entry
3199; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3200; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3201; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3202; GFX7-NEXT:    s_add_u32 s0, s0, 16
3203; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3204; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3205; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3206; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3207; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3208; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3209; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3210; GFX7-NEXT:    buffer_wbinvl1_vol
3211; GFX7-NEXT:    s_endpgm
3212;
3213; GFX10-WGP-LABEL: flat_agent_acquire_acquire_cmpxchg:
3214; GFX10-WGP:       ; %bb.0: ; %entry
3215; GFX10-WGP-NEXT:    s_clause 0x1
3216; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3217; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3218; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3219; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3220; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3221; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3222; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3223; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3224; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3225; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3226; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3227; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3228; GFX10-WGP-NEXT:    buffer_gl0_inv
3229; GFX10-WGP-NEXT:    buffer_gl1_inv
3230; GFX10-WGP-NEXT:    s_endpgm
3231;
3232; GFX10-CU-LABEL: flat_agent_acquire_acquire_cmpxchg:
3233; GFX10-CU:       ; %bb.0: ; %entry
3234; GFX10-CU-NEXT:    s_clause 0x1
3235; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3236; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3237; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3238; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3239; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3240; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3241; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3242; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3243; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3244; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3245; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3246; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3247; GFX10-CU-NEXT:    buffer_gl0_inv
3248; GFX10-CU-NEXT:    buffer_gl1_inv
3249; GFX10-CU-NEXT:    s_endpgm
3250;
3251; SKIP-CACHE-INV-LABEL: flat_agent_acquire_acquire_cmpxchg:
3252; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3253; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3254; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3255; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3256; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3257; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3258; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3259; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3260; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3261; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3262; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3263; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3264; SKIP-CACHE-INV-NEXT:    s_endpgm
3265;
3266; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_acquire_cmpxchg:
3267; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3268; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3269; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3270; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3271; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3272; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3273; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3274; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3275; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3276; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3277;
3278; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_acquire_cmpxchg:
3279; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3280; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3281; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3282; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3283; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3284; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3285; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3286; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3287; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3288; GFX90A-TGSPLIT-NEXT:    s_endpgm
3289;
3290; GFX940-NOTTGSPLIT-LABEL: flat_agent_acquire_acquire_cmpxchg:
3291; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3292; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3293; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3294; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3295; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3296; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3297; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3298; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3299; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
3300; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3301;
3302; GFX940-TGSPLIT-LABEL: flat_agent_acquire_acquire_cmpxchg:
3303; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3304; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3305; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3306; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3307; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3308; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3309; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3310; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3311; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
3312; GFX940-TGSPLIT-NEXT:    s_endpgm
3313;
3314; GFX11-WGP-LABEL: flat_agent_acquire_acquire_cmpxchg:
3315; GFX11-WGP:       ; %bb.0: ; %entry
3316; GFX11-WGP-NEXT:    s_clause 0x1
3317; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3318; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3319; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3320; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3321; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3322; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3323; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3324; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3325; GFX11-WGP-NEXT:    buffer_gl0_inv
3326; GFX11-WGP-NEXT:    buffer_gl1_inv
3327; GFX11-WGP-NEXT:    s_endpgm
3328;
3329; GFX11-CU-LABEL: flat_agent_acquire_acquire_cmpxchg:
3330; GFX11-CU:       ; %bb.0: ; %entry
3331; GFX11-CU-NEXT:    s_clause 0x1
3332; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3333; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3334; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3335; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3336; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3337; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3338; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3339; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3340; GFX11-CU-NEXT:    buffer_gl0_inv
3341; GFX11-CU-NEXT:    buffer_gl1_inv
3342; GFX11-CU-NEXT:    s_endpgm
3343    i32* %out, i32 %in, i32 %old) {
3344entry:
3345  %gep = getelementptr i32, i32* %out, i32 4
3346  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire acquire
3347  ret void
3348}
3349
3350define amdgpu_kernel void @flat_agent_release_acquire_cmpxchg(
3351; GFX7-LABEL: flat_agent_release_acquire_cmpxchg:
3352; GFX7:       ; %bb.0: ; %entry
3353; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3354; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3355; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3356; GFX7-NEXT:    s_add_u32 s0, s0, 16
3357; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3358; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3359; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3360; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3361; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3362; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3363; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3364; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3365; GFX7-NEXT:    buffer_wbinvl1_vol
3366; GFX7-NEXT:    s_endpgm
3367;
3368; GFX10-WGP-LABEL: flat_agent_release_acquire_cmpxchg:
3369; GFX10-WGP:       ; %bb.0: ; %entry
3370; GFX10-WGP-NEXT:    s_clause 0x1
3371; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3372; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3373; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3374; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3375; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3376; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3377; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3378; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3379; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3380; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3381; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3382; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3383; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3384; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3385; GFX10-WGP-NEXT:    buffer_gl0_inv
3386; GFX10-WGP-NEXT:    buffer_gl1_inv
3387; GFX10-WGP-NEXT:    s_endpgm
3388;
3389; GFX10-CU-LABEL: flat_agent_release_acquire_cmpxchg:
3390; GFX10-CU:       ; %bb.0: ; %entry
3391; GFX10-CU-NEXT:    s_clause 0x1
3392; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3393; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3394; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3395; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3396; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3397; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3398; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3399; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3400; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3401; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3402; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3403; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3404; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3405; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3406; GFX10-CU-NEXT:    buffer_gl0_inv
3407; GFX10-CU-NEXT:    buffer_gl1_inv
3408; GFX10-CU-NEXT:    s_endpgm
3409;
3410; SKIP-CACHE-INV-LABEL: flat_agent_release_acquire_cmpxchg:
3411; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3412; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3413; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3414; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3415; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3416; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3417; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3418; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3419; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3420; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3421; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3422; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3423; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3424; SKIP-CACHE-INV-NEXT:    s_endpgm
3425;
3426; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_acquire_cmpxchg:
3427; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3428; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3429; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3430; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3431; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3432; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3433; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3434; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3435; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3436; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3437; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3438;
3439; GFX90A-TGSPLIT-LABEL: flat_agent_release_acquire_cmpxchg:
3440; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3441; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3442; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3443; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3444; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3445; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3446; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3447; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3448; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3449; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3450; GFX90A-TGSPLIT-NEXT:    s_endpgm
3451;
3452; GFX940-NOTTGSPLIT-LABEL: flat_agent_release_acquire_cmpxchg:
3453; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3454; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3455; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3456; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3457; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3458; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3459; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
3460; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3461; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3462; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3463; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
3464; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3465;
3466; GFX940-TGSPLIT-LABEL: flat_agent_release_acquire_cmpxchg:
3467; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3468; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3469; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3470; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3471; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3472; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3473; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
3474; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3475; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3476; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3477; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
3478; GFX940-TGSPLIT-NEXT:    s_endpgm
3479;
3480; GFX11-WGP-LABEL: flat_agent_release_acquire_cmpxchg:
3481; GFX11-WGP:       ; %bb.0: ; %entry
3482; GFX11-WGP-NEXT:    s_clause 0x1
3483; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3484; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3485; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3486; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3487; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3488; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3489; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3490; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3491; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3492; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3493; GFX11-WGP-NEXT:    buffer_gl0_inv
3494; GFX11-WGP-NEXT:    buffer_gl1_inv
3495; GFX11-WGP-NEXT:    s_endpgm
3496;
3497; GFX11-CU-LABEL: flat_agent_release_acquire_cmpxchg:
3498; GFX11-CU:       ; %bb.0: ; %entry
3499; GFX11-CU-NEXT:    s_clause 0x1
3500; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3501; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3502; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3503; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3504; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3505; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3506; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3507; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3508; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3509; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3510; GFX11-CU-NEXT:    buffer_gl0_inv
3511; GFX11-CU-NEXT:    buffer_gl1_inv
3512; GFX11-CU-NEXT:    s_endpgm
3513    i32* %out, i32 %in, i32 %old) {
3514entry:
3515  %gep = getelementptr i32, i32* %out, i32 4
3516  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release acquire
3517  ret void
3518}
3519
3520define amdgpu_kernel void @flat_agent_acq_rel_acquire_cmpxchg(
3521; GFX7-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
3522; GFX7:       ; %bb.0: ; %entry
3523; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3524; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3525; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3526; GFX7-NEXT:    s_add_u32 s0, s0, 16
3527; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3528; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3529; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3530; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3531; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3532; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3533; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3534; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3535; GFX7-NEXT:    buffer_wbinvl1_vol
3536; GFX7-NEXT:    s_endpgm
3537;
3538; GFX10-WGP-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
3539; GFX10-WGP:       ; %bb.0: ; %entry
3540; GFX10-WGP-NEXT:    s_clause 0x1
3541; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3542; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3543; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3544; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3545; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3546; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3547; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3548; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3549; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3550; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3551; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3552; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3553; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3554; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3555; GFX10-WGP-NEXT:    buffer_gl0_inv
3556; GFX10-WGP-NEXT:    buffer_gl1_inv
3557; GFX10-WGP-NEXT:    s_endpgm
3558;
3559; GFX10-CU-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
3560; GFX10-CU:       ; %bb.0: ; %entry
3561; GFX10-CU-NEXT:    s_clause 0x1
3562; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3563; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3564; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3565; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3566; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3567; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3568; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3569; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3570; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3571; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3572; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3573; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3574; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3575; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3576; GFX10-CU-NEXT:    buffer_gl0_inv
3577; GFX10-CU-NEXT:    buffer_gl1_inv
3578; GFX10-CU-NEXT:    s_endpgm
3579;
3580; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
3581; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3582; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3583; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3584; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3585; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3586; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3587; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3588; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3589; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3590; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3591; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3592; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3593; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3594; SKIP-CACHE-INV-NEXT:    s_endpgm
3595;
3596; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
3597; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3598; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3599; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3600; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3601; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3602; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3603; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3604; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3605; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3606; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3607; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3608;
3609; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
3610; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3611; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3612; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3613; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3614; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3615; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3616; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3617; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3618; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3619; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3620; GFX90A-TGSPLIT-NEXT:    s_endpgm
3621;
3622; GFX940-NOTTGSPLIT-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
3623; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3624; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3625; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3626; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3627; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3628; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3629; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
3630; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3631; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3632; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3633; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
3634; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3635;
3636; GFX940-TGSPLIT-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
3637; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3638; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3639; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3640; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3641; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3642; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3643; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
3644; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3645; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3646; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3647; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
3648; GFX940-TGSPLIT-NEXT:    s_endpgm
3649;
3650; GFX11-WGP-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
3651; GFX11-WGP:       ; %bb.0: ; %entry
3652; GFX11-WGP-NEXT:    s_clause 0x1
3653; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3654; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3655; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3656; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3657; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3658; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3659; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3660; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3661; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3662; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3663; GFX11-WGP-NEXT:    buffer_gl0_inv
3664; GFX11-WGP-NEXT:    buffer_gl1_inv
3665; GFX11-WGP-NEXT:    s_endpgm
3666;
3667; GFX11-CU-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
3668; GFX11-CU:       ; %bb.0: ; %entry
3669; GFX11-CU-NEXT:    s_clause 0x1
3670; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3671; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3672; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3673; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3674; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3675; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3676; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3677; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3678; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3679; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3680; GFX11-CU-NEXT:    buffer_gl0_inv
3681; GFX11-CU-NEXT:    buffer_gl1_inv
3682; GFX11-CU-NEXT:    s_endpgm
3683    i32* %out, i32 %in, i32 %old) {
3684entry:
3685  %gep = getelementptr i32, i32* %out, i32 4
3686  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel acquire
3687  ret void
3688}
3689
3690define amdgpu_kernel void @flat_agent_seq_cst_acquire_cmpxchg(
3691; GFX7-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
3692; GFX7:       ; %bb.0: ; %entry
3693; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3694; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3695; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3696; GFX7-NEXT:    s_add_u32 s0, s0, 16
3697; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3698; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3699; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3700; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3701; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3702; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3703; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3704; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3705; GFX7-NEXT:    buffer_wbinvl1_vol
3706; GFX7-NEXT:    s_endpgm
3707;
3708; GFX10-WGP-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
3709; GFX10-WGP:       ; %bb.0: ; %entry
3710; GFX10-WGP-NEXT:    s_clause 0x1
3711; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3712; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3713; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3714; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3715; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3716; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3717; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3718; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3719; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3720; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3721; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3722; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3723; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3724; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3725; GFX10-WGP-NEXT:    buffer_gl0_inv
3726; GFX10-WGP-NEXT:    buffer_gl1_inv
3727; GFX10-WGP-NEXT:    s_endpgm
3728;
3729; GFX10-CU-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
3730; GFX10-CU:       ; %bb.0: ; %entry
3731; GFX10-CU-NEXT:    s_clause 0x1
3732; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3733; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3734; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3735; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3736; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3737; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3738; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3739; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3740; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3741; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3742; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3743; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3744; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3745; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3746; GFX10-CU-NEXT:    buffer_gl0_inv
3747; GFX10-CU-NEXT:    buffer_gl1_inv
3748; GFX10-CU-NEXT:    s_endpgm
3749;
3750; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
3751; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3752; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3753; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3754; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3755; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3756; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3757; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3758; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3759; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3760; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3761; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3762; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3763; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3764; SKIP-CACHE-INV-NEXT:    s_endpgm
3765;
3766; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
3767; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3768; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3769; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3770; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3771; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3772; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3773; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3774; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3775; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3776; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3777; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3778;
3779; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
3780; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3781; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3782; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3783; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3784; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3785; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3786; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3787; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3788; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3789; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3790; GFX90A-TGSPLIT-NEXT:    s_endpgm
3791;
3792; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
3793; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3794; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3795; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3796; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3797; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3798; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3799; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
3800; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3801; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3802; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3803; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
3804; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3805;
3806; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
3807; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3808; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3809; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3810; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3811; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3812; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3813; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
3814; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3815; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3816; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3817; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
3818; GFX940-TGSPLIT-NEXT:    s_endpgm
3819;
3820; GFX11-WGP-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
3821; GFX11-WGP:       ; %bb.0: ; %entry
3822; GFX11-WGP-NEXT:    s_clause 0x1
3823; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3824; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3825; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3826; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3827; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3828; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3829; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3830; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3831; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3832; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3833; GFX11-WGP-NEXT:    buffer_gl0_inv
3834; GFX11-WGP-NEXT:    buffer_gl1_inv
3835; GFX11-WGP-NEXT:    s_endpgm
3836;
3837; GFX11-CU-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
3838; GFX11-CU:       ; %bb.0: ; %entry
3839; GFX11-CU-NEXT:    s_clause 0x1
3840; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3841; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3842; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3843; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3844; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3845; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3846; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3847; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3848; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3849; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3850; GFX11-CU-NEXT:    buffer_gl0_inv
3851; GFX11-CU-NEXT:    buffer_gl1_inv
3852; GFX11-CU-NEXT:    s_endpgm
3853    i32* %out, i32 %in, i32 %old) {
3854entry:
3855  %gep = getelementptr i32, i32* %out, i32 4
3856  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst acquire
3857  ret void
3858}
3859
3860define amdgpu_kernel void @flat_agent_monotonic_seq_cst_cmpxchg(
3861; GFX7-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
3862; GFX7:       ; %bb.0: ; %entry
3863; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3864; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3865; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3866; GFX7-NEXT:    s_add_u32 s0, s0, 16
3867; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3868; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3869; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3870; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3871; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3872; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3873; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3874; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3875; GFX7-NEXT:    buffer_wbinvl1_vol
3876; GFX7-NEXT:    s_endpgm
3877;
3878; GFX10-WGP-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
3879; GFX10-WGP:       ; %bb.0: ; %entry
3880; GFX10-WGP-NEXT:    s_clause 0x1
3881; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3882; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3883; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3884; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3885; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3886; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3887; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3888; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3889; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3890; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3891; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3892; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3893; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3894; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3895; GFX10-WGP-NEXT:    buffer_gl0_inv
3896; GFX10-WGP-NEXT:    buffer_gl1_inv
3897; GFX10-WGP-NEXT:    s_endpgm
3898;
3899; GFX10-CU-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
3900; GFX10-CU:       ; %bb.0: ; %entry
3901; GFX10-CU-NEXT:    s_clause 0x1
3902; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3903; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3904; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3905; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3906; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3907; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3908; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3909; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3910; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3911; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3912; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3913; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3914; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3915; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3916; GFX10-CU-NEXT:    buffer_gl0_inv
3917; GFX10-CU-NEXT:    buffer_gl1_inv
3918; GFX10-CU-NEXT:    s_endpgm
3919;
3920; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
3921; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3922; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3923; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3924; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3925; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3926; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3927; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3928; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3929; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3930; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3931; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3932; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3933; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3934; SKIP-CACHE-INV-NEXT:    s_endpgm
3935;
3936; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
3937; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3938; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3939; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3940; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3941; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3942; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3943; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3944; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3945; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3946; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3947; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3948;
3949; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
3950; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3951; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3952; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3953; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3954; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3955; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3956; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3957; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3958; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3959; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3960; GFX90A-TGSPLIT-NEXT:    s_endpgm
3961;
3962; GFX940-NOTTGSPLIT-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
3963; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3964; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3965; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3966; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3967; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3968; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3969; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
3970; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3971; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3972; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3973; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
3974; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3975;
3976; GFX940-TGSPLIT-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
3977; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3978; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3979; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3980; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3981; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3982; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3983; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
3984; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3985; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3986; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3987; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
3988; GFX940-TGSPLIT-NEXT:    s_endpgm
3989;
3990; GFX11-WGP-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
3991; GFX11-WGP:       ; %bb.0: ; %entry
3992; GFX11-WGP-NEXT:    s_clause 0x1
3993; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3994; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3995; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3996; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
3997; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
3998; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3999; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4000; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4001; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4002; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4003; GFX11-WGP-NEXT:    buffer_gl0_inv
4004; GFX11-WGP-NEXT:    buffer_gl1_inv
4005; GFX11-WGP-NEXT:    s_endpgm
4006;
4007; GFX11-CU-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
4008; GFX11-CU:       ; %bb.0: ; %entry
4009; GFX11-CU-NEXT:    s_clause 0x1
4010; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4011; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4012; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4013; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4014; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4015; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4016; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4017; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4018; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4019; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4020; GFX11-CU-NEXT:    buffer_gl0_inv
4021; GFX11-CU-NEXT:    buffer_gl1_inv
4022; GFX11-CU-NEXT:    s_endpgm
4023    i32* %out, i32 %in, i32 %old) {
4024entry:
4025  %gep = getelementptr i32, i32* %out, i32 4
4026  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") monotonic seq_cst
4027  ret void
4028}
4029
4030define amdgpu_kernel void @flat_agent_acquire_seq_cst_cmpxchg(
4031; GFX7-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
4032; GFX7:       ; %bb.0: ; %entry
4033; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4034; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4035; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4036; GFX7-NEXT:    s_add_u32 s0, s0, 16
4037; GFX7-NEXT:    s_addc_u32 s1, s1, 0
4038; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4039; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4040; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4041; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4042; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4043; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4044; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4045; GFX7-NEXT:    buffer_wbinvl1_vol
4046; GFX7-NEXT:    s_endpgm
4047;
4048; GFX10-WGP-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
4049; GFX10-WGP:       ; %bb.0: ; %entry
4050; GFX10-WGP-NEXT:    s_clause 0x1
4051; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4052; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4053; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4054; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
4055; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
4056; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4057; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4058; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4059; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4060; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4061; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4062; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4063; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4064; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4065; GFX10-WGP-NEXT:    buffer_gl0_inv
4066; GFX10-WGP-NEXT:    buffer_gl1_inv
4067; GFX10-WGP-NEXT:    s_endpgm
4068;
4069; GFX10-CU-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
4070; GFX10-CU:       ; %bb.0: ; %entry
4071; GFX10-CU-NEXT:    s_clause 0x1
4072; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4073; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4074; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4075; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
4076; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
4077; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4078; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4079; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4080; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4081; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4082; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4083; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4084; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4085; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4086; GFX10-CU-NEXT:    buffer_gl0_inv
4087; GFX10-CU-NEXT:    buffer_gl1_inv
4088; GFX10-CU-NEXT:    s_endpgm
4089;
4090; SKIP-CACHE-INV-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
4091; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4092; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4093; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4094; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4095; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
4096; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
4097; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4098; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4099; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4100; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4101; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4102; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4103; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4104; SKIP-CACHE-INV-NEXT:    s_endpgm
4105;
4106; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
4107; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4108; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4109; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4110; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4111; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4112; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4113; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4114; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4115; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4116; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4117; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4118;
4119; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
4120; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4121; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4122; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4123; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4124; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4125; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4126; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4127; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4128; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4129; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4130; GFX90A-TGSPLIT-NEXT:    s_endpgm
4131;
4132; GFX940-NOTTGSPLIT-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
4133; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4134; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4135; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4136; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4137; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4138; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4139; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
4140; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4141; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4142; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4143; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
4144; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4145;
4146; GFX940-TGSPLIT-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
4147; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4148; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4149; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4150; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4151; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4152; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4153; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
4154; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4155; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4156; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4157; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
4158; GFX940-TGSPLIT-NEXT:    s_endpgm
4159;
4160; GFX11-WGP-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
4161; GFX11-WGP:       ; %bb.0: ; %entry
4162; GFX11-WGP-NEXT:    s_clause 0x1
4163; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4164; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4165; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4166; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4167; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4168; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4169; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4170; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4171; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4172; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4173; GFX11-WGP-NEXT:    buffer_gl0_inv
4174; GFX11-WGP-NEXT:    buffer_gl1_inv
4175; GFX11-WGP-NEXT:    s_endpgm
4176;
4177; GFX11-CU-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
4178; GFX11-CU:       ; %bb.0: ; %entry
4179; GFX11-CU-NEXT:    s_clause 0x1
4180; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4181; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4182; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4183; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4184; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4185; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4186; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4187; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4188; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4189; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4190; GFX11-CU-NEXT:    buffer_gl0_inv
4191; GFX11-CU-NEXT:    buffer_gl1_inv
4192; GFX11-CU-NEXT:    s_endpgm
4193    i32* %out, i32 %in, i32 %old) {
4194entry:
4195  %gep = getelementptr i32, i32* %out, i32 4
4196  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire seq_cst
4197  ret void
4198}
4199
4200define amdgpu_kernel void @flat_agent_release_seq_cst_cmpxchg(
4201; GFX7-LABEL: flat_agent_release_seq_cst_cmpxchg:
4202; GFX7:       ; %bb.0: ; %entry
4203; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4204; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4205; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4206; GFX7-NEXT:    s_add_u32 s0, s0, 16
4207; GFX7-NEXT:    s_addc_u32 s1, s1, 0
4208; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4209; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4210; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4211; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4212; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4213; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4214; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4215; GFX7-NEXT:    buffer_wbinvl1_vol
4216; GFX7-NEXT:    s_endpgm
4217;
4218; GFX10-WGP-LABEL: flat_agent_release_seq_cst_cmpxchg:
4219; GFX10-WGP:       ; %bb.0: ; %entry
4220; GFX10-WGP-NEXT:    s_clause 0x1
4221; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4222; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4223; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4224; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
4225; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
4226; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4227; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4228; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4229; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4230; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4231; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4232; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4233; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4234; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4235; GFX10-WGP-NEXT:    buffer_gl0_inv
4236; GFX10-WGP-NEXT:    buffer_gl1_inv
4237; GFX10-WGP-NEXT:    s_endpgm
4238;
4239; GFX10-CU-LABEL: flat_agent_release_seq_cst_cmpxchg:
4240; GFX10-CU:       ; %bb.0: ; %entry
4241; GFX10-CU-NEXT:    s_clause 0x1
4242; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4243; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4244; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4245; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
4246; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
4247; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4248; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4249; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4250; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4251; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4252; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4253; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4254; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4255; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4256; GFX10-CU-NEXT:    buffer_gl0_inv
4257; GFX10-CU-NEXT:    buffer_gl1_inv
4258; GFX10-CU-NEXT:    s_endpgm
4259;
4260; SKIP-CACHE-INV-LABEL: flat_agent_release_seq_cst_cmpxchg:
4261; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4262; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4263; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4264; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4265; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
4266; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
4267; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4268; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4269; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4270; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4271; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4272; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4273; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4274; SKIP-CACHE-INV-NEXT:    s_endpgm
4275;
4276; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_seq_cst_cmpxchg:
4277; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4278; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4279; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4280; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4281; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4282; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4283; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4284; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4285; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4286; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4287; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4288;
4289; GFX90A-TGSPLIT-LABEL: flat_agent_release_seq_cst_cmpxchg:
4290; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4291; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4292; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4293; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4294; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4295; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4296; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4297; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4298; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4299; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4300; GFX90A-TGSPLIT-NEXT:    s_endpgm
4301;
4302; GFX940-NOTTGSPLIT-LABEL: flat_agent_release_seq_cst_cmpxchg:
4303; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4304; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4305; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4306; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4307; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4308; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4309; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
4310; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4311; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4312; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4313; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
4314; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4315;
4316; GFX940-TGSPLIT-LABEL: flat_agent_release_seq_cst_cmpxchg:
4317; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4318; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4319; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4320; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4321; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4322; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4323; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
4324; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4325; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4326; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4327; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
4328; GFX940-TGSPLIT-NEXT:    s_endpgm
4329;
4330; GFX11-WGP-LABEL: flat_agent_release_seq_cst_cmpxchg:
4331; GFX11-WGP:       ; %bb.0: ; %entry
4332; GFX11-WGP-NEXT:    s_clause 0x1
4333; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4334; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4335; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4336; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4337; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4338; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4339; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4340; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4341; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4342; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4343; GFX11-WGP-NEXT:    buffer_gl0_inv
4344; GFX11-WGP-NEXT:    buffer_gl1_inv
4345; GFX11-WGP-NEXT:    s_endpgm
4346;
4347; GFX11-CU-LABEL: flat_agent_release_seq_cst_cmpxchg:
4348; GFX11-CU:       ; %bb.0: ; %entry
4349; GFX11-CU-NEXT:    s_clause 0x1
4350; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4351; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4352; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4353; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4354; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4355; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4356; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4357; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4358; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4359; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4360; GFX11-CU-NEXT:    buffer_gl0_inv
4361; GFX11-CU-NEXT:    buffer_gl1_inv
4362; GFX11-CU-NEXT:    s_endpgm
4363    i32* %out, i32 %in, i32 %old) {
4364entry:
4365  %gep = getelementptr i32, i32* %out, i32 4
4366  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release seq_cst
4367  ret void
4368}
4369
4370define amdgpu_kernel void @flat_agent_acq_rel_seq_cst_cmpxchg(
4371; GFX7-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
4372; GFX7:       ; %bb.0: ; %entry
4373; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4374; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4375; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4376; GFX7-NEXT:    s_add_u32 s0, s0, 16
4377; GFX7-NEXT:    s_addc_u32 s1, s1, 0
4378; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4379; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4380; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4381; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4382; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4383; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4384; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4385; GFX7-NEXT:    buffer_wbinvl1_vol
4386; GFX7-NEXT:    s_endpgm
4387;
4388; GFX10-WGP-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
4389; GFX10-WGP:       ; %bb.0: ; %entry
4390; GFX10-WGP-NEXT:    s_clause 0x1
4391; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4392; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4393; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4394; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
4395; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
4396; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4397; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4398; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4399; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4400; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4401; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4402; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4403; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4404; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4405; GFX10-WGP-NEXT:    buffer_gl0_inv
4406; GFX10-WGP-NEXT:    buffer_gl1_inv
4407; GFX10-WGP-NEXT:    s_endpgm
4408;
4409; GFX10-CU-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
4410; GFX10-CU:       ; %bb.0: ; %entry
4411; GFX10-CU-NEXT:    s_clause 0x1
4412; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4413; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4414; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4415; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
4416; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
4417; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4418; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4419; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4420; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4421; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4422; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4423; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4424; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4425; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4426; GFX10-CU-NEXT:    buffer_gl0_inv
4427; GFX10-CU-NEXT:    buffer_gl1_inv
4428; GFX10-CU-NEXT:    s_endpgm
4429;
4430; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
4431; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4432; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4433; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4434; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4435; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
4436; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
4437; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4438; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4439; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4440; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4441; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4442; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4443; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4444; SKIP-CACHE-INV-NEXT:    s_endpgm
4445;
4446; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
4447; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4448; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4449; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4450; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4451; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4452; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4453; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4454; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4455; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4456; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4457; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4458;
4459; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
4460; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4461; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4462; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4463; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4464; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4465; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4466; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4467; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4468; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4469; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4470; GFX90A-TGSPLIT-NEXT:    s_endpgm
4471;
4472; GFX940-NOTTGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
4473; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4474; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4475; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4476; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4477; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4478; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4479; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
4480; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4481; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4482; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4483; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
4484; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4485;
4486; GFX940-TGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
4487; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4488; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4489; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4490; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4491; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4492; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4493; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
4494; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4495; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4496; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4497; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
4498; GFX940-TGSPLIT-NEXT:    s_endpgm
4499;
4500; GFX11-WGP-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
4501; GFX11-WGP:       ; %bb.0: ; %entry
4502; GFX11-WGP-NEXT:    s_clause 0x1
4503; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4504; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4505; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4506; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4507; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4508; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4509; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4510; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4511; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4512; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4513; GFX11-WGP-NEXT:    buffer_gl0_inv
4514; GFX11-WGP-NEXT:    buffer_gl1_inv
4515; GFX11-WGP-NEXT:    s_endpgm
4516;
4517; GFX11-CU-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
4518; GFX11-CU:       ; %bb.0: ; %entry
4519; GFX11-CU-NEXT:    s_clause 0x1
4520; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4521; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4522; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4523; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4524; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4525; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4526; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4527; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4528; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4529; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4530; GFX11-CU-NEXT:    buffer_gl0_inv
4531; GFX11-CU-NEXT:    buffer_gl1_inv
4532; GFX11-CU-NEXT:    s_endpgm
4533    i32* %out, i32 %in, i32 %old) {
4534entry:
4535  %gep = getelementptr i32, i32* %out, i32 4
4536  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel seq_cst
4537  ret void
4538}
4539
4540define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_cmpxchg(
4541; GFX7-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
4542; GFX7:       ; %bb.0: ; %entry
4543; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4544; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4545; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4546; GFX7-NEXT:    s_add_u32 s0, s0, 16
4547; GFX7-NEXT:    s_addc_u32 s1, s1, 0
4548; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4549; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4550; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4551; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4552; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4553; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4554; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4555; GFX7-NEXT:    buffer_wbinvl1_vol
4556; GFX7-NEXT:    s_endpgm
4557;
4558; GFX10-WGP-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
4559; GFX10-WGP:       ; %bb.0: ; %entry
4560; GFX10-WGP-NEXT:    s_clause 0x1
4561; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4562; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4563; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4564; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
4565; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
4566; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4567; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4568; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4569; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4570; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4571; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4572; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4573; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4574; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4575; GFX10-WGP-NEXT:    buffer_gl0_inv
4576; GFX10-WGP-NEXT:    buffer_gl1_inv
4577; GFX10-WGP-NEXT:    s_endpgm
4578;
4579; GFX10-CU-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
4580; GFX10-CU:       ; %bb.0: ; %entry
4581; GFX10-CU-NEXT:    s_clause 0x1
4582; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4583; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4584; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4585; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
4586; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
4587; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4588; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4589; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4590; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4591; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4592; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4593; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4594; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4595; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4596; GFX10-CU-NEXT:    buffer_gl0_inv
4597; GFX10-CU-NEXT:    buffer_gl1_inv
4598; GFX10-CU-NEXT:    s_endpgm
4599;
4600; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
4601; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4602; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4603; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4604; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4605; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
4606; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
4607; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4608; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4609; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4610; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4611; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4612; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4613; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4614; SKIP-CACHE-INV-NEXT:    s_endpgm
4615;
4616; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
4617; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4618; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4619; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4620; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4621; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4622; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4623; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4624; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4625; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4626; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4627; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4628;
4629; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
4630; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4631; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4632; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4633; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4634; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4635; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4636; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4637; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4638; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4639; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4640; GFX90A-TGSPLIT-NEXT:    s_endpgm
4641;
4642; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
4643; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4644; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4645; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4646; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4647; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4648; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4649; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
4650; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4651; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4652; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4653; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
4654; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4655;
4656; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
4657; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4658; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4659; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4660; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4661; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4662; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4663; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
4664; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4665; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4666; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4667; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
4668; GFX940-TGSPLIT-NEXT:    s_endpgm
4669;
4670; GFX11-WGP-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
4671; GFX11-WGP:       ; %bb.0: ; %entry
4672; GFX11-WGP-NEXT:    s_clause 0x1
4673; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4674; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4675; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4676; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4677; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4678; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4679; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4680; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4681; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4682; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4683; GFX11-WGP-NEXT:    buffer_gl0_inv
4684; GFX11-WGP-NEXT:    buffer_gl1_inv
4685; GFX11-WGP-NEXT:    s_endpgm
4686;
4687; GFX11-CU-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
4688; GFX11-CU:       ; %bb.0: ; %entry
4689; GFX11-CU-NEXT:    s_clause 0x1
4690; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4691; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4692; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4693; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4694; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4695; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4696; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4697; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4698; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4699; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4700; GFX11-CU-NEXT:    buffer_gl0_inv
4701; GFX11-CU-NEXT:    buffer_gl1_inv
4702; GFX11-CU-NEXT:    s_endpgm
4703    i32* %out, i32 %in, i32 %old) {
4704entry:
4705  %gep = getelementptr i32, i32* %out, i32 4
4706  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
4707  ret void
4708}
4709
4710define amdgpu_kernel void @flat_agent_monotonic_monotonic_ret_cmpxchg(
4711; GFX7-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
4712; GFX7:       ; %bb.0: ; %entry
4713; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4714; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4715; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4716; GFX7-NEXT:    s_add_u32 s4, s0, 16
4717; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4718; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4719; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4720; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4721; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4722; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4723; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4724; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4725; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4726; GFX7-NEXT:    flat_store_dword v[0:1], v2
4727; GFX7-NEXT:    s_endpgm
4728;
4729; GFX10-WGP-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
4730; GFX10-WGP:       ; %bb.0: ; %entry
4731; GFX10-WGP-NEXT:    s_clause 0x1
4732; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4733; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4734; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4735; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4736; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4737; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4738; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4739; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4740; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4741; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4742; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4743; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4744; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4745; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4746; GFX10-WGP-NEXT:    s_endpgm
4747;
4748; GFX10-CU-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
4749; GFX10-CU:       ; %bb.0: ; %entry
4750; GFX10-CU-NEXT:    s_clause 0x1
4751; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4752; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4753; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4754; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4755; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4756; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4757; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4758; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4759; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4760; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4761; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4762; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4763; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4764; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4765; GFX10-CU-NEXT:    s_endpgm
4766;
4767; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
4768; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4769; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4770; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4771; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4772; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4773; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4774; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4775; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4776; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4777; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4778; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4779; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4780; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4781; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4782; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4783; SKIP-CACHE-INV-NEXT:    s_endpgm
4784;
4785; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
4786; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4787; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4788; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4789; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4790; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4791; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4792; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4793; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4794; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4795; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4796;
4797; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
4798; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4799; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4800; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4801; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4802; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4803; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4804; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4805; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4806; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4807; GFX90A-TGSPLIT-NEXT:    s_endpgm
4808;
4809; GFX940-NOTTGSPLIT-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
4810; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4811; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4812; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4813; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4814; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4815; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4816; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4817; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4818; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4819; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4820;
4821; GFX940-TGSPLIT-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
4822; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4823; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4824; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4825; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4826; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4827; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4828; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4829; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4830; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4831; GFX940-TGSPLIT-NEXT:    s_endpgm
4832;
4833; GFX11-WGP-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
4834; GFX11-WGP:       ; %bb.0: ; %entry
4835; GFX11-WGP-NEXT:    s_clause 0x1
4836; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4837; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4838; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4839; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4840; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4841; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4842; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4843; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
4844; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4845; GFX11-WGP-NEXT:    s_endpgm
4846;
4847; GFX11-CU-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
4848; GFX11-CU:       ; %bb.0: ; %entry
4849; GFX11-CU-NEXT:    s_clause 0x1
4850; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4851; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4852; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4853; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
4854; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
4855; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4856; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4857; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
4858; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4859; GFX11-CU-NEXT:    s_endpgm
4860    i32* %out, i32 %in, i32 %old) {
4861entry:
4862  %gep = getelementptr i32, i32* %out, i32 4
4863  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") monotonic monotonic
4864  %val0 = extractvalue { i32, i1 } %val, 0
4865  store i32 %val0, i32* %out, align 4
4866  ret void
4867}
4868
4869define amdgpu_kernel void @flat_agent_acquire_monotonic_ret_cmpxchg(
4870; GFX7-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
4871; GFX7:       ; %bb.0: ; %entry
4872; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4873; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4874; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4875; GFX7-NEXT:    s_add_u32 s4, s0, 16
4876; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4877; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4878; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4879; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4880; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4881; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4882; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4883; GFX7-NEXT:    buffer_wbinvl1_vol
4884; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4885; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4886; GFX7-NEXT:    flat_store_dword v[0:1], v2
4887; GFX7-NEXT:    s_endpgm
4888;
4889; GFX10-WGP-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
4890; GFX10-WGP:       ; %bb.0: ; %entry
4891; GFX10-WGP-NEXT:    s_clause 0x1
4892; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4893; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4894; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4895; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4896; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4897; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4898; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4899; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4900; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4901; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4902; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4903; GFX10-WGP-NEXT:    buffer_gl0_inv
4904; GFX10-WGP-NEXT:    buffer_gl1_inv
4905; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4906; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4907; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4908; GFX10-WGP-NEXT:    s_endpgm
4909;
4910; GFX10-CU-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
4911; GFX10-CU:       ; %bb.0: ; %entry
4912; GFX10-CU-NEXT:    s_clause 0x1
4913; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4914; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4915; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4916; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4917; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4918; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4919; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4920; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4921; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4922; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4923; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4924; GFX10-CU-NEXT:    buffer_gl0_inv
4925; GFX10-CU-NEXT:    buffer_gl1_inv
4926; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4927; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4928; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4929; GFX10-CU-NEXT:    s_endpgm
4930;
4931; SKIP-CACHE-INV-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
4932; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4933; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4934; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4935; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4936; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4937; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4938; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4939; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4940; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4941; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4942; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4943; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4944; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4945; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4946; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4947; SKIP-CACHE-INV-NEXT:    s_endpgm
4948;
4949; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
4950; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4951; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4952; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4953; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4954; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4955; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4956; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4957; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4958; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4959; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4960; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4961;
4962; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
4963; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4964; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4965; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4966; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4967; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4968; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4969; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4970; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4971; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4972; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4973; GFX90A-TGSPLIT-NEXT:    s_endpgm
4974;
4975; GFX940-NOTTGSPLIT-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
4976; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4977; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4978; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4979; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4980; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4981; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4982; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4983; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4984; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
4985; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4986; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4987;
4988; GFX940-TGSPLIT-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
4989; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4990; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4991; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4992; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4993; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4994; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4995; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4996; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4997; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
4998; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4999; GFX940-TGSPLIT-NEXT:    s_endpgm
5000;
5001; GFX11-WGP-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
5002; GFX11-WGP:       ; %bb.0: ; %entry
5003; GFX11-WGP-NEXT:    s_clause 0x1
5004; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5005; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5006; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5007; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5008; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5009; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5010; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5011; GFX11-WGP-NEXT:    buffer_gl0_inv
5012; GFX11-WGP-NEXT:    buffer_gl1_inv
5013; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5014; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5015; GFX11-WGP-NEXT:    s_endpgm
5016;
5017; GFX11-CU-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
5018; GFX11-CU:       ; %bb.0: ; %entry
5019; GFX11-CU-NEXT:    s_clause 0x1
5020; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5021; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5022; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5023; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5024; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5025; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5026; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5027; GFX11-CU-NEXT:    buffer_gl0_inv
5028; GFX11-CU-NEXT:    buffer_gl1_inv
5029; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5030; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5031; GFX11-CU-NEXT:    s_endpgm
5032    i32* %out, i32 %in, i32 %old) {
5033entry:
5034  %gep = getelementptr i32, i32* %out, i32 4
5035  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire monotonic
5036  %val0 = extractvalue { i32, i1 } %val, 0
5037  store i32 %val0, i32* %out, align 4
5038  ret void
5039}
5040
5041define amdgpu_kernel void @flat_agent_release_monotonic_ret_cmpxchg(
5042; GFX7-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
5043; GFX7:       ; %bb.0: ; %entry
5044; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5045; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5046; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5047; GFX7-NEXT:    s_add_u32 s4, s0, 16
5048; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5049; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5050; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5051; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5052; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5053; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5054; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5055; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5056; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5057; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5058; GFX7-NEXT:    flat_store_dword v[0:1], v2
5059; GFX7-NEXT:    s_endpgm
5060;
5061; GFX10-WGP-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
5062; GFX10-WGP:       ; %bb.0: ; %entry
5063; GFX10-WGP-NEXT:    s_clause 0x1
5064; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5065; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5066; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5067; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5068; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5069; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5070; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5071; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5072; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5073; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5074; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5075; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5076; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5077; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5078; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5079; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5080; GFX10-WGP-NEXT:    s_endpgm
5081;
5082; GFX10-CU-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
5083; GFX10-CU:       ; %bb.0: ; %entry
5084; GFX10-CU-NEXT:    s_clause 0x1
5085; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5086; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5087; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5088; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5089; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5090; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5091; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5092; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5093; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5094; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5095; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5096; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5097; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5098; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5099; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5100; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5101; GFX10-CU-NEXT:    s_endpgm
5102;
5103; SKIP-CACHE-INV-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
5104; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5105; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5106; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5107; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5108; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5109; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5110; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5111; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5112; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5113; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5114; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5115; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5116; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5117; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5118; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5119; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5120; SKIP-CACHE-INV-NEXT:    s_endpgm
5121;
5122; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
5123; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5124; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5125; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5126; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5127; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5128; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5129; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5130; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5131; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5132; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5133; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5134;
5135; GFX90A-TGSPLIT-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
5136; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5137; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5138; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5139; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5140; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5141; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5142; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5143; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5144; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5145; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5146; GFX90A-TGSPLIT-NEXT:    s_endpgm
5147;
5148; GFX940-NOTTGSPLIT-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
5149; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5150; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5151; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5152; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5153; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5154; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5155; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
5156; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5157; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5158; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5159; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5160; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5161;
5162; GFX940-TGSPLIT-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
5163; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5164; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5165; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5166; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5167; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5168; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5169; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
5170; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5171; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5172; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5173; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5174; GFX940-TGSPLIT-NEXT:    s_endpgm
5175;
5176; GFX11-WGP-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
5177; GFX11-WGP:       ; %bb.0: ; %entry
5178; GFX11-WGP-NEXT:    s_clause 0x1
5179; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5180; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5181; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5182; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5183; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5184; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5185; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5186; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5187; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5188; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5189; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5190; GFX11-WGP-NEXT:    s_endpgm
5191;
5192; GFX11-CU-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
5193; GFX11-CU:       ; %bb.0: ; %entry
5194; GFX11-CU-NEXT:    s_clause 0x1
5195; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5196; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5197; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5198; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5199; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5200; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5201; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5202; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5203; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5204; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5205; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5206; GFX11-CU-NEXT:    s_endpgm
5207    i32* %out, i32 %in, i32 %old) {
5208entry:
5209  %gep = getelementptr i32, i32* %out, i32 4
5210  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release monotonic
5211  %val0 = extractvalue { i32, i1 } %val, 0
5212  store i32 %val0, i32* %out, align 4
5213  ret void
5214}
5215
5216define amdgpu_kernel void @flat_agent_acq_rel_monotonic_ret_cmpxchg(
5217; GFX7-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
5218; GFX7:       ; %bb.0: ; %entry
5219; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5220; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5221; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5222; GFX7-NEXT:    s_add_u32 s4, s0, 16
5223; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5224; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5225; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5226; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5227; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5228; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5229; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5230; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5231; GFX7-NEXT:    buffer_wbinvl1_vol
5232; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5233; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5234; GFX7-NEXT:    flat_store_dword v[0:1], v2
5235; GFX7-NEXT:    s_endpgm
5236;
5237; GFX10-WGP-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
5238; GFX10-WGP:       ; %bb.0: ; %entry
5239; GFX10-WGP-NEXT:    s_clause 0x1
5240; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5241; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5242; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5243; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5244; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5245; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5246; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5247; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5248; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5249; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5250; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5251; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5252; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5253; GFX10-WGP-NEXT:    buffer_gl0_inv
5254; GFX10-WGP-NEXT:    buffer_gl1_inv
5255; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5256; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5257; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5258; GFX10-WGP-NEXT:    s_endpgm
5259;
5260; GFX10-CU-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
5261; GFX10-CU:       ; %bb.0: ; %entry
5262; GFX10-CU-NEXT:    s_clause 0x1
5263; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5264; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5265; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5266; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5267; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5268; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5269; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5270; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5271; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5272; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5273; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5274; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5275; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5276; GFX10-CU-NEXT:    buffer_gl0_inv
5277; GFX10-CU-NEXT:    buffer_gl1_inv
5278; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5279; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5280; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5281; GFX10-CU-NEXT:    s_endpgm
5282;
5283; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
5284; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5285; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5286; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5287; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5288; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5289; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5290; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5291; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5292; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5293; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5294; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5295; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5296; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5297; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5298; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5299; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5300; SKIP-CACHE-INV-NEXT:    s_endpgm
5301;
5302; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
5303; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5304; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5305; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5306; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5307; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5308; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5309; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5310; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5311; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5312; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5313; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5314; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5315;
5316; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
5317; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5318; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5319; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5320; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5321; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5322; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5323; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5324; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5325; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5326; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5327; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5328; GFX90A-TGSPLIT-NEXT:    s_endpgm
5329;
5330; GFX940-NOTTGSPLIT-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
5331; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5332; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5333; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5334; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5335; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5336; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5337; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
5338; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5339; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5340; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5341; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
5342; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5343; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5344;
5345; GFX940-TGSPLIT-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
5346; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5347; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5348; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5349; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5350; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5351; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5352; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
5353; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5354; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5355; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5356; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
5357; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5358; GFX940-TGSPLIT-NEXT:    s_endpgm
5359;
5360; GFX11-WGP-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
5361; GFX11-WGP:       ; %bb.0: ; %entry
5362; GFX11-WGP-NEXT:    s_clause 0x1
5363; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5364; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5365; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5366; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5367; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5368; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5369; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5370; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5371; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5372; GFX11-WGP-NEXT:    buffer_gl0_inv
5373; GFX11-WGP-NEXT:    buffer_gl1_inv
5374; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5375; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5376; GFX11-WGP-NEXT:    s_endpgm
5377;
5378; GFX11-CU-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
5379; GFX11-CU:       ; %bb.0: ; %entry
5380; GFX11-CU-NEXT:    s_clause 0x1
5381; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5382; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5383; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5384; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5385; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5386; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5387; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5388; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5389; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5390; GFX11-CU-NEXT:    buffer_gl0_inv
5391; GFX11-CU-NEXT:    buffer_gl1_inv
5392; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5393; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5394; GFX11-CU-NEXT:    s_endpgm
5395    i32* %out, i32 %in, i32 %old) {
5396entry:
5397  %gep = getelementptr i32, i32* %out, i32 4
5398  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel monotonic
5399  %val0 = extractvalue { i32, i1 } %val, 0
5400  store i32 %val0, i32* %out, align 4
5401  ret void
5402}
5403
5404define amdgpu_kernel void @flat_agent_seq_cst_monotonic_ret_cmpxchg(
5405; GFX7-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
5406; GFX7:       ; %bb.0: ; %entry
5407; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5408; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5409; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5410; GFX7-NEXT:    s_add_u32 s4, s0, 16
5411; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5412; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5413; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5414; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5415; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5416; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5417; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5418; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5419; GFX7-NEXT:    buffer_wbinvl1_vol
5420; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5421; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5422; GFX7-NEXT:    flat_store_dword v[0:1], v2
5423; GFX7-NEXT:    s_endpgm
5424;
5425; GFX10-WGP-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
5426; GFX10-WGP:       ; %bb.0: ; %entry
5427; GFX10-WGP-NEXT:    s_clause 0x1
5428; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5429; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5430; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5431; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5432; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5433; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5434; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5435; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5436; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5437; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5438; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5439; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5440; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5441; GFX10-WGP-NEXT:    buffer_gl0_inv
5442; GFX10-WGP-NEXT:    buffer_gl1_inv
5443; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5444; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5445; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5446; GFX10-WGP-NEXT:    s_endpgm
5447;
5448; GFX10-CU-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
5449; GFX10-CU:       ; %bb.0: ; %entry
5450; GFX10-CU-NEXT:    s_clause 0x1
5451; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5452; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5453; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5454; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5455; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5456; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5457; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5458; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5459; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5460; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5461; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5462; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5463; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5464; GFX10-CU-NEXT:    buffer_gl0_inv
5465; GFX10-CU-NEXT:    buffer_gl1_inv
5466; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5467; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5468; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5469; GFX10-CU-NEXT:    s_endpgm
5470;
5471; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
5472; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5473; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5474; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5475; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5476; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5477; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5478; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5479; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5480; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5481; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5482; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5483; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5484; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5485; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5486; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5487; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5488; SKIP-CACHE-INV-NEXT:    s_endpgm
5489;
5490; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
5491; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5492; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5493; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5494; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5495; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5496; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5497; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5498; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5499; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5500; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5501; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5502; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5503;
5504; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
5505; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5506; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5507; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5508; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5509; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5510; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5511; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5512; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5513; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5514; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5515; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5516; GFX90A-TGSPLIT-NEXT:    s_endpgm
5517;
5518; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
5519; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5520; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5521; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5522; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5523; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5524; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5525; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
5526; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5527; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5528; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5529; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
5530; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5531; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5532;
5533; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
5534; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5535; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5536; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5537; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5538; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5539; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5540; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
5541; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5542; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5543; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5544; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
5545; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5546; GFX940-TGSPLIT-NEXT:    s_endpgm
5547;
5548; GFX11-WGP-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
5549; GFX11-WGP:       ; %bb.0: ; %entry
5550; GFX11-WGP-NEXT:    s_clause 0x1
5551; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5552; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5553; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5554; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5555; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5556; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5557; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5558; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5559; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5560; GFX11-WGP-NEXT:    buffer_gl0_inv
5561; GFX11-WGP-NEXT:    buffer_gl1_inv
5562; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5563; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5564; GFX11-WGP-NEXT:    s_endpgm
5565;
5566; GFX11-CU-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
5567; GFX11-CU:       ; %bb.0: ; %entry
5568; GFX11-CU-NEXT:    s_clause 0x1
5569; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5570; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5571; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5572; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5573; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5574; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5575; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5576; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5577; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5578; GFX11-CU-NEXT:    buffer_gl0_inv
5579; GFX11-CU-NEXT:    buffer_gl1_inv
5580; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5581; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5582; GFX11-CU-NEXT:    s_endpgm
5583    i32* %out, i32 %in, i32 %old) {
5584entry:
5585  %gep = getelementptr i32, i32* %out, i32 4
5586  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst monotonic
5587  %val0 = extractvalue { i32, i1 } %val, 0
5588  store i32 %val0, i32* %out, align 4
5589  ret void
5590}
5591
5592define amdgpu_kernel void @flat_agent_monotonic_acquire_ret_cmpxchg(
5593; GFX7-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
5594; GFX7:       ; %bb.0: ; %entry
5595; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5596; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5597; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5598; GFX7-NEXT:    s_add_u32 s4, s0, 16
5599; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5600; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5601; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5602; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5603; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5604; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5605; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5606; GFX7-NEXT:    buffer_wbinvl1_vol
5607; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5608; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5609; GFX7-NEXT:    flat_store_dword v[0:1], v2
5610; GFX7-NEXT:    s_endpgm
5611;
5612; GFX10-WGP-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
5613; GFX10-WGP:       ; %bb.0: ; %entry
5614; GFX10-WGP-NEXT:    s_clause 0x1
5615; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5616; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5617; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5618; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5619; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5620; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5621; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5622; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5623; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5624; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5625; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5626; GFX10-WGP-NEXT:    buffer_gl0_inv
5627; GFX10-WGP-NEXT:    buffer_gl1_inv
5628; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5629; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5630; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5631; GFX10-WGP-NEXT:    s_endpgm
5632;
5633; GFX10-CU-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
5634; GFX10-CU:       ; %bb.0: ; %entry
5635; GFX10-CU-NEXT:    s_clause 0x1
5636; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5637; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5638; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5639; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5640; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5641; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5642; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5643; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5644; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5645; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5646; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5647; GFX10-CU-NEXT:    buffer_gl0_inv
5648; GFX10-CU-NEXT:    buffer_gl1_inv
5649; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5650; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5651; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5652; GFX10-CU-NEXT:    s_endpgm
5653;
5654; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
5655; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5656; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5657; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5658; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5659; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5660; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5661; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5662; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5663; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5664; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5665; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5666; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5667; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5668; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5669; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5670; SKIP-CACHE-INV-NEXT:    s_endpgm
5671;
5672; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
5673; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5674; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5675; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5676; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5677; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5678; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5679; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5680; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5681; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5682; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5683; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5684;
5685; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
5686; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5687; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5688; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5689; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5690; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5691; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5692; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5693; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5694; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5695; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5696; GFX90A-TGSPLIT-NEXT:    s_endpgm
5697;
5698; GFX940-NOTTGSPLIT-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
5699; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5700; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5701; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5702; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5703; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5704; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5705; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5706; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5707; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
5708; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5709; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5710;
5711; GFX940-TGSPLIT-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
5712; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5713; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5714; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5715; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5716; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5717; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5718; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5719; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5720; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
5721; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5722; GFX940-TGSPLIT-NEXT:    s_endpgm
5723;
5724; GFX11-WGP-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
5725; GFX11-WGP:       ; %bb.0: ; %entry
5726; GFX11-WGP-NEXT:    s_clause 0x1
5727; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5728; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5729; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5730; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5731; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5732; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5733; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5734; GFX11-WGP-NEXT:    buffer_gl0_inv
5735; GFX11-WGP-NEXT:    buffer_gl1_inv
5736; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5737; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5738; GFX11-WGP-NEXT:    s_endpgm
5739;
5740; GFX11-CU-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
5741; GFX11-CU:       ; %bb.0: ; %entry
5742; GFX11-CU-NEXT:    s_clause 0x1
5743; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5744; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5745; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5746; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5747; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5748; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5749; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5750; GFX11-CU-NEXT:    buffer_gl0_inv
5751; GFX11-CU-NEXT:    buffer_gl1_inv
5752; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5753; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5754; GFX11-CU-NEXT:    s_endpgm
5755    i32* %out, i32 %in, i32 %old) {
5756entry:
5757  %gep = getelementptr i32, i32* %out, i32 4
5758  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") monotonic acquire
5759  %val0 = extractvalue { i32, i1 } %val, 0
5760  store i32 %val0, i32* %out, align 4
5761  ret void
5762}
5763
5764define amdgpu_kernel void @flat_agent_acquire_acquire_ret_cmpxchg(
5765; GFX7-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
5766; GFX7:       ; %bb.0: ; %entry
5767; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5768; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5769; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5770; GFX7-NEXT:    s_add_u32 s4, s0, 16
5771; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5772; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5773; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5774; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5775; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5776; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5777; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5778; GFX7-NEXT:    buffer_wbinvl1_vol
5779; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5780; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5781; GFX7-NEXT:    flat_store_dword v[0:1], v2
5782; GFX7-NEXT:    s_endpgm
5783;
5784; GFX10-WGP-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
5785; GFX10-WGP:       ; %bb.0: ; %entry
5786; GFX10-WGP-NEXT:    s_clause 0x1
5787; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5788; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5789; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5790; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5791; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5792; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5793; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5794; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5795; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5796; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5797; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5798; GFX10-WGP-NEXT:    buffer_gl0_inv
5799; GFX10-WGP-NEXT:    buffer_gl1_inv
5800; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5801; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5802; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5803; GFX10-WGP-NEXT:    s_endpgm
5804;
5805; GFX10-CU-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
5806; GFX10-CU:       ; %bb.0: ; %entry
5807; GFX10-CU-NEXT:    s_clause 0x1
5808; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5809; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5810; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5811; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5812; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5813; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5814; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5815; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5816; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5817; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5818; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5819; GFX10-CU-NEXT:    buffer_gl0_inv
5820; GFX10-CU-NEXT:    buffer_gl1_inv
5821; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5822; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5823; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5824; GFX10-CU-NEXT:    s_endpgm
5825;
5826; SKIP-CACHE-INV-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
5827; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5828; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5829; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5830; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5831; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5832; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5833; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5834; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5835; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5836; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5837; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5838; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5839; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5840; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5841; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5842; SKIP-CACHE-INV-NEXT:    s_endpgm
5843;
5844; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
5845; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5846; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5847; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5848; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5849; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5850; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5851; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5852; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5853; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5854; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5855; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5856;
5857; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
5858; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5859; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5860; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5861; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5862; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5863; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5864; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5865; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5866; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5867; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5868; GFX90A-TGSPLIT-NEXT:    s_endpgm
5869;
5870; GFX940-NOTTGSPLIT-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
5871; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5872; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5873; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5874; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5875; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5876; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5877; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5878; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5879; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
5880; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5881; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5882;
5883; GFX940-TGSPLIT-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
5884; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5885; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5886; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5887; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5888; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5889; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5890; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5891; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5892; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
5893; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5894; GFX940-TGSPLIT-NEXT:    s_endpgm
5895;
5896; GFX11-WGP-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
5897; GFX11-WGP:       ; %bb.0: ; %entry
5898; GFX11-WGP-NEXT:    s_clause 0x1
5899; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5900; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5901; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5902; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5903; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5904; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5905; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5906; GFX11-WGP-NEXT:    buffer_gl0_inv
5907; GFX11-WGP-NEXT:    buffer_gl1_inv
5908; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5909; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5910; GFX11-WGP-NEXT:    s_endpgm
5911;
5912; GFX11-CU-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
5913; GFX11-CU:       ; %bb.0: ; %entry
5914; GFX11-CU-NEXT:    s_clause 0x1
5915; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5916; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5917; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5918; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5919; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
5920; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5921; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5922; GFX11-CU-NEXT:    buffer_gl0_inv
5923; GFX11-CU-NEXT:    buffer_gl1_inv
5924; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5925; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
5926; GFX11-CU-NEXT:    s_endpgm
5927    i32* %out, i32 %in, i32 %old) {
5928entry:
5929  %gep = getelementptr i32, i32* %out, i32 4
5930  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire acquire
5931  %val0 = extractvalue { i32, i1 } %val, 0
5932  store i32 %val0, i32* %out, align 4
5933  ret void
5934}
5935
5936define amdgpu_kernel void @flat_agent_release_acquire_ret_cmpxchg(
5937; GFX7-LABEL: flat_agent_release_acquire_ret_cmpxchg:
5938; GFX7:       ; %bb.0: ; %entry
5939; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5940; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5941; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5942; GFX7-NEXT:    s_add_u32 s4, s0, 16
5943; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5944; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5945; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5946; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5947; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5948; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5949; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5950; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5951; GFX7-NEXT:    buffer_wbinvl1_vol
5952; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5953; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5954; GFX7-NEXT:    flat_store_dword v[0:1], v2
5955; GFX7-NEXT:    s_endpgm
5956;
5957; GFX10-WGP-LABEL: flat_agent_release_acquire_ret_cmpxchg:
5958; GFX10-WGP:       ; %bb.0: ; %entry
5959; GFX10-WGP-NEXT:    s_clause 0x1
5960; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5961; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5962; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5963; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5964; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5965; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5966; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5967; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5968; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5969; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5970; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5971; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5972; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5973; GFX10-WGP-NEXT:    buffer_gl0_inv
5974; GFX10-WGP-NEXT:    buffer_gl1_inv
5975; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5976; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5977; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5978; GFX10-WGP-NEXT:    s_endpgm
5979;
5980; GFX10-CU-LABEL: flat_agent_release_acquire_ret_cmpxchg:
5981; GFX10-CU:       ; %bb.0: ; %entry
5982; GFX10-CU-NEXT:    s_clause 0x1
5983; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5984; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5985; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5986; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5987; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5988; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5989; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5990; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5991; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5992; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5993; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5994; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5995; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5996; GFX10-CU-NEXT:    buffer_gl0_inv
5997; GFX10-CU-NEXT:    buffer_gl1_inv
5998; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5999; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6000; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6001; GFX10-CU-NEXT:    s_endpgm
6002;
6003; SKIP-CACHE-INV-LABEL: flat_agent_release_acquire_ret_cmpxchg:
6004; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6005; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6006; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6007; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6008; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6009; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6010; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6011; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6012; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6013; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6014; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6015; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6016; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6017; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6018; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6019; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6020; SKIP-CACHE-INV-NEXT:    s_endpgm
6021;
6022; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_acquire_ret_cmpxchg:
6023; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6024; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6025; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6026; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6027; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6028; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6029; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6030; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6031; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6032; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6033; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6034; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6035;
6036; GFX90A-TGSPLIT-LABEL: flat_agent_release_acquire_ret_cmpxchg:
6037; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6038; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6039; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6040; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6041; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6042; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6043; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6044; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6045; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6046; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6047; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6048; GFX90A-TGSPLIT-NEXT:    s_endpgm
6049;
6050; GFX940-NOTTGSPLIT-LABEL: flat_agent_release_acquire_ret_cmpxchg:
6051; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6052; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6053; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6054; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6055; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6056; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6057; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
6058; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6059; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6060; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6061; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
6062; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6063; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6064;
6065; GFX940-TGSPLIT-LABEL: flat_agent_release_acquire_ret_cmpxchg:
6066; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6067; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6068; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6069; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6070; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6071; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6072; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
6073; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6074; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6075; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6076; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
6077; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6078; GFX940-TGSPLIT-NEXT:    s_endpgm
6079;
6080; GFX11-WGP-LABEL: flat_agent_release_acquire_ret_cmpxchg:
6081; GFX11-WGP:       ; %bb.0: ; %entry
6082; GFX11-WGP-NEXT:    s_clause 0x1
6083; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6084; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6085; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6086; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6087; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
6088; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6089; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6090; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6091; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6092; GFX11-WGP-NEXT:    buffer_gl0_inv
6093; GFX11-WGP-NEXT:    buffer_gl1_inv
6094; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6095; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6096; GFX11-WGP-NEXT:    s_endpgm
6097;
6098; GFX11-CU-LABEL: flat_agent_release_acquire_ret_cmpxchg:
6099; GFX11-CU:       ; %bb.0: ; %entry
6100; GFX11-CU-NEXT:    s_clause 0x1
6101; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6102; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6103; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6104; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6105; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
6106; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6107; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6108; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6109; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6110; GFX11-CU-NEXT:    buffer_gl0_inv
6111; GFX11-CU-NEXT:    buffer_gl1_inv
6112; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6113; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6114; GFX11-CU-NEXT:    s_endpgm
6115    i32* %out, i32 %in, i32 %old) {
6116entry:
6117  %gep = getelementptr i32, i32* %out, i32 4
6118  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release acquire
6119  %val0 = extractvalue { i32, i1 } %val, 0
6120  store i32 %val0, i32* %out, align 4
6121  ret void
6122}
6123
6124define amdgpu_kernel void @flat_agent_acq_rel_acquire_ret_cmpxchg(
6125; GFX7-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
6126; GFX7:       ; %bb.0: ; %entry
6127; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6128; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6129; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6130; GFX7-NEXT:    s_add_u32 s4, s0, 16
6131; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6132; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6133; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6134; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6135; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6136; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6137; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6138; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6139; GFX7-NEXT:    buffer_wbinvl1_vol
6140; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6141; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6142; GFX7-NEXT:    flat_store_dword v[0:1], v2
6143; GFX7-NEXT:    s_endpgm
6144;
6145; GFX10-WGP-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
6146; GFX10-WGP:       ; %bb.0: ; %entry
6147; GFX10-WGP-NEXT:    s_clause 0x1
6148; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6149; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6150; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6151; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6152; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6153; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6154; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6155; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6156; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6157; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6158; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6159; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6160; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6161; GFX10-WGP-NEXT:    buffer_gl0_inv
6162; GFX10-WGP-NEXT:    buffer_gl1_inv
6163; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6164; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6165; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6166; GFX10-WGP-NEXT:    s_endpgm
6167;
6168; GFX10-CU-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
6169; GFX10-CU:       ; %bb.0: ; %entry
6170; GFX10-CU-NEXT:    s_clause 0x1
6171; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6172; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6173; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6174; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6175; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6176; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6177; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6178; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6179; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6180; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6181; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6182; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6183; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6184; GFX10-CU-NEXT:    buffer_gl0_inv
6185; GFX10-CU-NEXT:    buffer_gl1_inv
6186; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6187; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6188; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6189; GFX10-CU-NEXT:    s_endpgm
6190;
6191; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
6192; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6193; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6194; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6195; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6196; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6197; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6198; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6199; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6200; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6201; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6202; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6203; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6204; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6205; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6206; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6207; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6208; SKIP-CACHE-INV-NEXT:    s_endpgm
6209;
6210; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
6211; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6212; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6213; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6214; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6215; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6216; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6217; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6218; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6219; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6220; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6221; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6222; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6223;
6224; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
6225; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6226; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6227; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6228; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6229; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6230; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6231; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6232; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6233; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6234; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6235; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6236; GFX90A-TGSPLIT-NEXT:    s_endpgm
6237;
6238; GFX940-NOTTGSPLIT-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
6239; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6240; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6241; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6242; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6243; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6244; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6245; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
6246; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6247; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6248; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6249; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
6250; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6251; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6252;
6253; GFX940-TGSPLIT-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
6254; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6255; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6256; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6257; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6258; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6259; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6260; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
6261; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6262; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6263; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6264; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
6265; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6266; GFX940-TGSPLIT-NEXT:    s_endpgm
6267;
6268; GFX11-WGP-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
6269; GFX11-WGP:       ; %bb.0: ; %entry
6270; GFX11-WGP-NEXT:    s_clause 0x1
6271; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6272; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6273; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6274; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6275; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
6276; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6277; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6278; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6279; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6280; GFX11-WGP-NEXT:    buffer_gl0_inv
6281; GFX11-WGP-NEXT:    buffer_gl1_inv
6282; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6283; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6284; GFX11-WGP-NEXT:    s_endpgm
6285;
6286; GFX11-CU-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
6287; GFX11-CU:       ; %bb.0: ; %entry
6288; GFX11-CU-NEXT:    s_clause 0x1
6289; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6290; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6291; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6292; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6293; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
6294; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6295; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6296; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6297; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6298; GFX11-CU-NEXT:    buffer_gl0_inv
6299; GFX11-CU-NEXT:    buffer_gl1_inv
6300; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6301; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6302; GFX11-CU-NEXT:    s_endpgm
6303    i32* %out, i32 %in, i32 %old) {
6304entry:
6305  %gep = getelementptr i32, i32* %out, i32 4
6306  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel acquire
6307  %val0 = extractvalue { i32, i1 } %val, 0
6308  store i32 %val0, i32* %out, align 4
6309  ret void
6310}
6311
6312define amdgpu_kernel void @flat_agent_seq_cst_acquire_ret_cmpxchg(
6313; GFX7-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
6314; GFX7:       ; %bb.0: ; %entry
6315; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6316; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6317; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6318; GFX7-NEXT:    s_add_u32 s4, s0, 16
6319; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6320; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6321; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6322; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6323; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6324; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6325; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6326; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6327; GFX7-NEXT:    buffer_wbinvl1_vol
6328; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6329; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6330; GFX7-NEXT:    flat_store_dword v[0:1], v2
6331; GFX7-NEXT:    s_endpgm
6332;
6333; GFX10-WGP-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
6334; GFX10-WGP:       ; %bb.0: ; %entry
6335; GFX10-WGP-NEXT:    s_clause 0x1
6336; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6337; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6338; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6339; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6340; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6341; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6342; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6343; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6344; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6345; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6346; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6347; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6348; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6349; GFX10-WGP-NEXT:    buffer_gl0_inv
6350; GFX10-WGP-NEXT:    buffer_gl1_inv
6351; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6352; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6353; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6354; GFX10-WGP-NEXT:    s_endpgm
6355;
6356; GFX10-CU-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
6357; GFX10-CU:       ; %bb.0: ; %entry
6358; GFX10-CU-NEXT:    s_clause 0x1
6359; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6360; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6361; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6362; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6363; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6364; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6365; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6366; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6367; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6368; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6369; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6370; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6371; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6372; GFX10-CU-NEXT:    buffer_gl0_inv
6373; GFX10-CU-NEXT:    buffer_gl1_inv
6374; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6375; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6376; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6377; GFX10-CU-NEXT:    s_endpgm
6378;
6379; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
6380; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6381; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6382; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6383; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6384; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6385; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6386; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6387; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6388; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6389; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6390; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6391; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6392; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6393; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6394; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6395; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6396; SKIP-CACHE-INV-NEXT:    s_endpgm
6397;
6398; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
6399; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6400; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6401; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6402; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6403; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6404; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6405; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6406; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6407; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6408; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6409; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6410; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6411;
6412; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
6413; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6414; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6415; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6416; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6417; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6418; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6419; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6420; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6421; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6422; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6423; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6424; GFX90A-TGSPLIT-NEXT:    s_endpgm
6425;
6426; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
6427; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6428; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6429; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6430; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6431; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6432; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6433; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
6434; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6435; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6436; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6437; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
6438; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6439; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6440;
6441; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
6442; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6443; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6444; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6445; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6446; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6447; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6448; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
6449; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6450; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6451; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6452; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
6453; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6454; GFX940-TGSPLIT-NEXT:    s_endpgm
6455;
6456; GFX11-WGP-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
6457; GFX11-WGP:       ; %bb.0: ; %entry
6458; GFX11-WGP-NEXT:    s_clause 0x1
6459; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6460; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6461; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6462; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6463; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
6464; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6465; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6466; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6467; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6468; GFX11-WGP-NEXT:    buffer_gl0_inv
6469; GFX11-WGP-NEXT:    buffer_gl1_inv
6470; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6471; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6472; GFX11-WGP-NEXT:    s_endpgm
6473;
6474; GFX11-CU-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
6475; GFX11-CU:       ; %bb.0: ; %entry
6476; GFX11-CU-NEXT:    s_clause 0x1
6477; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6478; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6479; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6480; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6481; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
6482; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6483; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6484; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6485; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6486; GFX11-CU-NEXT:    buffer_gl0_inv
6487; GFX11-CU-NEXT:    buffer_gl1_inv
6488; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6489; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6490; GFX11-CU-NEXT:    s_endpgm
6491    i32* %out, i32 %in, i32 %old) {
6492entry:
6493  %gep = getelementptr i32, i32* %out, i32 4
6494  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst acquire
6495  %val0 = extractvalue { i32, i1 } %val, 0
6496  store i32 %val0, i32* %out, align 4
6497  ret void
6498}
6499
6500define amdgpu_kernel void @flat_agent_monotonic_seq_cst_ret_cmpxchg(
6501; GFX7-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
6502; GFX7:       ; %bb.0: ; %entry
6503; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6504; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6505; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6506; GFX7-NEXT:    s_add_u32 s4, s0, 16
6507; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6508; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6509; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6510; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6511; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6512; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6513; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6514; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6515; GFX7-NEXT:    buffer_wbinvl1_vol
6516; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6517; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6518; GFX7-NEXT:    flat_store_dword v[0:1], v2
6519; GFX7-NEXT:    s_endpgm
6520;
6521; GFX10-WGP-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
6522; GFX10-WGP:       ; %bb.0: ; %entry
6523; GFX10-WGP-NEXT:    s_clause 0x1
6524; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6525; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6526; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6527; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6528; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6529; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6530; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6531; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6532; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6533; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6534; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6535; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6536; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6537; GFX10-WGP-NEXT:    buffer_gl0_inv
6538; GFX10-WGP-NEXT:    buffer_gl1_inv
6539; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6540; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6541; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6542; GFX10-WGP-NEXT:    s_endpgm
6543;
6544; GFX10-CU-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
6545; GFX10-CU:       ; %bb.0: ; %entry
6546; GFX10-CU-NEXT:    s_clause 0x1
6547; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6548; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6549; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6550; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6551; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6552; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6553; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6554; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6555; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6556; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6557; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6558; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6559; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6560; GFX10-CU-NEXT:    buffer_gl0_inv
6561; GFX10-CU-NEXT:    buffer_gl1_inv
6562; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6563; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6564; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6565; GFX10-CU-NEXT:    s_endpgm
6566;
6567; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
6568; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6569; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6570; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6571; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6572; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6573; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6574; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6575; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6576; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6577; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6578; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6579; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6580; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6581; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6582; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6583; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6584; SKIP-CACHE-INV-NEXT:    s_endpgm
6585;
6586; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
6587; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6588; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6589; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6590; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6591; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6592; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6593; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6594; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6595; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6596; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6597; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6598; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6599;
6600; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
6601; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6602; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6603; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6604; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6605; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6606; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6607; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6608; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6609; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6610; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6611; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6612; GFX90A-TGSPLIT-NEXT:    s_endpgm
6613;
6614; GFX940-NOTTGSPLIT-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
6615; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6616; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6617; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6618; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6619; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6620; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6621; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
6622; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6623; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6624; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6625; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
6626; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6627; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6628;
6629; GFX940-TGSPLIT-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
6630; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6631; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6632; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6633; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6634; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6635; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6636; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
6637; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6638; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6639; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6640; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
6641; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6642; GFX940-TGSPLIT-NEXT:    s_endpgm
6643;
6644; GFX11-WGP-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
6645; GFX11-WGP:       ; %bb.0: ; %entry
6646; GFX11-WGP-NEXT:    s_clause 0x1
6647; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6648; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6649; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6650; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6651; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
6652; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6653; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6654; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6655; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6656; GFX11-WGP-NEXT:    buffer_gl0_inv
6657; GFX11-WGP-NEXT:    buffer_gl1_inv
6658; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6659; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6660; GFX11-WGP-NEXT:    s_endpgm
6661;
6662; GFX11-CU-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
6663; GFX11-CU:       ; %bb.0: ; %entry
6664; GFX11-CU-NEXT:    s_clause 0x1
6665; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6666; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6667; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6668; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6669; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
6670; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6671; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6672; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6673; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6674; GFX11-CU-NEXT:    buffer_gl0_inv
6675; GFX11-CU-NEXT:    buffer_gl1_inv
6676; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6677; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6678; GFX11-CU-NEXT:    s_endpgm
6679    i32* %out, i32 %in, i32 %old) {
6680entry:
6681  %gep = getelementptr i32, i32* %out, i32 4
6682  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") monotonic seq_cst
6683  %val0 = extractvalue { i32, i1 } %val, 0
6684  store i32 %val0, i32* %out, align 4
6685  ret void
6686}
6687
6688define amdgpu_kernel void @flat_agent_acquire_seq_cst_ret_cmpxchg(
6689; GFX7-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
6690; GFX7:       ; %bb.0: ; %entry
6691; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6692; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6693; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6694; GFX7-NEXT:    s_add_u32 s4, s0, 16
6695; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6696; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6697; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6698; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6699; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6700; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6701; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6702; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6703; GFX7-NEXT:    buffer_wbinvl1_vol
6704; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6705; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6706; GFX7-NEXT:    flat_store_dword v[0:1], v2
6707; GFX7-NEXT:    s_endpgm
6708;
6709; GFX10-WGP-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
6710; GFX10-WGP:       ; %bb.0: ; %entry
6711; GFX10-WGP-NEXT:    s_clause 0x1
6712; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6713; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6714; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6715; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6716; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6717; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6718; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6719; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6720; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6721; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6722; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6723; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6724; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6725; GFX10-WGP-NEXT:    buffer_gl0_inv
6726; GFX10-WGP-NEXT:    buffer_gl1_inv
6727; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6728; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6729; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6730; GFX10-WGP-NEXT:    s_endpgm
6731;
6732; GFX10-CU-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
6733; GFX10-CU:       ; %bb.0: ; %entry
6734; GFX10-CU-NEXT:    s_clause 0x1
6735; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6736; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6737; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6738; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6739; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6740; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6741; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6742; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6743; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6744; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6745; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6746; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6747; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6748; GFX10-CU-NEXT:    buffer_gl0_inv
6749; GFX10-CU-NEXT:    buffer_gl1_inv
6750; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6751; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6752; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6753; GFX10-CU-NEXT:    s_endpgm
6754;
6755; SKIP-CACHE-INV-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
6756; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6757; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6758; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6759; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6760; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6761; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6762; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6763; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6764; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6765; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6766; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6767; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6768; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6769; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6770; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6771; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6772; SKIP-CACHE-INV-NEXT:    s_endpgm
6773;
6774; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
6775; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6776; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6777; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6778; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6779; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6780; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6781; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6782; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6783; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6784; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6785; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6786; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6787;
6788; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
6789; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6790; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6791; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6792; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6793; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6794; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6795; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6796; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6797; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6798; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6799; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6800; GFX90A-TGSPLIT-NEXT:    s_endpgm
6801;
6802; GFX940-NOTTGSPLIT-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
6803; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6804; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6805; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6806; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6807; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6808; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6809; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
6810; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6811; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6812; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6813; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
6814; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6815; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6816;
6817; GFX940-TGSPLIT-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
6818; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6819; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6820; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6821; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6822; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6823; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6824; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
6825; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6826; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6827; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6828; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
6829; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6830; GFX940-TGSPLIT-NEXT:    s_endpgm
6831;
6832; GFX11-WGP-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
6833; GFX11-WGP:       ; %bb.0: ; %entry
6834; GFX11-WGP-NEXT:    s_clause 0x1
6835; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6836; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6837; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6838; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6839; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
6840; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6841; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6842; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6843; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6844; GFX11-WGP-NEXT:    buffer_gl0_inv
6845; GFX11-WGP-NEXT:    buffer_gl1_inv
6846; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6847; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6848; GFX11-WGP-NEXT:    s_endpgm
6849;
6850; GFX11-CU-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
6851; GFX11-CU:       ; %bb.0: ; %entry
6852; GFX11-CU-NEXT:    s_clause 0x1
6853; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6854; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6855; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6856; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6857; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
6858; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6859; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6860; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6861; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6862; GFX11-CU-NEXT:    buffer_gl0_inv
6863; GFX11-CU-NEXT:    buffer_gl1_inv
6864; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6865; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
6866; GFX11-CU-NEXT:    s_endpgm
6867    i32* %out, i32 %in, i32 %old) {
6868entry:
6869  %gep = getelementptr i32, i32* %out, i32 4
6870  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire seq_cst
6871  %val0 = extractvalue { i32, i1 } %val, 0
6872  store i32 %val0, i32* %out, align 4
6873  ret void
6874}
6875
6876define amdgpu_kernel void @flat_agent_release_seq_cst_ret_cmpxchg(
6877; GFX7-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
6878; GFX7:       ; %bb.0: ; %entry
6879; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6880; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6881; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6882; GFX7-NEXT:    s_add_u32 s4, s0, 16
6883; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6884; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6885; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6886; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6887; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6888; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6889; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6890; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6891; GFX7-NEXT:    buffer_wbinvl1_vol
6892; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6893; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6894; GFX7-NEXT:    flat_store_dword v[0:1], v2
6895; GFX7-NEXT:    s_endpgm
6896;
6897; GFX10-WGP-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
6898; GFX10-WGP:       ; %bb.0: ; %entry
6899; GFX10-WGP-NEXT:    s_clause 0x1
6900; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6901; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6902; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6903; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6904; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6905; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6906; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6907; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6908; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6909; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6910; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6911; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6912; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6913; GFX10-WGP-NEXT:    buffer_gl0_inv
6914; GFX10-WGP-NEXT:    buffer_gl1_inv
6915; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6916; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6917; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6918; GFX10-WGP-NEXT:    s_endpgm
6919;
6920; GFX10-CU-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
6921; GFX10-CU:       ; %bb.0: ; %entry
6922; GFX10-CU-NEXT:    s_clause 0x1
6923; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6924; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6925; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6926; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6927; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6928; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6929; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6930; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6931; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6932; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6933; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6934; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6935; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6936; GFX10-CU-NEXT:    buffer_gl0_inv
6937; GFX10-CU-NEXT:    buffer_gl1_inv
6938; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6939; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6940; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6941; GFX10-CU-NEXT:    s_endpgm
6942;
6943; SKIP-CACHE-INV-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
6944; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6945; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6946; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6947; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6948; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6949; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6950; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6951; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6952; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6953; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6954; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6955; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6956; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6957; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6958; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6959; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6960; SKIP-CACHE-INV-NEXT:    s_endpgm
6961;
6962; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
6963; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6964; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6965; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6966; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6967; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6968; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6969; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6970; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6971; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6972; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6973; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6974; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6975;
6976; GFX90A-TGSPLIT-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
6977; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6978; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6979; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6980; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6981; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6982; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6983; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6984; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6985; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6986; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6987; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6988; GFX90A-TGSPLIT-NEXT:    s_endpgm
6989;
6990; GFX940-NOTTGSPLIT-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
6991; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6992; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6993; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6994; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6995; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6996; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6997; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
6998; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6999; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
7000; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7001; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
7002; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7003; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7004;
7005; GFX940-TGSPLIT-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
7006; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7007; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7008; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7009; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7010; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7011; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7012; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
7013; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7014; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
7015; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7016; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
7017; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7018; GFX940-TGSPLIT-NEXT:    s_endpgm
7019;
7020; GFX11-WGP-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
7021; GFX11-WGP:       ; %bb.0: ; %entry
7022; GFX11-WGP-NEXT:    s_clause 0x1
7023; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7024; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7025; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7026; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7027; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
7028; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7029; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7030; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7031; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7032; GFX11-WGP-NEXT:    buffer_gl0_inv
7033; GFX11-WGP-NEXT:    buffer_gl1_inv
7034; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7035; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7036; GFX11-WGP-NEXT:    s_endpgm
7037;
7038; GFX11-CU-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
7039; GFX11-CU:       ; %bb.0: ; %entry
7040; GFX11-CU-NEXT:    s_clause 0x1
7041; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7042; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7043; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7044; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7045; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
7046; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7047; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7048; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7049; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7050; GFX11-CU-NEXT:    buffer_gl0_inv
7051; GFX11-CU-NEXT:    buffer_gl1_inv
7052; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7053; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7054; GFX11-CU-NEXT:    s_endpgm
7055    i32* %out, i32 %in, i32 %old) {
7056entry:
7057  %gep = getelementptr i32, i32* %out, i32 4
7058  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release seq_cst
7059  %val0 = extractvalue { i32, i1 } %val, 0
7060  store i32 %val0, i32* %out, align 4
7061  ret void
7062}
7063
7064define amdgpu_kernel void @flat_agent_acq_rel_seq_cst_ret_cmpxchg(
7065; GFX7-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
7066; GFX7:       ; %bb.0: ; %entry
7067; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7068; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7069; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7070; GFX7-NEXT:    s_add_u32 s4, s0, 16
7071; GFX7-NEXT:    s_addc_u32 s5, s1, 0
7072; GFX7-NEXT:    v_mov_b32_e32 v0, s4
7073; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7074; GFX7-NEXT:    v_mov_b32_e32 v1, s5
7075; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7076; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7077; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7078; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7079; GFX7-NEXT:    buffer_wbinvl1_vol
7080; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7081; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7082; GFX7-NEXT:    flat_store_dword v[0:1], v2
7083; GFX7-NEXT:    s_endpgm
7084;
7085; GFX10-WGP-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
7086; GFX10-WGP:       ; %bb.0: ; %entry
7087; GFX10-WGP-NEXT:    s_clause 0x1
7088; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7089; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7090; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7091; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
7092; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
7093; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
7094; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7095; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
7096; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7097; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7098; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7099; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7100; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7101; GFX10-WGP-NEXT:    buffer_gl0_inv
7102; GFX10-WGP-NEXT:    buffer_gl1_inv
7103; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7104; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7105; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7106; GFX10-WGP-NEXT:    s_endpgm
7107;
7108; GFX10-CU-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
7109; GFX10-CU:       ; %bb.0: ; %entry
7110; GFX10-CU-NEXT:    s_clause 0x1
7111; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7112; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7113; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7114; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
7115; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
7116; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
7117; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7118; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
7119; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7120; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7121; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7122; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7123; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7124; GFX10-CU-NEXT:    buffer_gl0_inv
7125; GFX10-CU-NEXT:    buffer_gl1_inv
7126; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7127; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7128; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7129; GFX10-CU-NEXT:    s_endpgm
7130;
7131; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
7132; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7133; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7134; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7135; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7136; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
7137; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
7138; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7139; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7140; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7141; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7142; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7143; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7144; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7145; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7146; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7147; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7148; SKIP-CACHE-INV-NEXT:    s_endpgm
7149;
7150; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
7151; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7152; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7153; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7154; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7155; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7156; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7157; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7158; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7159; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7160; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7161; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7162; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7163;
7164; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
7165; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7166; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7167; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7168; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7169; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7170; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7171; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7172; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7173; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7174; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7175; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7176; GFX90A-TGSPLIT-NEXT:    s_endpgm
7177;
7178; GFX940-NOTTGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
7179; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7180; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7181; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7182; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7183; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7184; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7185; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
7186; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7187; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
7188; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7189; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
7190; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7191; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7192;
7193; GFX940-TGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
7194; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7195; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7196; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7197; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7198; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7199; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7200; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
7201; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7202; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
7203; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7204; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
7205; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7206; GFX940-TGSPLIT-NEXT:    s_endpgm
7207;
7208; GFX11-WGP-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
7209; GFX11-WGP:       ; %bb.0: ; %entry
7210; GFX11-WGP-NEXT:    s_clause 0x1
7211; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7212; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7213; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7214; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7215; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
7216; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7217; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7218; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7219; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7220; GFX11-WGP-NEXT:    buffer_gl0_inv
7221; GFX11-WGP-NEXT:    buffer_gl1_inv
7222; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7223; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7224; GFX11-WGP-NEXT:    s_endpgm
7225;
7226; GFX11-CU-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
7227; GFX11-CU:       ; %bb.0: ; %entry
7228; GFX11-CU-NEXT:    s_clause 0x1
7229; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7230; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7231; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7232; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7233; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
7234; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7235; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7236; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7237; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7238; GFX11-CU-NEXT:    buffer_gl0_inv
7239; GFX11-CU-NEXT:    buffer_gl1_inv
7240; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7241; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7242; GFX11-CU-NEXT:    s_endpgm
7243    i32* %out, i32 %in, i32 %old) {
7244entry:
7245  %gep = getelementptr i32, i32* %out, i32 4
7246  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel seq_cst
7247  %val0 = extractvalue { i32, i1 } %val, 0
7248  store i32 %val0, i32* %out, align 4
7249  ret void
7250}
7251
7252define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_ret_cmpxchg(
7253; GFX7-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
7254; GFX7:       ; %bb.0: ; %entry
7255; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7256; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7257; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7258; GFX7-NEXT:    s_add_u32 s4, s0, 16
7259; GFX7-NEXT:    s_addc_u32 s5, s1, 0
7260; GFX7-NEXT:    v_mov_b32_e32 v0, s4
7261; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7262; GFX7-NEXT:    v_mov_b32_e32 v1, s5
7263; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7264; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7265; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7266; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7267; GFX7-NEXT:    buffer_wbinvl1_vol
7268; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7269; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7270; GFX7-NEXT:    flat_store_dword v[0:1], v2
7271; GFX7-NEXT:    s_endpgm
7272;
7273; GFX10-WGP-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
7274; GFX10-WGP:       ; %bb.0: ; %entry
7275; GFX10-WGP-NEXT:    s_clause 0x1
7276; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7277; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7278; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7279; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
7280; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
7281; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
7282; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7283; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
7284; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7285; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7286; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7287; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7288; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7289; GFX10-WGP-NEXT:    buffer_gl0_inv
7290; GFX10-WGP-NEXT:    buffer_gl1_inv
7291; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7292; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7293; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7294; GFX10-WGP-NEXT:    s_endpgm
7295;
7296; GFX10-CU-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
7297; GFX10-CU:       ; %bb.0: ; %entry
7298; GFX10-CU-NEXT:    s_clause 0x1
7299; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7300; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7301; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7302; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
7303; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
7304; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
7305; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7306; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
7307; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7308; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7309; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7310; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7311; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7312; GFX10-CU-NEXT:    buffer_gl0_inv
7313; GFX10-CU-NEXT:    buffer_gl1_inv
7314; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7315; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7316; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7317; GFX10-CU-NEXT:    s_endpgm
7318;
7319; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
7320; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7321; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7322; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7323; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7324; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
7325; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
7326; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7327; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7328; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7329; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7330; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7331; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7332; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7333; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7334; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7335; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7336; SKIP-CACHE-INV-NEXT:    s_endpgm
7337;
7338; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
7339; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7340; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7341; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7342; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7343; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7344; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7345; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7346; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7347; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7348; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7349; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7350; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7351;
7352; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
7353; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7354; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7355; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7356; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7357; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7358; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7359; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7360; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7361; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7362; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7363; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7364; GFX90A-TGSPLIT-NEXT:    s_endpgm
7365;
7366; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
7367; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7368; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7369; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7370; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7371; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7372; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7373; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
7374; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7375; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
7376; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7377; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
7378; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7379; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7380;
7381; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
7382; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7383; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7384; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7385; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7386; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7387; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7388; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
7389; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7390; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
7391; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7392; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
7393; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7394; GFX940-TGSPLIT-NEXT:    s_endpgm
7395;
7396; GFX11-WGP-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
7397; GFX11-WGP:       ; %bb.0: ; %entry
7398; GFX11-WGP-NEXT:    s_clause 0x1
7399; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7400; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7401; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7402; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7403; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
7404; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7405; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7406; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7407; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7408; GFX11-WGP-NEXT:    buffer_gl0_inv
7409; GFX11-WGP-NEXT:    buffer_gl1_inv
7410; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7411; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7412; GFX11-WGP-NEXT:    s_endpgm
7413;
7414; GFX11-CU-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
7415; GFX11-CU:       ; %bb.0: ; %entry
7416; GFX11-CU-NEXT:    s_clause 0x1
7417; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7418; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7419; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7420; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7421; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
7422; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7423; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7424; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7425; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7426; GFX11-CU-NEXT:    buffer_gl0_inv
7427; GFX11-CU-NEXT:    buffer_gl1_inv
7428; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7429; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7430; GFX11-CU-NEXT:    s_endpgm
7431    i32* %out, i32 %in, i32 %old) {
7432entry:
7433  %gep = getelementptr i32, i32* %out, i32 4
7434  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
7435  %val0 = extractvalue { i32, i1 } %val, 0
7436  store i32 %val0, i32* %out, align 4
7437  ret void
7438}
7439
7440define amdgpu_kernel void @flat_agent_one_as_unordered_load(
7441; GFX7-LABEL: flat_agent_one_as_unordered_load:
7442; GFX7:       ; %bb.0: ; %entry
7443; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7444; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7445; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7446; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7447; GFX7-NEXT:    flat_load_dword v2, v[0:1]
7448; GFX7-NEXT:    v_mov_b32_e32 v0, s2
7449; GFX7-NEXT:    v_mov_b32_e32 v1, s3
7450; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7451; GFX7-NEXT:    flat_store_dword v[0:1], v2
7452; GFX7-NEXT:    s_endpgm
7453;
7454; GFX10-WGP-LABEL: flat_agent_one_as_unordered_load:
7455; GFX10-WGP:       ; %bb.0: ; %entry
7456; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7457; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7458; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7459; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7460; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
7461; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
7462; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
7463; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7464; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7465; GFX10-WGP-NEXT:    s_endpgm
7466;
7467; GFX10-CU-LABEL: flat_agent_one_as_unordered_load:
7468; GFX10-CU:       ; %bb.0: ; %entry
7469; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7470; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7471; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7472; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7473; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
7474; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
7475; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
7476; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7477; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7478; GFX10-CU-NEXT:    s_endpgm
7479;
7480; SKIP-CACHE-INV-LABEL: flat_agent_one_as_unordered_load:
7481; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7482; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7483; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7484; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7485; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
7486; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
7487; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7488; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7489; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7490; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7491; SKIP-CACHE-INV-NEXT:    s_endpgm
7492;
7493; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_unordered_load:
7494; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7495; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7496; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7497; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7498; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7499; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
7500; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7501; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7502; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7503; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7504; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7505;
7506; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_unordered_load:
7507; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7508; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7509; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7510; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7511; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7512; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
7513; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7514; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7515; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7516; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7517; GFX90A-TGSPLIT-NEXT:    s_endpgm
7518;
7519; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_unordered_load:
7520; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7521; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7522; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7523; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7524; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7525; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
7526; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7527; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7528; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7529; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7530; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7531;
7532; GFX940-TGSPLIT-LABEL: flat_agent_one_as_unordered_load:
7533; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7534; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7535; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7536; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7537; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7538; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
7539; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7540; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7541; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7542; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7543; GFX940-TGSPLIT-NEXT:    s_endpgm
7544;
7545; GFX11-WGP-LABEL: flat_agent_one_as_unordered_load:
7546; GFX11-WGP:       ; %bb.0: ; %entry
7547; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7548; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7549; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7550; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1]
7551; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7552; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7553; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7554; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7555; GFX11-WGP-NEXT:    s_endpgm
7556;
7557; GFX11-CU-LABEL: flat_agent_one_as_unordered_load:
7558; GFX11-CU:       ; %bb.0: ; %entry
7559; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7560; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7561; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7562; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
7563; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7564; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7565; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7566; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7567; GFX11-CU-NEXT:    s_endpgm
7568    i32* %in, i32* %out) {
7569entry:
7570  %val = load atomic i32, i32* %in syncscope("agent-one-as") unordered, align 4
7571  store i32 %val, i32* %out
7572  ret void
7573}
7574
7575define amdgpu_kernel void @flat_agent_one_as_monotonic_load(
7576; GFX7-LABEL: flat_agent_one_as_monotonic_load:
7577; GFX7:       ; %bb.0: ; %entry
7578; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7579; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7580; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7581; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7582; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
7583; GFX7-NEXT:    v_mov_b32_e32 v0, s2
7584; GFX7-NEXT:    v_mov_b32_e32 v1, s3
7585; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7586; GFX7-NEXT:    flat_store_dword v[0:1], v2
7587; GFX7-NEXT:    s_endpgm
7588;
7589; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_load:
7590; GFX10-WGP:       ; %bb.0: ; %entry
7591; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7592; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7593; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7594; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7595; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
7596; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
7597; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
7598; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7599; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7600; GFX10-WGP-NEXT:    s_endpgm
7601;
7602; GFX10-CU-LABEL: flat_agent_one_as_monotonic_load:
7603; GFX10-CU:       ; %bb.0: ; %entry
7604; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7605; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7606; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7607; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7608; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
7609; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
7610; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
7611; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7612; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7613; GFX10-CU-NEXT:    s_endpgm
7614;
7615; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_load:
7616; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7617; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7618; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7619; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7620; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
7621; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
7622; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7623; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7624; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7625; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7626; SKIP-CACHE-INV-NEXT:    s_endpgm
7627;
7628; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_load:
7629; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7630; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7631; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7632; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7633; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7634; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
7635; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7636; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7637; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7638; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7639; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7640;
7641; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_load:
7642; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7643; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7644; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7645; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7646; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7647; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
7648; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7649; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7650; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7651; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7652; GFX90A-TGSPLIT-NEXT:    s_endpgm
7653;
7654; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_load:
7655; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7656; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7657; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7658; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7659; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7660; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc1
7661; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7662; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7663; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7664; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7665; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7666;
7667; GFX940-TGSPLIT-LABEL: flat_agent_one_as_monotonic_load:
7668; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7669; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7670; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7671; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7672; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7673; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc1
7674; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7675; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7676; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7677; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7678; GFX940-TGSPLIT-NEXT:    s_endpgm
7679;
7680; GFX11-WGP-LABEL: flat_agent_one_as_monotonic_load:
7681; GFX11-WGP:       ; %bb.0: ; %entry
7682; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7683; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7684; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7685; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc
7686; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7687; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7688; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7689; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7690; GFX11-WGP-NEXT:    s_endpgm
7691;
7692; GFX11-CU-LABEL: flat_agent_one_as_monotonic_load:
7693; GFX11-CU:       ; %bb.0: ; %entry
7694; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7695; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7696; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7697; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1] glc
7698; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7699; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7700; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7701; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7702; GFX11-CU-NEXT:    s_endpgm
7703    i32* %in, i32* %out) {
7704entry:
7705  %val = load atomic i32, i32* %in syncscope("agent-one-as") monotonic, align 4
7706  store i32 %val, i32* %out
7707  ret void
7708}
7709
7710define amdgpu_kernel void @flat_agent_one_as_acquire_load(
7711; GFX7-LABEL: flat_agent_one_as_acquire_load:
7712; GFX7:       ; %bb.0: ; %entry
7713; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7714; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7715; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7716; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7717; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
7718; GFX7-NEXT:    s_waitcnt vmcnt(0)
7719; GFX7-NEXT:    buffer_wbinvl1_vol
7720; GFX7-NEXT:    v_mov_b32_e32 v0, s2
7721; GFX7-NEXT:    v_mov_b32_e32 v1, s3
7722; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7723; GFX7-NEXT:    flat_store_dword v[0:1], v2
7724; GFX7-NEXT:    s_endpgm
7725;
7726; GFX10-WGP-LABEL: flat_agent_one_as_acquire_load:
7727; GFX10-WGP:       ; %bb.0: ; %entry
7728; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7729; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7730; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7731; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7732; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
7733; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7734; GFX10-WGP-NEXT:    buffer_gl0_inv
7735; GFX10-WGP-NEXT:    buffer_gl1_inv
7736; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
7737; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
7738; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7739; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7740; GFX10-WGP-NEXT:    s_endpgm
7741;
7742; GFX10-CU-LABEL: flat_agent_one_as_acquire_load:
7743; GFX10-CU:       ; %bb.0: ; %entry
7744; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7745; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7746; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7747; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7748; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
7749; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7750; GFX10-CU-NEXT:    buffer_gl0_inv
7751; GFX10-CU-NEXT:    buffer_gl1_inv
7752; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
7753; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
7754; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7755; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7756; GFX10-CU-NEXT:    s_endpgm
7757;
7758; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_load:
7759; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7760; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7761; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7762; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7763; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
7764; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
7765; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7766; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7767; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7768; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7769; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7770; SKIP-CACHE-INV-NEXT:    s_endpgm
7771;
7772; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_load:
7773; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7774; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7775; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7776; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7777; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7778; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
7779; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7780; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7781; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7782; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7783; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7784; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7785; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7786;
7787; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_load:
7788; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7789; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7790; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7791; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7792; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7793; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
7794; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7795; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7796; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7797; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7798; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7799; GFX90A-TGSPLIT-NEXT:    s_endpgm
7800;
7801; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_load:
7802; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7803; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7804; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7805; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7806; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7807; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc1
7808; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7809; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
7810; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7811; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7812; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7813; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7814; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7815;
7816; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acquire_load:
7817; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7818; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7819; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7820; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7821; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7822; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc1
7823; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7824; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
7825; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7826; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7827; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7828; GFX940-TGSPLIT-NEXT:    s_endpgm
7829;
7830; GFX11-WGP-LABEL: flat_agent_one_as_acquire_load:
7831; GFX11-WGP:       ; %bb.0: ; %entry
7832; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7833; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7834; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7835; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc
7836; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
7837; GFX11-WGP-NEXT:    buffer_gl0_inv
7838; GFX11-WGP-NEXT:    buffer_gl1_inv
7839; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7840; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7841; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7842; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7843; GFX11-WGP-NEXT:    s_endpgm
7844;
7845; GFX11-CU-LABEL: flat_agent_one_as_acquire_load:
7846; GFX11-CU:       ; %bb.0: ; %entry
7847; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7848; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7849; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7850; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1] glc
7851; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
7852; GFX11-CU-NEXT:    buffer_gl0_inv
7853; GFX11-CU-NEXT:    buffer_gl1_inv
7854; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7855; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7856; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7857; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
7858; GFX11-CU-NEXT:    s_endpgm
7859    i32* %in, i32* %out) {
7860entry:
7861  %val = load atomic i32, i32* %in syncscope("agent-one-as") acquire, align 4
7862  store i32 %val, i32* %out
7863  ret void
7864}
7865
7866define amdgpu_kernel void @flat_agent_one_as_seq_cst_load(
7867; GFX7-LABEL: flat_agent_one_as_seq_cst_load:
7868; GFX7:       ; %bb.0: ; %entry
7869; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7870; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7871; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7872; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7873; GFX7-NEXT:    s_waitcnt vmcnt(0)
7874; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
7875; GFX7-NEXT:    s_waitcnt vmcnt(0)
7876; GFX7-NEXT:    buffer_wbinvl1_vol
7877; GFX7-NEXT:    v_mov_b32_e32 v0, s2
7878; GFX7-NEXT:    v_mov_b32_e32 v1, s3
7879; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7880; GFX7-NEXT:    flat_store_dword v[0:1], v2
7881; GFX7-NEXT:    s_endpgm
7882;
7883; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_load:
7884; GFX10-WGP:       ; %bb.0: ; %entry
7885; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7886; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7887; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7888; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7889; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7890; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7891; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
7892; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7893; GFX10-WGP-NEXT:    buffer_gl0_inv
7894; GFX10-WGP-NEXT:    buffer_gl1_inv
7895; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
7896; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
7897; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7898; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7899; GFX10-WGP-NEXT:    s_endpgm
7900;
7901; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_load:
7902; GFX10-CU:       ; %bb.0: ; %entry
7903; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7904; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7905; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7906; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7907; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7908; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7909; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
7910; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7911; GFX10-CU-NEXT:    buffer_gl0_inv
7912; GFX10-CU-NEXT:    buffer_gl1_inv
7913; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
7914; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
7915; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7916; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7917; GFX10-CU-NEXT:    s_endpgm
7918;
7919; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_load:
7920; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7921; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7922; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7923; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7924; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
7925; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7926; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
7927; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7928; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7929; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7930; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7931; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7932; SKIP-CACHE-INV-NEXT:    s_endpgm
7933;
7934; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_load:
7935; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7936; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7937; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7938; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7939; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7940; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7941; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
7942; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7943; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7944; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7945; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7946; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7947; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7948; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7949;
7950; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_load:
7951; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7952; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7953; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7954; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7955; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7956; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7957; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
7958; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7959; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7960; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7961; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7962; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7963; GFX90A-TGSPLIT-NEXT:    s_endpgm
7964;
7965; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_load:
7966; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7967; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7968; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7969; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7970; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7971; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7972; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc1
7973; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7974; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
7975; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7976; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7977; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7978; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7979; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7980;
7981; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_load:
7982; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7983; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7984; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7985; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7986; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7987; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7988; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc1
7989; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7990; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
7991; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7992; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7993; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7994; GFX940-TGSPLIT-NEXT:    s_endpgm
7995;
7996; GFX11-WGP-LABEL: flat_agent_one_as_seq_cst_load:
7997; GFX11-WGP:       ; %bb.0: ; %entry
7998; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7999; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8000; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8001; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8002; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8003; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc
8004; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8005; GFX11-WGP-NEXT:    buffer_gl0_inv
8006; GFX11-WGP-NEXT:    buffer_gl1_inv
8007; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8008; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8009; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
8010; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8011; GFX11-WGP-NEXT:    s_endpgm
8012;
8013; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_load:
8014; GFX11-CU:       ; %bb.0: ; %entry
8015; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8016; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8017; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8018; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
8019; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8020; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1] glc
8021; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
8022; GFX11-CU-NEXT:    buffer_gl0_inv
8023; GFX11-CU-NEXT:    buffer_gl1_inv
8024; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8025; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8026; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8027; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8028; GFX11-CU-NEXT:    s_endpgm
8029    i32* %in, i32* %out) {
8030entry:
8031  %val = load atomic i32, i32* %in syncscope("agent-one-as") seq_cst, align 4
8032  store i32 %val, i32* %out
8033  ret void
8034}
8035
8036define amdgpu_kernel void @flat_agent_one_as_unordered_store(
8037; GFX7-LABEL: flat_agent_one_as_unordered_store:
8038; GFX7:       ; %bb.0: ; %entry
8039; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
8040; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
8041; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8042; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8043; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8044; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8045; GFX7-NEXT:    flat_store_dword v[0:1], v2
8046; GFX7-NEXT:    s_endpgm
8047;
8048; GFX10-WGP-LABEL: flat_agent_one_as_unordered_store:
8049; GFX10-WGP:       ; %bb.0: ; %entry
8050; GFX10-WGP-NEXT:    s_clause 0x1
8051; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8052; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
8053; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8054; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8055; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8056; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8057; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8058; GFX10-WGP-NEXT:    s_endpgm
8059;
8060; GFX10-CU-LABEL: flat_agent_one_as_unordered_store:
8061; GFX10-CU:       ; %bb.0: ; %entry
8062; GFX10-CU-NEXT:    s_clause 0x1
8063; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8064; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
8065; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8066; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8067; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8068; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8069; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8070; GFX10-CU-NEXT:    s_endpgm
8071;
8072; SKIP-CACHE-INV-LABEL: flat_agent_one_as_unordered_store:
8073; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8074; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
8075; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
8076; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8077; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8078; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8079; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8080; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8081; SKIP-CACHE-INV-NEXT:    s_endpgm
8082;
8083; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_unordered_store:
8084; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8085; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8086; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
8087; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8088; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8089; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8090; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8091; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8092;
8093; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_unordered_store:
8094; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8095; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8096; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
8097; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8098; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8099; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8100; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8101; GFX90A-TGSPLIT-NEXT:    s_endpgm
8102;
8103; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_unordered_store:
8104; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8105; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8106; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
8107; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8108; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8109; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8110; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8111; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8112;
8113; GFX940-TGSPLIT-LABEL: flat_agent_one_as_unordered_store:
8114; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8115; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8116; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
8117; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8118; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8119; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8120; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8121; GFX940-TGSPLIT-NEXT:    s_endpgm
8122;
8123; GFX11-WGP-LABEL: flat_agent_one_as_unordered_store:
8124; GFX11-WGP:       ; %bb.0: ; %entry
8125; GFX11-WGP-NEXT:    s_clause 0x1
8126; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8127; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
8128; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8129; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8130; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8131; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
8132; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8133; GFX11-WGP-NEXT:    s_endpgm
8134;
8135; GFX11-CU-LABEL: flat_agent_one_as_unordered_store:
8136; GFX11-CU:       ; %bb.0: ; %entry
8137; GFX11-CU-NEXT:    s_clause 0x1
8138; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8139; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
8140; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8141; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8142; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8143; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8144; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8145; GFX11-CU-NEXT:    s_endpgm
8146    i32 %in, i32* %out) {
8147entry:
8148  store atomic i32 %in, i32* %out syncscope("agent-one-as") unordered, align 4
8149  ret void
8150}
8151
8152define amdgpu_kernel void @flat_agent_one_as_monotonic_store(
8153; GFX7-LABEL: flat_agent_one_as_monotonic_store:
8154; GFX7:       ; %bb.0: ; %entry
8155; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
8156; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
8157; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8158; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8159; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8160; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8161; GFX7-NEXT:    flat_store_dword v[0:1], v2
8162; GFX7-NEXT:    s_endpgm
8163;
8164; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_store:
8165; GFX10-WGP:       ; %bb.0: ; %entry
8166; GFX10-WGP-NEXT:    s_clause 0x1
8167; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8168; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
8169; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8170; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8171; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8172; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8173; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8174; GFX10-WGP-NEXT:    s_endpgm
8175;
8176; GFX10-CU-LABEL: flat_agent_one_as_monotonic_store:
8177; GFX10-CU:       ; %bb.0: ; %entry
8178; GFX10-CU-NEXT:    s_clause 0x1
8179; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8180; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
8181; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8182; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8183; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8184; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8185; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8186; GFX10-CU-NEXT:    s_endpgm
8187;
8188; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_store:
8189; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8190; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
8191; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
8192; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8193; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8194; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8195; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8196; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8197; SKIP-CACHE-INV-NEXT:    s_endpgm
8198;
8199; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_store:
8200; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8201; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8202; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
8203; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8204; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8205; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8206; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8207; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8208;
8209; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_store:
8210; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8211; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8212; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
8213; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8214; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8215; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8216; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8217; GFX90A-TGSPLIT-NEXT:    s_endpgm
8218;
8219; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_store:
8220; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8221; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8222; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
8223; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8224; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8225; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8226; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc1
8227; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8228;
8229; GFX940-TGSPLIT-LABEL: flat_agent_one_as_monotonic_store:
8230; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8231; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8232; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
8233; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8234; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8235; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8236; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc1
8237; GFX940-TGSPLIT-NEXT:    s_endpgm
8238;
8239; GFX11-WGP-LABEL: flat_agent_one_as_monotonic_store:
8240; GFX11-WGP:       ; %bb.0: ; %entry
8241; GFX11-WGP-NEXT:    s_clause 0x1
8242; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8243; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
8244; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8245; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8246; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8247; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
8248; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8249; GFX11-WGP-NEXT:    s_endpgm
8250;
8251; GFX11-CU-LABEL: flat_agent_one_as_monotonic_store:
8252; GFX11-CU:       ; %bb.0: ; %entry
8253; GFX11-CU-NEXT:    s_clause 0x1
8254; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8255; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
8256; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8257; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8258; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8259; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8260; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8261; GFX11-CU-NEXT:    s_endpgm
8262    i32 %in, i32* %out) {
8263entry:
8264  store atomic i32 %in, i32* %out syncscope("agent-one-as") monotonic, align 4
8265  ret void
8266}
8267
8268define amdgpu_kernel void @flat_agent_one_as_release_store(
8269; GFX7-LABEL: flat_agent_one_as_release_store:
8270; GFX7:       ; %bb.0: ; %entry
8271; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
8272; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
8273; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8274; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8275; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8276; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8277; GFX7-NEXT:    s_waitcnt vmcnt(0)
8278; GFX7-NEXT:    flat_store_dword v[0:1], v2
8279; GFX7-NEXT:    s_endpgm
8280;
8281; GFX10-WGP-LABEL: flat_agent_one_as_release_store:
8282; GFX10-WGP:       ; %bb.0: ; %entry
8283; GFX10-WGP-NEXT:    s_clause 0x1
8284; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8285; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
8286; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8287; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8288; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8289; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8290; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8291; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8292; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8293; GFX10-WGP-NEXT:    s_endpgm
8294;
8295; GFX10-CU-LABEL: flat_agent_one_as_release_store:
8296; GFX10-CU:       ; %bb.0: ; %entry
8297; GFX10-CU-NEXT:    s_clause 0x1
8298; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8299; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
8300; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8301; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8302; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8303; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8304; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8305; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8306; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8307; GFX10-CU-NEXT:    s_endpgm
8308;
8309; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_store:
8310; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8311; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
8312; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
8313; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8314; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8315; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8316; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8317; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8318; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8319; SKIP-CACHE-INV-NEXT:    s_endpgm
8320;
8321; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_store:
8322; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8323; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8324; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
8325; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8326; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8327; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8328; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8329; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8330; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8331;
8332; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_store:
8333; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8334; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8335; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
8336; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8337; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8338; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8339; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8340; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8341; GFX90A-TGSPLIT-NEXT:    s_endpgm
8342;
8343; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_release_store:
8344; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8345; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8346; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
8347; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8348; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8349; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8350; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
8351; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8352; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc1
8353; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8354;
8355; GFX940-TGSPLIT-LABEL: flat_agent_one_as_release_store:
8356; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8357; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8358; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
8359; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8360; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8361; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8362; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
8363; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8364; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc1
8365; GFX940-TGSPLIT-NEXT:    s_endpgm
8366;
8367; GFX11-WGP-LABEL: flat_agent_one_as_release_store:
8368; GFX11-WGP:       ; %bb.0: ; %entry
8369; GFX11-WGP-NEXT:    s_clause 0x1
8370; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8371; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
8372; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8373; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8374; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8375; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8376; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8377; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
8378; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8379; GFX11-WGP-NEXT:    s_endpgm
8380;
8381; GFX11-CU-LABEL: flat_agent_one_as_release_store:
8382; GFX11-CU:       ; %bb.0: ; %entry
8383; GFX11-CU-NEXT:    s_clause 0x1
8384; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8385; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
8386; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8387; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8388; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8389; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
8390; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8391; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8392; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8393; GFX11-CU-NEXT:    s_endpgm
8394    i32 %in, i32* %out) {
8395entry:
8396  store atomic i32 %in, i32* %out syncscope("agent-one-as") release, align 4
8397  ret void
8398}
8399
8400define amdgpu_kernel void @flat_agent_one_as_seq_cst_store(
8401; GFX7-LABEL: flat_agent_one_as_seq_cst_store:
8402; GFX7:       ; %bb.0: ; %entry
8403; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
8404; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
8405; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8406; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8407; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8408; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8409; GFX7-NEXT:    s_waitcnt vmcnt(0)
8410; GFX7-NEXT:    flat_store_dword v[0:1], v2
8411; GFX7-NEXT:    s_endpgm
8412;
8413; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_store:
8414; GFX10-WGP:       ; %bb.0: ; %entry
8415; GFX10-WGP-NEXT:    s_clause 0x1
8416; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8417; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
8418; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8419; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8420; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8421; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8422; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8423; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8424; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8425; GFX10-WGP-NEXT:    s_endpgm
8426;
8427; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_store:
8428; GFX10-CU:       ; %bb.0: ; %entry
8429; GFX10-CU-NEXT:    s_clause 0x1
8430; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8431; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
8432; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8433; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8434; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8435; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8436; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8437; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8438; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8439; GFX10-CU-NEXT:    s_endpgm
8440;
8441; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_store:
8442; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8443; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
8444; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
8445; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8446; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8447; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8448; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8449; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8450; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8451; SKIP-CACHE-INV-NEXT:    s_endpgm
8452;
8453; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_store:
8454; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8455; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8456; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
8457; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8458; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8459; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8460; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8461; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8462; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8463;
8464; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_store:
8465; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8466; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8467; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
8468; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8469; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8470; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8471; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8472; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8473; GFX90A-TGSPLIT-NEXT:    s_endpgm
8474;
8475; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_store:
8476; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8477; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8478; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
8479; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8480; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8481; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8482; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
8483; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8484; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc1
8485; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8486;
8487; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_store:
8488; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8489; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8490; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
8491; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8492; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8493; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8494; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
8495; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8496; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc1
8497; GFX940-TGSPLIT-NEXT:    s_endpgm
8498;
8499; GFX11-WGP-LABEL: flat_agent_one_as_seq_cst_store:
8500; GFX11-WGP:       ; %bb.0: ; %entry
8501; GFX11-WGP-NEXT:    s_clause 0x1
8502; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8503; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
8504; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8505; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8506; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8507; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8508; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8509; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
8510; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8511; GFX11-WGP-NEXT:    s_endpgm
8512;
8513; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_store:
8514; GFX11-CU:       ; %bb.0: ; %entry
8515; GFX11-CU-NEXT:    s_clause 0x1
8516; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8517; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
8518; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8519; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8520; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8521; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
8522; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8523; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8524; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8525; GFX11-CU-NEXT:    s_endpgm
8526    i32 %in, i32* %out) {
8527entry:
8528  store atomic i32 %in, i32* %out syncscope("agent-one-as") seq_cst, align 4
8529  ret void
8530}
8531
8532define amdgpu_kernel void @flat_agent_one_as_monotonic_atomicrmw(
8533; GFX7-LABEL: flat_agent_one_as_monotonic_atomicrmw:
8534; GFX7:       ; %bb.0: ; %entry
8535; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8536; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
8537; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8538; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8539; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8540; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8541; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
8542; GFX7-NEXT:    s_endpgm
8543;
8544; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_atomicrmw:
8545; GFX10-WGP:       ; %bb.0: ; %entry
8546; GFX10-WGP-NEXT:    s_clause 0x1
8547; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8548; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
8549; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8550; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8551; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8552; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8553; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
8554; GFX10-WGP-NEXT:    s_endpgm
8555;
8556; GFX10-CU-LABEL: flat_agent_one_as_monotonic_atomicrmw:
8557; GFX10-CU:       ; %bb.0: ; %entry
8558; GFX10-CU-NEXT:    s_clause 0x1
8559; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8560; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8561; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8562; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8563; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8564; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8565; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
8566; GFX10-CU-NEXT:    s_endpgm
8567;
8568; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_atomicrmw:
8569; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8570; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8571; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
8572; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8573; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8574; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8575; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8576; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
8577; SKIP-CACHE-INV-NEXT:    s_endpgm
8578;
8579; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_atomicrmw:
8580; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8581; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8582; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8583; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8584; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8585; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8586; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8587; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8588;
8589; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_atomicrmw:
8590; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8591; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8592; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8593; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8594; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8595; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8596; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8597; GFX90A-TGSPLIT-NEXT:    s_endpgm
8598;
8599; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_atomicrmw:
8600; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8601; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8602; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8603; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8604; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8605; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8606; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8607; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8608;
8609; GFX940-TGSPLIT-LABEL: flat_agent_one_as_monotonic_atomicrmw:
8610; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8611; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8612; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8613; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8614; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8615; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8616; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8617; GFX940-TGSPLIT-NEXT:    s_endpgm
8618;
8619; GFX11-WGP-LABEL: flat_agent_one_as_monotonic_atomicrmw:
8620; GFX11-WGP:       ; %bb.0: ; %entry
8621; GFX11-WGP-NEXT:    s_clause 0x1
8622; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8623; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
8624; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8625; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8626; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8627; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
8628; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8629; GFX11-WGP-NEXT:    s_endpgm
8630;
8631; GFX11-CU-LABEL: flat_agent_one_as_monotonic_atomicrmw:
8632; GFX11-CU:       ; %bb.0: ; %entry
8633; GFX11-CU-NEXT:    s_clause 0x1
8634; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8635; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
8636; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8637; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8638; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8639; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
8640; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8641; GFX11-CU-NEXT:    s_endpgm
8642    i32* %out, i32 %in) {
8643entry:
8644  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") monotonic
8645  ret void
8646}
8647
8648define amdgpu_kernel void @flat_agent_one_as_acquire_atomicrmw(
8649; GFX7-LABEL: flat_agent_one_as_acquire_atomicrmw:
8650; GFX7:       ; %bb.0: ; %entry
8651; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8652; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
8653; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8654; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8655; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8656; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8657; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
8658; GFX7-NEXT:    s_waitcnt vmcnt(0)
8659; GFX7-NEXT:    buffer_wbinvl1_vol
8660; GFX7-NEXT:    s_endpgm
8661;
8662; GFX10-WGP-LABEL: flat_agent_one_as_acquire_atomicrmw:
8663; GFX10-WGP:       ; %bb.0: ; %entry
8664; GFX10-WGP-NEXT:    s_clause 0x1
8665; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8666; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
8667; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8668; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8669; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8670; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8671; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
8672; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8673; GFX10-WGP-NEXT:    buffer_gl0_inv
8674; GFX10-WGP-NEXT:    buffer_gl1_inv
8675; GFX10-WGP-NEXT:    s_endpgm
8676;
8677; GFX10-CU-LABEL: flat_agent_one_as_acquire_atomicrmw:
8678; GFX10-CU:       ; %bb.0: ; %entry
8679; GFX10-CU-NEXT:    s_clause 0x1
8680; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8681; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8682; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8683; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8684; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8685; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8686; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
8687; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8688; GFX10-CU-NEXT:    buffer_gl0_inv
8689; GFX10-CU-NEXT:    buffer_gl1_inv
8690; GFX10-CU-NEXT:    s_endpgm
8691;
8692; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_atomicrmw:
8693; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8694; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8695; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
8696; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8697; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8698; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8699; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8700; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
8701; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8702; SKIP-CACHE-INV-NEXT:    s_endpgm
8703;
8704; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_atomicrmw:
8705; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8706; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8707; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8708; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8709; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8710; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8711; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8712; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8713; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
8714; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8715;
8716; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_atomicrmw:
8717; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8718; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8719; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8720; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8721; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8722; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8723; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8724; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8725; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8726; GFX90A-TGSPLIT-NEXT:    s_endpgm
8727;
8728; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_atomicrmw:
8729; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8730; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8731; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8732; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8733; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8734; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8735; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8736; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8737; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
8738; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8739;
8740; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acquire_atomicrmw:
8741; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8742; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8743; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8744; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8745; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8746; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8747; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8748; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8749; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
8750; GFX940-TGSPLIT-NEXT:    s_endpgm
8751;
8752; GFX11-WGP-LABEL: flat_agent_one_as_acquire_atomicrmw:
8753; GFX11-WGP:       ; %bb.0: ; %entry
8754; GFX11-WGP-NEXT:    s_clause 0x1
8755; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8756; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
8757; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8758; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8759; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8760; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
8761; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8762; GFX11-WGP-NEXT:    buffer_gl0_inv
8763; GFX11-WGP-NEXT:    buffer_gl1_inv
8764; GFX11-WGP-NEXT:    s_endpgm
8765;
8766; GFX11-CU-LABEL: flat_agent_one_as_acquire_atomicrmw:
8767; GFX11-CU:       ; %bb.0: ; %entry
8768; GFX11-CU-NEXT:    s_clause 0x1
8769; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8770; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
8771; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8772; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8773; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8774; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
8775; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8776; GFX11-CU-NEXT:    buffer_gl0_inv
8777; GFX11-CU-NEXT:    buffer_gl1_inv
8778; GFX11-CU-NEXT:    s_endpgm
8779    i32* %out, i32 %in) {
8780entry:
8781  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") acquire
8782  ret void
8783}
8784
8785define amdgpu_kernel void @flat_agent_one_as_release_atomicrmw(
8786; GFX7-LABEL: flat_agent_one_as_release_atomicrmw:
8787; GFX7:       ; %bb.0: ; %entry
8788; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8789; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
8790; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8791; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8792; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8793; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8794; GFX7-NEXT:    s_waitcnt vmcnt(0)
8795; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
8796; GFX7-NEXT:    s_endpgm
8797;
8798; GFX10-WGP-LABEL: flat_agent_one_as_release_atomicrmw:
8799; GFX10-WGP:       ; %bb.0: ; %entry
8800; GFX10-WGP-NEXT:    s_clause 0x1
8801; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8802; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
8803; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8804; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8805; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8806; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8807; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8808; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8809; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
8810; GFX10-WGP-NEXT:    s_endpgm
8811;
8812; GFX10-CU-LABEL: flat_agent_one_as_release_atomicrmw:
8813; GFX10-CU:       ; %bb.0: ; %entry
8814; GFX10-CU-NEXT:    s_clause 0x1
8815; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8816; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8817; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8818; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8819; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8820; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8821; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8822; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8823; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
8824; GFX10-CU-NEXT:    s_endpgm
8825;
8826; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_atomicrmw:
8827; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8828; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8829; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
8830; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8831; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8832; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8833; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8834; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8835; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
8836; SKIP-CACHE-INV-NEXT:    s_endpgm
8837;
8838; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_atomicrmw:
8839; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8840; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8841; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8842; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8843; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8844; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8845; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8846; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8847; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8848;
8849; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_atomicrmw:
8850; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8851; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8852; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8853; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8854; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8855; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8856; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8857; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8858; GFX90A-TGSPLIT-NEXT:    s_endpgm
8859;
8860; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_release_atomicrmw:
8861; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8862; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8863; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8864; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8865; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8866; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8867; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
8868; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8869; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8870; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8871;
8872; GFX940-TGSPLIT-LABEL: flat_agent_one_as_release_atomicrmw:
8873; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8874; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8875; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8876; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8877; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8878; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8879; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
8880; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8881; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8882; GFX940-TGSPLIT-NEXT:    s_endpgm
8883;
8884; GFX11-WGP-LABEL: flat_agent_one_as_release_atomicrmw:
8885; GFX11-WGP:       ; %bb.0: ; %entry
8886; GFX11-WGP-NEXT:    s_clause 0x1
8887; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8888; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
8889; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8890; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8891; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8892; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8893; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8894; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
8895; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8896; GFX11-WGP-NEXT:    s_endpgm
8897;
8898; GFX11-CU-LABEL: flat_agent_one_as_release_atomicrmw:
8899; GFX11-CU:       ; %bb.0: ; %entry
8900; GFX11-CU-NEXT:    s_clause 0x1
8901; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8902; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
8903; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8904; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
8905; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8906; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
8907; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8908; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
8909; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
8910; GFX11-CU-NEXT:    s_endpgm
8911    i32* %out, i32 %in) {
8912entry:
8913  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") release
8914  ret void
8915}
8916
8917define amdgpu_kernel void @flat_agent_one_as_acq_rel_atomicrmw(
8918; GFX7-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
8919; GFX7:       ; %bb.0: ; %entry
8920; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8921; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
8922; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8923; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8924; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8925; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8926; GFX7-NEXT:    s_waitcnt vmcnt(0)
8927; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
8928; GFX7-NEXT:    s_waitcnt vmcnt(0)
8929; GFX7-NEXT:    buffer_wbinvl1_vol
8930; GFX7-NEXT:    s_endpgm
8931;
8932; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
8933; GFX10-WGP:       ; %bb.0: ; %entry
8934; GFX10-WGP-NEXT:    s_clause 0x1
8935; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8936; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
8937; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8938; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8939; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8940; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8941; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8942; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8943; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
8944; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8945; GFX10-WGP-NEXT:    buffer_gl0_inv
8946; GFX10-WGP-NEXT:    buffer_gl1_inv
8947; GFX10-WGP-NEXT:    s_endpgm
8948;
8949; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
8950; GFX10-CU:       ; %bb.0: ; %entry
8951; GFX10-CU-NEXT:    s_clause 0x1
8952; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8953; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8954; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8955; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8956; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8957; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8958; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8959; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8960; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
8961; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8962; GFX10-CU-NEXT:    buffer_gl0_inv
8963; GFX10-CU-NEXT:    buffer_gl1_inv
8964; GFX10-CU-NEXT:    s_endpgm
8965;
8966; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
8967; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8968; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8969; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
8970; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8971; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8972; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8973; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8974; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8975; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
8976; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8977; SKIP-CACHE-INV-NEXT:    s_endpgm
8978;
8979; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
8980; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8981; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8982; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8983; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8984; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8985; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8986; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8987; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8988; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8989; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
8990; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8991;
8992; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
8993; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8994; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8995; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8996; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8997; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8998; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8999; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9000; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
9001; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9002; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9003; GFX90A-TGSPLIT-NEXT:    s_endpgm
9004;
9005; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
9006; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9007; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9008; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9009; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9010; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9011; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9012; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
9013; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9014; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
9015; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9016; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
9017; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9018;
9019; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
9020; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9021; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9022; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9023; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9024; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9025; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9026; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
9027; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9028; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
9029; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9030; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
9031; GFX940-TGSPLIT-NEXT:    s_endpgm
9032;
9033; GFX11-WGP-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
9034; GFX11-WGP:       ; %bb.0: ; %entry
9035; GFX11-WGP-NEXT:    s_clause 0x1
9036; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9037; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
9038; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9039; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9040; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9041; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9042; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9043; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
9044; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9045; GFX11-WGP-NEXT:    buffer_gl0_inv
9046; GFX11-WGP-NEXT:    buffer_gl1_inv
9047; GFX11-WGP-NEXT:    s_endpgm
9048;
9049; GFX11-CU-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
9050; GFX11-CU:       ; %bb.0: ; %entry
9051; GFX11-CU-NEXT:    s_clause 0x1
9052; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9053; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
9054; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9055; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9056; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9057; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9058; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9059; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
9060; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9061; GFX11-CU-NEXT:    buffer_gl0_inv
9062; GFX11-CU-NEXT:    buffer_gl1_inv
9063; GFX11-CU-NEXT:    s_endpgm
9064    i32* %out, i32 %in) {
9065entry:
9066  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") acq_rel
9067  ret void
9068}
9069
9070define amdgpu_kernel void @flat_agent_one_as_seq_cst_atomicrmw(
9071; GFX7-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
9072; GFX7:       ; %bb.0: ; %entry
9073; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9074; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
9075; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9076; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9077; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9078; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9079; GFX7-NEXT:    s_waitcnt vmcnt(0)
9080; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
9081; GFX7-NEXT:    s_waitcnt vmcnt(0)
9082; GFX7-NEXT:    buffer_wbinvl1_vol
9083; GFX7-NEXT:    s_endpgm
9084;
9085; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
9086; GFX10-WGP:       ; %bb.0: ; %entry
9087; GFX10-WGP-NEXT:    s_clause 0x1
9088; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9089; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
9090; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9091; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9092; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9093; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9094; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9095; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9096; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
9097; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9098; GFX10-WGP-NEXT:    buffer_gl0_inv
9099; GFX10-WGP-NEXT:    buffer_gl1_inv
9100; GFX10-WGP-NEXT:    s_endpgm
9101;
9102; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
9103; GFX10-CU:       ; %bb.0: ; %entry
9104; GFX10-CU-NEXT:    s_clause 0x1
9105; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9106; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
9107; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9108; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9109; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9110; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9111; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9112; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9113; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
9114; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9115; GFX10-CU-NEXT:    buffer_gl0_inv
9116; GFX10-CU-NEXT:    buffer_gl1_inv
9117; GFX10-CU-NEXT:    s_endpgm
9118;
9119; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
9120; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9121; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9122; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
9123; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9124; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9125; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9126; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9127; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9128; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
9129; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9130; SKIP-CACHE-INV-NEXT:    s_endpgm
9131;
9132; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
9133; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9134; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9135; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9136; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9137; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9138; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9139; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9140; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
9141; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9142; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9143; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9144;
9145; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
9146; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9147; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9148; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9149; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9150; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9151; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9152; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9153; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
9154; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9155; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9156; GFX90A-TGSPLIT-NEXT:    s_endpgm
9157;
9158; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
9159; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9160; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9161; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9162; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9163; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9164; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9165; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
9166; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9167; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
9168; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9169; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
9170; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9171;
9172; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
9173; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9174; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9175; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9176; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9177; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9178; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9179; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
9180; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9181; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
9182; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9183; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
9184; GFX940-TGSPLIT-NEXT:    s_endpgm
9185;
9186; GFX11-WGP-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
9187; GFX11-WGP:       ; %bb.0: ; %entry
9188; GFX11-WGP-NEXT:    s_clause 0x1
9189; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9190; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
9191; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9192; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9193; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9194; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9195; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9196; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
9197; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9198; GFX11-WGP-NEXT:    buffer_gl0_inv
9199; GFX11-WGP-NEXT:    buffer_gl1_inv
9200; GFX11-WGP-NEXT:    s_endpgm
9201;
9202; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
9203; GFX11-CU:       ; %bb.0: ; %entry
9204; GFX11-CU-NEXT:    s_clause 0x1
9205; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9206; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
9207; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9208; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9209; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9210; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9211; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9212; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
9213; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9214; GFX11-CU-NEXT:    buffer_gl0_inv
9215; GFX11-CU-NEXT:    buffer_gl1_inv
9216; GFX11-CU-NEXT:    s_endpgm
9217    i32* %out, i32 %in) {
9218entry:
9219  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") seq_cst
9220  ret void
9221}
9222
9223define amdgpu_kernel void @flat_agent_one_as_acquire_ret_atomicrmw(
9224; GFX7-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
9225; GFX7:       ; %bb.0: ; %entry
9226; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9227; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
9228; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9229; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9230; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9231; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9232; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9233; GFX7-NEXT:    s_waitcnt vmcnt(0)
9234; GFX7-NEXT:    buffer_wbinvl1_vol
9235; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9236; GFX7-NEXT:    flat_store_dword v[0:1], v2
9237; GFX7-NEXT:    s_endpgm
9238;
9239; GFX10-WGP-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
9240; GFX10-WGP:       ; %bb.0: ; %entry
9241; GFX10-WGP-NEXT:    s_clause 0x1
9242; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9243; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
9244; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9245; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9246; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9247; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9248; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9249; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9250; GFX10-WGP-NEXT:    buffer_gl0_inv
9251; GFX10-WGP-NEXT:    buffer_gl1_inv
9252; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9253; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
9254; GFX10-WGP-NEXT:    s_endpgm
9255;
9256; GFX10-CU-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
9257; GFX10-CU:       ; %bb.0: ; %entry
9258; GFX10-CU-NEXT:    s_clause 0x1
9259; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9260; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
9261; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9262; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9263; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9264; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9265; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9266; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9267; GFX10-CU-NEXT:    buffer_gl0_inv
9268; GFX10-CU-NEXT:    buffer_gl1_inv
9269; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9270; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
9271; GFX10-CU-NEXT:    s_endpgm
9272;
9273; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
9274; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9275; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9276; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
9277; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9278; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9279; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9280; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9281; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9282; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9283; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
9284; SKIP-CACHE-INV-NEXT:    s_endpgm
9285;
9286; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
9287; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9288; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9289; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9290; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9291; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9292; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9293; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9294; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9295; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9296; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9297; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9298; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9299;
9300; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
9301; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9302; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9303; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9304; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9305; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9306; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9307; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9308; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9309; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9310; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9311; GFX90A-TGSPLIT-NEXT:    s_endpgm
9312;
9313; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
9314; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9315; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9316; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9317; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9318; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9319; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9320; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
9321; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9322; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
9323; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9324; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9325; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9326;
9327; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
9328; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9329; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9330; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9331; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9332; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9333; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9334; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
9335; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9336; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
9337; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9338; GFX940-TGSPLIT-NEXT:    s_endpgm
9339;
9340; GFX11-WGP-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
9341; GFX11-WGP:       ; %bb.0: ; %entry
9342; GFX11-WGP-NEXT:    s_clause 0x1
9343; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9344; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
9345; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9346; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9347; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9348; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
9349; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9350; GFX11-WGP-NEXT:    buffer_gl0_inv
9351; GFX11-WGP-NEXT:    buffer_gl1_inv
9352; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9353; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
9354; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9355; GFX11-WGP-NEXT:    s_endpgm
9356;
9357; GFX11-CU-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
9358; GFX11-CU:       ; %bb.0: ; %entry
9359; GFX11-CU-NEXT:    s_clause 0x1
9360; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9361; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
9362; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9363; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9364; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9365; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
9366; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9367; GFX11-CU-NEXT:    buffer_gl0_inv
9368; GFX11-CU-NEXT:    buffer_gl1_inv
9369; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9370; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
9371; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9372; GFX11-CU-NEXT:    s_endpgm
9373    i32* %out, i32 %in) {
9374entry:
9375  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") acquire
9376  store i32 %val, i32* %out, align 4
9377  ret void
9378}
9379
9380define amdgpu_kernel void @flat_agent_one_as_acq_rel_ret_atomicrmw(
9381; GFX7-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
9382; GFX7:       ; %bb.0: ; %entry
9383; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9384; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
9385; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9386; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9387; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9388; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9389; GFX7-NEXT:    s_waitcnt vmcnt(0)
9390; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9391; GFX7-NEXT:    s_waitcnt vmcnt(0)
9392; GFX7-NEXT:    buffer_wbinvl1_vol
9393; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9394; GFX7-NEXT:    flat_store_dword v[0:1], v2
9395; GFX7-NEXT:    s_endpgm
9396;
9397; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
9398; GFX10-WGP:       ; %bb.0: ; %entry
9399; GFX10-WGP-NEXT:    s_clause 0x1
9400; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9401; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
9402; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9403; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9404; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9405; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9406; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9407; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9408; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9409; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9410; GFX10-WGP-NEXT:    buffer_gl0_inv
9411; GFX10-WGP-NEXT:    buffer_gl1_inv
9412; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9413; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
9414; GFX10-WGP-NEXT:    s_endpgm
9415;
9416; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
9417; GFX10-CU:       ; %bb.0: ; %entry
9418; GFX10-CU-NEXT:    s_clause 0x1
9419; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9420; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
9421; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9422; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9423; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9424; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9425; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9426; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9427; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9428; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9429; GFX10-CU-NEXT:    buffer_gl0_inv
9430; GFX10-CU-NEXT:    buffer_gl1_inv
9431; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9432; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
9433; GFX10-CU-NEXT:    s_endpgm
9434;
9435; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
9436; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9437; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9438; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
9439; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9440; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9441; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9442; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9443; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9444; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9445; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9446; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
9447; SKIP-CACHE-INV-NEXT:    s_endpgm
9448;
9449; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
9450; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9451; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9452; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9453; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9454; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9455; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9456; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9457; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9458; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9459; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9460; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9461; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9462; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9463;
9464; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
9465; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9466; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9467; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9468; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9469; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9470; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9471; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9472; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9473; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9474; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9475; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9476; GFX90A-TGSPLIT-NEXT:    s_endpgm
9477;
9478; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
9479; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9480; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9481; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9482; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9483; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9484; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9485; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
9486; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9487; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
9488; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9489; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
9490; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9491; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9492; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9493;
9494; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
9495; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9496; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9497; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9498; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9499; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9500; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9501; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
9502; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9503; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
9504; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9505; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
9506; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9507; GFX940-TGSPLIT-NEXT:    s_endpgm
9508;
9509; GFX11-WGP-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
9510; GFX11-WGP:       ; %bb.0: ; %entry
9511; GFX11-WGP-NEXT:    s_clause 0x1
9512; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9513; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
9514; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9515; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9516; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9517; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9518; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9519; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
9520; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9521; GFX11-WGP-NEXT:    buffer_gl0_inv
9522; GFX11-WGP-NEXT:    buffer_gl1_inv
9523; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9524; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
9525; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9526; GFX11-WGP-NEXT:    s_endpgm
9527;
9528; GFX11-CU-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
9529; GFX11-CU:       ; %bb.0: ; %entry
9530; GFX11-CU-NEXT:    s_clause 0x1
9531; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9532; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
9533; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9534; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9535; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9536; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9537; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9538; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
9539; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9540; GFX11-CU-NEXT:    buffer_gl0_inv
9541; GFX11-CU-NEXT:    buffer_gl1_inv
9542; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9543; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
9544; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9545; GFX11-CU-NEXT:    s_endpgm
9546    i32* %out, i32 %in) {
9547entry:
9548  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") acq_rel
9549  store i32 %val, i32* %out, align 4
9550  ret void
9551}
9552
9553define amdgpu_kernel void @flat_agent_one_as_seq_cst_ret_atomicrmw(
9554; GFX7-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
9555; GFX7:       ; %bb.0: ; %entry
9556; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9557; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
9558; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9559; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9560; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9561; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9562; GFX7-NEXT:    s_waitcnt vmcnt(0)
9563; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9564; GFX7-NEXT:    s_waitcnt vmcnt(0)
9565; GFX7-NEXT:    buffer_wbinvl1_vol
9566; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9567; GFX7-NEXT:    flat_store_dword v[0:1], v2
9568; GFX7-NEXT:    s_endpgm
9569;
9570; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
9571; GFX10-WGP:       ; %bb.0: ; %entry
9572; GFX10-WGP-NEXT:    s_clause 0x1
9573; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9574; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
9575; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9576; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9577; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9578; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9579; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9580; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9581; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9582; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9583; GFX10-WGP-NEXT:    buffer_gl0_inv
9584; GFX10-WGP-NEXT:    buffer_gl1_inv
9585; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9586; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
9587; GFX10-WGP-NEXT:    s_endpgm
9588;
9589; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
9590; GFX10-CU:       ; %bb.0: ; %entry
9591; GFX10-CU-NEXT:    s_clause 0x1
9592; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9593; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
9594; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9595; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9596; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9597; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9598; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9599; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9600; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9601; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9602; GFX10-CU-NEXT:    buffer_gl0_inv
9603; GFX10-CU-NEXT:    buffer_gl1_inv
9604; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9605; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
9606; GFX10-CU-NEXT:    s_endpgm
9607;
9608; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
9609; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9610; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9611; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
9612; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9613; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9614; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9615; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9616; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9617; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9618; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9619; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
9620; SKIP-CACHE-INV-NEXT:    s_endpgm
9621;
9622; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
9623; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9624; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9625; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9626; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9627; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9628; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9629; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9630; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9631; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9632; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9633; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9634; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9635; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9636;
9637; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
9638; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9639; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9640; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9641; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9642; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9643; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9644; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9645; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9646; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9647; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9648; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9649; GFX90A-TGSPLIT-NEXT:    s_endpgm
9650;
9651; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
9652; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9653; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9654; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9655; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9656; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9657; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9658; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
9659; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9660; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
9661; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9662; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
9663; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9664; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9665; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9666;
9667; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
9668; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9669; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9670; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9671; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9672; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9673; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9674; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
9675; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9676; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
9677; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9678; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
9679; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9680; GFX940-TGSPLIT-NEXT:    s_endpgm
9681;
9682; GFX11-WGP-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
9683; GFX11-WGP:       ; %bb.0: ; %entry
9684; GFX11-WGP-NEXT:    s_clause 0x1
9685; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9686; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
9687; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9688; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9689; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9690; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9691; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9692; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
9693; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9694; GFX11-WGP-NEXT:    buffer_gl0_inv
9695; GFX11-WGP-NEXT:    buffer_gl1_inv
9696; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9697; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
9698; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9699; GFX11-WGP-NEXT:    s_endpgm
9700;
9701; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
9702; GFX11-CU:       ; %bb.0: ; %entry
9703; GFX11-CU-NEXT:    s_clause 0x1
9704; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9705; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
9706; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9707; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9708; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9709; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9710; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9711; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
9712; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9713; GFX11-CU-NEXT:    buffer_gl0_inv
9714; GFX11-CU-NEXT:    buffer_gl1_inv
9715; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9716; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
9717; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9718; GFX11-CU-NEXT:    s_endpgm
9719    i32* %out, i32 %in) {
9720entry:
9721  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") seq_cst
9722  store i32 %val, i32* %out, align 4
9723  ret void
9724}
9725
9726define amdgpu_kernel void @flat_agent_one_as_monotonic_monotonic_cmpxchg(
9727; GFX7-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
9728; GFX7:       ; %bb.0: ; %entry
9729; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9730; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9731; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9732; GFX7-NEXT:    s_add_u32 s0, s0, 16
9733; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9734; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9735; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9736; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9737; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9738; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9739; GFX7-NEXT:    s_endpgm
9740;
9741; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
9742; GFX10-WGP:       ; %bb.0: ; %entry
9743; GFX10-WGP-NEXT:    s_clause 0x1
9744; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9745; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9746; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9747; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9748; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9749; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9750; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9751; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9752; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9753; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9754; GFX10-WGP-NEXT:    s_endpgm
9755;
9756; GFX10-CU-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
9757; GFX10-CU:       ; %bb.0: ; %entry
9758; GFX10-CU-NEXT:    s_clause 0x1
9759; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9760; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9761; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9762; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9763; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9764; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9765; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9766; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9767; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9768; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9769; GFX10-CU-NEXT:    s_endpgm
9770;
9771; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
9772; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9773; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9774; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9775; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9776; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9777; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9778; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9779; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9780; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9781; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9782; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9783; SKIP-CACHE-INV-NEXT:    s_endpgm
9784;
9785; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
9786; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9787; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9788; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9789; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9790; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9791; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9792; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9793; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9794;
9795; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
9796; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9797; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9798; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9799; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9800; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9801; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9802; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9803; GFX90A-TGSPLIT-NEXT:    s_endpgm
9804;
9805; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
9806; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9807; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9808; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9809; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9810; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9811; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9812; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9813; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9814;
9815; GFX940-TGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
9816; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9817; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9818; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9819; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9820; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9821; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9822; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9823; GFX940-TGSPLIT-NEXT:    s_endpgm
9824;
9825; GFX11-WGP-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
9826; GFX11-WGP:       ; %bb.0: ; %entry
9827; GFX11-WGP-NEXT:    s_clause 0x1
9828; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9829; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9830; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9831; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9832; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
9833; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9834; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9835; GFX11-WGP-NEXT:    s_endpgm
9836;
9837; GFX11-CU-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
9838; GFX11-CU:       ; %bb.0: ; %entry
9839; GFX11-CU-NEXT:    s_clause 0x1
9840; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9841; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9842; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9843; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9844; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
9845; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9846; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
9847; GFX11-CU-NEXT:    s_endpgm
9848    i32* %out, i32 %in, i32 %old) {
9849entry:
9850  %gep = getelementptr i32, i32* %out, i32 4
9851  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") monotonic monotonic
9852  ret void
9853}
9854
9855define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_cmpxchg(
9856; GFX7-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
9857; GFX7:       ; %bb.0: ; %entry
9858; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9859; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9860; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9861; GFX7-NEXT:    s_add_u32 s0, s0, 16
9862; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9863; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9864; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9865; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9866; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9867; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9868; GFX7-NEXT:    s_waitcnt vmcnt(0)
9869; GFX7-NEXT:    buffer_wbinvl1_vol
9870; GFX7-NEXT:    s_endpgm
9871;
9872; GFX10-WGP-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
9873; GFX10-WGP:       ; %bb.0: ; %entry
9874; GFX10-WGP-NEXT:    s_clause 0x1
9875; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9876; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9877; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9878; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9879; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9880; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9881; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9882; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9883; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9884; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9885; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9886; GFX10-WGP-NEXT:    buffer_gl0_inv
9887; GFX10-WGP-NEXT:    buffer_gl1_inv
9888; GFX10-WGP-NEXT:    s_endpgm
9889;
9890; GFX10-CU-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
9891; GFX10-CU:       ; %bb.0: ; %entry
9892; GFX10-CU-NEXT:    s_clause 0x1
9893; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9894; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9895; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9896; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9897; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9898; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9899; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9900; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9901; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9902; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9903; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9904; GFX10-CU-NEXT:    buffer_gl0_inv
9905; GFX10-CU-NEXT:    buffer_gl1_inv
9906; GFX10-CU-NEXT:    s_endpgm
9907;
9908; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
9909; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9910; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9911; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9912; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9913; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9914; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9915; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9916; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9917; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9918; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9919; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9920; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9921; SKIP-CACHE-INV-NEXT:    s_endpgm
9922;
9923; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
9924; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9925; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9926; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9927; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9928; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9929; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9930; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9931; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9932; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9933; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9934;
9935; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
9936; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9937; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9938; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9939; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9940; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9941; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9942; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9943; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9944; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9945; GFX90A-TGSPLIT-NEXT:    s_endpgm
9946;
9947; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
9948; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9949; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9950; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9951; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9952; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9953; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9954; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9955; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9956; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
9957; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9958;
9959; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
9960; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9961; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9962; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9963; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9964; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9965; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9966; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9967; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9968; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
9969; GFX940-TGSPLIT-NEXT:    s_endpgm
9970;
9971; GFX11-WGP-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
9972; GFX11-WGP:       ; %bb.0: ; %entry
9973; GFX11-WGP-NEXT:    s_clause 0x1
9974; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9975; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9976; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9977; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9978; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
9979; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9980; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9981; GFX11-WGP-NEXT:    buffer_gl0_inv
9982; GFX11-WGP-NEXT:    buffer_gl1_inv
9983; GFX11-WGP-NEXT:    s_endpgm
9984;
9985; GFX11-CU-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
9986; GFX11-CU:       ; %bb.0: ; %entry
9987; GFX11-CU-NEXT:    s_clause 0x1
9988; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9989; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9990; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9991; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
9992; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
9993; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9994; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9995; GFX11-CU-NEXT:    buffer_gl0_inv
9996; GFX11-CU-NEXT:    buffer_gl1_inv
9997; GFX11-CU-NEXT:    s_endpgm
9998    i32* %out, i32 %in, i32 %old) {
9999entry:
10000  %gep = getelementptr i32, i32* %out, i32 4
10001  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire monotonic
10002  ret void
10003}
10004
10005define amdgpu_kernel void @flat_agent_one_as_release_monotonic_cmpxchg(
10006; GFX7-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
10007; GFX7:       ; %bb.0: ; %entry
10008; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10009; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10010; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10011; GFX7-NEXT:    s_add_u32 s0, s0, 16
10012; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10013; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10014; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10015; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10016; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10017; GFX7-NEXT:    s_waitcnt vmcnt(0)
10018; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10019; GFX7-NEXT:    s_endpgm
10020;
10021; GFX10-WGP-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
10022; GFX10-WGP:       ; %bb.0: ; %entry
10023; GFX10-WGP-NEXT:    s_clause 0x1
10024; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10025; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10026; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10027; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10028; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10029; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10030; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10031; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10032; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10033; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10034; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10035; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10036; GFX10-WGP-NEXT:    s_endpgm
10037;
10038; GFX10-CU-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
10039; GFX10-CU:       ; %bb.0: ; %entry
10040; GFX10-CU-NEXT:    s_clause 0x1
10041; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10042; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10043; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10044; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10045; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10046; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10047; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10048; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10049; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10050; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10051; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10052; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10053; GFX10-CU-NEXT:    s_endpgm
10054;
10055; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
10056; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10057; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10058; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10059; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10060; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10061; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10062; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10063; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10064; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10065; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10066; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10067; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10068; SKIP-CACHE-INV-NEXT:    s_endpgm
10069;
10070; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
10071; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10072; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10073; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10074; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10075; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10076; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10077; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10078; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10079; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10080;
10081; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
10082; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10083; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10084; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10085; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10086; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10087; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10088; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10089; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10090; GFX90A-TGSPLIT-NEXT:    s_endpgm
10091;
10092; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
10093; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10094; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10095; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10096; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10097; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10098; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10099; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
10100; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10101; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10102; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10103;
10104; GFX940-TGSPLIT-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
10105; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10106; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10107; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10108; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10109; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10110; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10111; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
10112; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10113; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10114; GFX940-TGSPLIT-NEXT:    s_endpgm
10115;
10116; GFX11-WGP-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
10117; GFX11-WGP:       ; %bb.0: ; %entry
10118; GFX11-WGP-NEXT:    s_clause 0x1
10119; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10120; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10121; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10122; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10123; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10124; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
10125; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10126; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10127; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10128; GFX11-WGP-NEXT:    s_endpgm
10129;
10130; GFX11-CU-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
10131; GFX11-CU:       ; %bb.0: ; %entry
10132; GFX11-CU-NEXT:    s_clause 0x1
10133; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10134; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10135; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10136; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10137; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10138; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
10139; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10140; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10141; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
10142; GFX11-CU-NEXT:    s_endpgm
10143    i32* %out, i32 %in, i32 %old) {
10144entry:
10145  %gep = getelementptr i32, i32* %out, i32 4
10146  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release monotonic
10147  ret void
10148}
10149
10150define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_cmpxchg(
10151; GFX7-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
10152; GFX7:       ; %bb.0: ; %entry
10153; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10154; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10155; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10156; GFX7-NEXT:    s_add_u32 s0, s0, 16
10157; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10158; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10159; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10160; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10161; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10162; GFX7-NEXT:    s_waitcnt vmcnt(0)
10163; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10164; GFX7-NEXT:    s_waitcnt vmcnt(0)
10165; GFX7-NEXT:    buffer_wbinvl1_vol
10166; GFX7-NEXT:    s_endpgm
10167;
10168; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
10169; GFX10-WGP:       ; %bb.0: ; %entry
10170; GFX10-WGP-NEXT:    s_clause 0x1
10171; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10172; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10173; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10174; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10175; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10176; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10177; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10178; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10179; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10180; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10181; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10182; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10183; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10184; GFX10-WGP-NEXT:    buffer_gl0_inv
10185; GFX10-WGP-NEXT:    buffer_gl1_inv
10186; GFX10-WGP-NEXT:    s_endpgm
10187;
10188; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
10189; GFX10-CU:       ; %bb.0: ; %entry
10190; GFX10-CU-NEXT:    s_clause 0x1
10191; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10192; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10193; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10194; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10195; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10196; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10197; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10198; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10199; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10200; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10201; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10202; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10203; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10204; GFX10-CU-NEXT:    buffer_gl0_inv
10205; GFX10-CU-NEXT:    buffer_gl1_inv
10206; GFX10-CU-NEXT:    s_endpgm
10207;
10208; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
10209; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10210; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10211; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10212; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10213; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10214; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10215; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10216; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10217; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10218; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10219; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10220; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10221; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10222; SKIP-CACHE-INV-NEXT:    s_endpgm
10223;
10224; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
10225; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10226; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10227; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10228; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10229; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10230; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10231; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10232; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10233; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10234; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
10235; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10236;
10237; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
10238; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10239; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10240; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10241; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10242; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10243; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10244; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10245; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10246; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10247; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10248; GFX90A-TGSPLIT-NEXT:    s_endpgm
10249;
10250; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
10251; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10252; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10253; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10254; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10255; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10256; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10257; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
10258; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10259; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10260; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10261; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
10262; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10263;
10264; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
10265; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10266; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10267; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10268; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10269; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10270; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10271; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
10272; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10273; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10274; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10275; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
10276; GFX940-TGSPLIT-NEXT:    s_endpgm
10277;
10278; GFX11-WGP-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
10279; GFX11-WGP:       ; %bb.0: ; %entry
10280; GFX11-WGP-NEXT:    s_clause 0x1
10281; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10282; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10283; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10284; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10285; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10286; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
10287; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10288; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10289; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10290; GFX11-WGP-NEXT:    buffer_gl0_inv
10291; GFX11-WGP-NEXT:    buffer_gl1_inv
10292; GFX11-WGP-NEXT:    s_endpgm
10293;
10294; GFX11-CU-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
10295; GFX11-CU:       ; %bb.0: ; %entry
10296; GFX11-CU-NEXT:    s_clause 0x1
10297; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10298; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10299; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10300; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10301; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10302; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
10303; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10304; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10305; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10306; GFX11-CU-NEXT:    buffer_gl0_inv
10307; GFX11-CU-NEXT:    buffer_gl1_inv
10308; GFX11-CU-NEXT:    s_endpgm
10309    i32* %out, i32 %in, i32 %old) {
10310entry:
10311  %gep = getelementptr i32, i32* %out, i32 4
10312  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel monotonic
10313  ret void
10314}
10315
10316define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_cmpxchg(
10317; GFX7-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
10318; GFX7:       ; %bb.0: ; %entry
10319; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10320; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10321; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10322; GFX7-NEXT:    s_add_u32 s0, s0, 16
10323; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10324; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10325; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10326; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10327; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10328; GFX7-NEXT:    s_waitcnt vmcnt(0)
10329; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10330; GFX7-NEXT:    s_waitcnt vmcnt(0)
10331; GFX7-NEXT:    buffer_wbinvl1_vol
10332; GFX7-NEXT:    s_endpgm
10333;
10334; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
10335; GFX10-WGP:       ; %bb.0: ; %entry
10336; GFX10-WGP-NEXT:    s_clause 0x1
10337; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10338; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10339; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10340; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10341; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10342; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10343; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10344; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10345; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10346; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10347; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10348; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10349; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10350; GFX10-WGP-NEXT:    buffer_gl0_inv
10351; GFX10-WGP-NEXT:    buffer_gl1_inv
10352; GFX10-WGP-NEXT:    s_endpgm
10353;
10354; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
10355; GFX10-CU:       ; %bb.0: ; %entry
10356; GFX10-CU-NEXT:    s_clause 0x1
10357; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10358; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10359; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10360; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10361; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10362; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10363; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10364; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10365; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10366; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10367; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10368; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10369; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10370; GFX10-CU-NEXT:    buffer_gl0_inv
10371; GFX10-CU-NEXT:    buffer_gl1_inv
10372; GFX10-CU-NEXT:    s_endpgm
10373;
10374; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
10375; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10376; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10377; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10378; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10379; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10380; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10381; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10382; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10383; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10384; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10385; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10386; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10387; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10388; SKIP-CACHE-INV-NEXT:    s_endpgm
10389;
10390; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
10391; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10392; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10393; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10394; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10395; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10396; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10397; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10398; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10399; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10400; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
10401; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10402;
10403; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
10404; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10405; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10406; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10407; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10408; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10409; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10410; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10411; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10412; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10413; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10414; GFX90A-TGSPLIT-NEXT:    s_endpgm
10415;
10416; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
10417; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10418; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10419; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10420; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10421; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10422; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10423; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
10424; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10425; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10426; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10427; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
10428; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10429;
10430; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
10431; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10432; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10433; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10434; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10435; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10436; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10437; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
10438; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10439; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10440; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10441; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
10442; GFX940-TGSPLIT-NEXT:    s_endpgm
10443;
10444; GFX11-WGP-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
10445; GFX11-WGP:       ; %bb.0: ; %entry
10446; GFX11-WGP-NEXT:    s_clause 0x1
10447; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10448; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10449; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10450; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10451; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10452; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
10453; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10454; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10455; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10456; GFX11-WGP-NEXT:    buffer_gl0_inv
10457; GFX11-WGP-NEXT:    buffer_gl1_inv
10458; GFX11-WGP-NEXT:    s_endpgm
10459;
10460; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
10461; GFX11-CU:       ; %bb.0: ; %entry
10462; GFX11-CU-NEXT:    s_clause 0x1
10463; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10464; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10465; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10466; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10467; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10468; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
10469; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10470; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10471; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10472; GFX11-CU-NEXT:    buffer_gl0_inv
10473; GFX11-CU-NEXT:    buffer_gl1_inv
10474; GFX11-CU-NEXT:    s_endpgm
10475    i32* %out, i32 %in, i32 %old) {
10476entry:
10477  %gep = getelementptr i32, i32* %out, i32 4
10478  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst monotonic
10479  ret void
10480}
10481
10482define amdgpu_kernel void @flat_agent_one_as_monotonic_acquire_cmpxchg(
10483; GFX7-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
10484; GFX7:       ; %bb.0: ; %entry
10485; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10486; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10487; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10488; GFX7-NEXT:    s_add_u32 s0, s0, 16
10489; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10490; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10491; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10492; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10493; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10494; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10495; GFX7-NEXT:    s_waitcnt vmcnt(0)
10496; GFX7-NEXT:    buffer_wbinvl1_vol
10497; GFX7-NEXT:    s_endpgm
10498;
10499; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
10500; GFX10-WGP:       ; %bb.0: ; %entry
10501; GFX10-WGP-NEXT:    s_clause 0x1
10502; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10503; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10504; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10505; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10506; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10507; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10508; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10509; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10510; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10511; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10512; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10513; GFX10-WGP-NEXT:    buffer_gl0_inv
10514; GFX10-WGP-NEXT:    buffer_gl1_inv
10515; GFX10-WGP-NEXT:    s_endpgm
10516;
10517; GFX10-CU-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
10518; GFX10-CU:       ; %bb.0: ; %entry
10519; GFX10-CU-NEXT:    s_clause 0x1
10520; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10521; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10522; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10523; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10524; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10525; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10526; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10527; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10528; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10529; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10530; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10531; GFX10-CU-NEXT:    buffer_gl0_inv
10532; GFX10-CU-NEXT:    buffer_gl1_inv
10533; GFX10-CU-NEXT:    s_endpgm
10534;
10535; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
10536; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10537; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10538; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10539; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10540; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10541; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10542; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10543; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10544; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10545; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10546; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10547; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10548; SKIP-CACHE-INV-NEXT:    s_endpgm
10549;
10550; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
10551; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10552; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10553; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10554; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10555; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10556; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10557; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10558; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10559; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
10560; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10561;
10562; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
10563; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10564; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10565; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10566; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10567; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10568; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10569; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10570; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10571; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10572; GFX90A-TGSPLIT-NEXT:    s_endpgm
10573;
10574; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
10575; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10576; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10577; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10578; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10579; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10580; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10581; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10582; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10583; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
10584; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10585;
10586; GFX940-TGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
10587; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10588; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10589; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10590; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10591; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10592; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10593; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10594; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10595; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
10596; GFX940-TGSPLIT-NEXT:    s_endpgm
10597;
10598; GFX11-WGP-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
10599; GFX11-WGP:       ; %bb.0: ; %entry
10600; GFX11-WGP-NEXT:    s_clause 0x1
10601; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10602; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10603; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10604; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10605; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10606; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10607; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10608; GFX11-WGP-NEXT:    buffer_gl0_inv
10609; GFX11-WGP-NEXT:    buffer_gl1_inv
10610; GFX11-WGP-NEXT:    s_endpgm
10611;
10612; GFX11-CU-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
10613; GFX11-CU:       ; %bb.0: ; %entry
10614; GFX11-CU-NEXT:    s_clause 0x1
10615; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10616; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10617; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10618; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10619; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10620; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10621; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10622; GFX11-CU-NEXT:    buffer_gl0_inv
10623; GFX11-CU-NEXT:    buffer_gl1_inv
10624; GFX11-CU-NEXT:    s_endpgm
10625    i32* %out, i32 %in, i32 %old) {
10626entry:
10627  %gep = getelementptr i32, i32* %out, i32 4
10628  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") monotonic acquire
10629  ret void
10630}
10631
10632define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_cmpxchg(
10633; GFX7-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
10634; GFX7:       ; %bb.0: ; %entry
10635; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10636; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10637; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10638; GFX7-NEXT:    s_add_u32 s0, s0, 16
10639; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10640; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10641; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10642; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10643; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10644; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10645; GFX7-NEXT:    s_waitcnt vmcnt(0)
10646; GFX7-NEXT:    buffer_wbinvl1_vol
10647; GFX7-NEXT:    s_endpgm
10648;
10649; GFX10-WGP-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
10650; GFX10-WGP:       ; %bb.0: ; %entry
10651; GFX10-WGP-NEXT:    s_clause 0x1
10652; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10653; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10654; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10655; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10656; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10657; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10658; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10659; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10660; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10661; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10662; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10663; GFX10-WGP-NEXT:    buffer_gl0_inv
10664; GFX10-WGP-NEXT:    buffer_gl1_inv
10665; GFX10-WGP-NEXT:    s_endpgm
10666;
10667; GFX10-CU-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
10668; GFX10-CU:       ; %bb.0: ; %entry
10669; GFX10-CU-NEXT:    s_clause 0x1
10670; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10671; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10672; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10673; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10674; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10675; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10676; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10677; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10678; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10679; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10680; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10681; GFX10-CU-NEXT:    buffer_gl0_inv
10682; GFX10-CU-NEXT:    buffer_gl1_inv
10683; GFX10-CU-NEXT:    s_endpgm
10684;
10685; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
10686; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10687; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10688; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10689; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10690; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10691; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10692; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10693; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10694; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10695; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10696; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10697; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10698; SKIP-CACHE-INV-NEXT:    s_endpgm
10699;
10700; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
10701; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10702; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10703; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10704; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10705; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10706; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10707; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10708; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10709; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
10710; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10711;
10712; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
10713; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10714; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10715; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10716; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10717; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10718; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10719; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10720; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10721; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10722; GFX90A-TGSPLIT-NEXT:    s_endpgm
10723;
10724; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
10725; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10726; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10727; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10728; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10729; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10730; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10731; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10732; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10733; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
10734; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10735;
10736; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
10737; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10738; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10739; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10740; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10741; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10742; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10743; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10744; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10745; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
10746; GFX940-TGSPLIT-NEXT:    s_endpgm
10747;
10748; GFX11-WGP-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
10749; GFX11-WGP:       ; %bb.0: ; %entry
10750; GFX11-WGP-NEXT:    s_clause 0x1
10751; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10752; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10753; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10754; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10755; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10756; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10757; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10758; GFX11-WGP-NEXT:    buffer_gl0_inv
10759; GFX11-WGP-NEXT:    buffer_gl1_inv
10760; GFX11-WGP-NEXT:    s_endpgm
10761;
10762; GFX11-CU-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
10763; GFX11-CU:       ; %bb.0: ; %entry
10764; GFX11-CU-NEXT:    s_clause 0x1
10765; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10766; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10767; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10768; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10769; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10770; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10771; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10772; GFX11-CU-NEXT:    buffer_gl0_inv
10773; GFX11-CU-NEXT:    buffer_gl1_inv
10774; GFX11-CU-NEXT:    s_endpgm
10775    i32* %out, i32 %in, i32 %old) {
10776entry:
10777  %gep = getelementptr i32, i32* %out, i32 4
10778  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire acquire
10779  ret void
10780}
10781
10782define amdgpu_kernel void @flat_agent_one_as_release_acquire_cmpxchg(
10783; GFX7-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
10784; GFX7:       ; %bb.0: ; %entry
10785; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10786; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10787; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10788; GFX7-NEXT:    s_add_u32 s0, s0, 16
10789; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10790; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10791; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10792; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10793; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10794; GFX7-NEXT:    s_waitcnt vmcnt(0)
10795; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10796; GFX7-NEXT:    s_waitcnt vmcnt(0)
10797; GFX7-NEXT:    buffer_wbinvl1_vol
10798; GFX7-NEXT:    s_endpgm
10799;
10800; GFX10-WGP-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
10801; GFX10-WGP:       ; %bb.0: ; %entry
10802; GFX10-WGP-NEXT:    s_clause 0x1
10803; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10804; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10805; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10806; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10807; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10808; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10809; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10810; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10811; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10812; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10813; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10814; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10815; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10816; GFX10-WGP-NEXT:    buffer_gl0_inv
10817; GFX10-WGP-NEXT:    buffer_gl1_inv
10818; GFX10-WGP-NEXT:    s_endpgm
10819;
10820; GFX10-CU-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
10821; GFX10-CU:       ; %bb.0: ; %entry
10822; GFX10-CU-NEXT:    s_clause 0x1
10823; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10824; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10825; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10826; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10827; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10828; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10829; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10830; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10831; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10832; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10833; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10834; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10835; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10836; GFX10-CU-NEXT:    buffer_gl0_inv
10837; GFX10-CU-NEXT:    buffer_gl1_inv
10838; GFX10-CU-NEXT:    s_endpgm
10839;
10840; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
10841; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10842; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10843; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10844; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10845; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10846; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10847; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10848; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10849; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10850; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10851; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10852; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10853; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10854; SKIP-CACHE-INV-NEXT:    s_endpgm
10855;
10856; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
10857; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10858; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10859; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10860; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10861; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10862; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10863; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10864; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10865; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10866; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
10867; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10868;
10869; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
10870; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10871; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10872; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10873; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10874; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10875; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10876; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10877; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10878; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10879; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10880; GFX90A-TGSPLIT-NEXT:    s_endpgm
10881;
10882; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
10883; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10884; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10885; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10886; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10887; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10888; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10889; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
10890; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10891; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10892; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10893; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
10894; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10895;
10896; GFX940-TGSPLIT-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
10897; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10898; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10899; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10900; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10901; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10902; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10903; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
10904; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10905; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10906; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10907; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
10908; GFX940-TGSPLIT-NEXT:    s_endpgm
10909;
10910; GFX11-WGP-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
10911; GFX11-WGP:       ; %bb.0: ; %entry
10912; GFX11-WGP-NEXT:    s_clause 0x1
10913; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10914; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10915; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10916; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10917; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10918; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
10919; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10920; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10921; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10922; GFX11-WGP-NEXT:    buffer_gl0_inv
10923; GFX11-WGP-NEXT:    buffer_gl1_inv
10924; GFX11-WGP-NEXT:    s_endpgm
10925;
10926; GFX11-CU-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
10927; GFX11-CU:       ; %bb.0: ; %entry
10928; GFX11-CU-NEXT:    s_clause 0x1
10929; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10930; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10931; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10932; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
10933; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
10934; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
10935; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10936; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10937; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10938; GFX11-CU-NEXT:    buffer_gl0_inv
10939; GFX11-CU-NEXT:    buffer_gl1_inv
10940; GFX11-CU-NEXT:    s_endpgm
10941    i32* %out, i32 %in, i32 %old) {
10942entry:
10943  %gep = getelementptr i32, i32* %out, i32 4
10944  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release acquire
10945  ret void
10946}
10947
10948define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_cmpxchg(
10949; GFX7-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
10950; GFX7:       ; %bb.0: ; %entry
10951; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10952; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10953; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10954; GFX7-NEXT:    s_add_u32 s0, s0, 16
10955; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10956; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10957; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10958; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10959; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10960; GFX7-NEXT:    s_waitcnt vmcnt(0)
10961; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10962; GFX7-NEXT:    s_waitcnt vmcnt(0)
10963; GFX7-NEXT:    buffer_wbinvl1_vol
10964; GFX7-NEXT:    s_endpgm
10965;
10966; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
10967; GFX10-WGP:       ; %bb.0: ; %entry
10968; GFX10-WGP-NEXT:    s_clause 0x1
10969; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10970; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10971; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10972; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10973; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10974; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10975; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10976; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10977; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10978; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10979; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10980; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10981; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10982; GFX10-WGP-NEXT:    buffer_gl0_inv
10983; GFX10-WGP-NEXT:    buffer_gl1_inv
10984; GFX10-WGP-NEXT:    s_endpgm
10985;
10986; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
10987; GFX10-CU:       ; %bb.0: ; %entry
10988; GFX10-CU-NEXT:    s_clause 0x1
10989; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10990; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10991; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10992; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10993; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10994; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10995; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10996; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10997; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10998; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10999; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11000; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11001; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11002; GFX10-CU-NEXT:    buffer_gl0_inv
11003; GFX10-CU-NEXT:    buffer_gl1_inv
11004; GFX10-CU-NEXT:    s_endpgm
11005;
11006; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
11007; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11008; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11009; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11010; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11011; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
11012; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
11013; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11014; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11015; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11016; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11017; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11018; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11019; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11020; SKIP-CACHE-INV-NEXT:    s_endpgm
11021;
11022; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
11023; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11024; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11025; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11026; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11027; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11028; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11029; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11030; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11031; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11032; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
11033; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11034;
11035; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
11036; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11037; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11038; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11039; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11040; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11041; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11042; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11043; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11044; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11045; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11046; GFX90A-TGSPLIT-NEXT:    s_endpgm
11047;
11048; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
11049; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11050; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11051; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11052; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11053; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11054; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11055; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
11056; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11057; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11058; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11059; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
11060; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11061;
11062; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
11063; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11064; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11065; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11066; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11067; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11068; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11069; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
11070; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11071; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11072; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11073; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
11074; GFX940-TGSPLIT-NEXT:    s_endpgm
11075;
11076; GFX11-WGP-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
11077; GFX11-WGP:       ; %bb.0: ; %entry
11078; GFX11-WGP-NEXT:    s_clause 0x1
11079; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11080; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11081; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11082; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11083; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11084; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11085; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11086; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11087; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11088; GFX11-WGP-NEXT:    buffer_gl0_inv
11089; GFX11-WGP-NEXT:    buffer_gl1_inv
11090; GFX11-WGP-NEXT:    s_endpgm
11091;
11092; GFX11-CU-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
11093; GFX11-CU:       ; %bb.0: ; %entry
11094; GFX11-CU-NEXT:    s_clause 0x1
11095; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11096; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11097; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11098; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11099; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11100; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11101; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11102; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11103; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11104; GFX11-CU-NEXT:    buffer_gl0_inv
11105; GFX11-CU-NEXT:    buffer_gl1_inv
11106; GFX11-CU-NEXT:    s_endpgm
11107    i32* %out, i32 %in, i32 %old) {
11108entry:
11109  %gep = getelementptr i32, i32* %out, i32 4
11110  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel acquire
11111  ret void
11112}
11113
11114define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_cmpxchg(
11115; GFX7-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
11116; GFX7:       ; %bb.0: ; %entry
11117; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11118; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11119; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11120; GFX7-NEXT:    s_add_u32 s0, s0, 16
11121; GFX7-NEXT:    s_addc_u32 s1, s1, 0
11122; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11123; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11124; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11125; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11126; GFX7-NEXT:    s_waitcnt vmcnt(0)
11127; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11128; GFX7-NEXT:    s_waitcnt vmcnt(0)
11129; GFX7-NEXT:    buffer_wbinvl1_vol
11130; GFX7-NEXT:    s_endpgm
11131;
11132; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
11133; GFX10-WGP:       ; %bb.0: ; %entry
11134; GFX10-WGP-NEXT:    s_clause 0x1
11135; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11136; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11137; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11138; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
11139; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
11140; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11141; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11142; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11143; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11144; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11145; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11146; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11147; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11148; GFX10-WGP-NEXT:    buffer_gl0_inv
11149; GFX10-WGP-NEXT:    buffer_gl1_inv
11150; GFX10-WGP-NEXT:    s_endpgm
11151;
11152; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
11153; GFX10-CU:       ; %bb.0: ; %entry
11154; GFX10-CU-NEXT:    s_clause 0x1
11155; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11156; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11157; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11158; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
11159; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
11160; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11161; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11162; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11163; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11164; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11165; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11166; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11167; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11168; GFX10-CU-NEXT:    buffer_gl0_inv
11169; GFX10-CU-NEXT:    buffer_gl1_inv
11170; GFX10-CU-NEXT:    s_endpgm
11171;
11172; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
11173; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11174; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11175; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11176; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11177; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
11178; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
11179; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11180; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11181; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11182; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11183; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11184; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11185; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11186; SKIP-CACHE-INV-NEXT:    s_endpgm
11187;
11188; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
11189; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11190; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11191; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11192; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11193; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11194; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11195; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11196; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11197; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11198; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
11199; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11200;
11201; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
11202; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11203; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11204; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11205; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11206; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11207; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11208; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11209; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11210; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11211; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11212; GFX90A-TGSPLIT-NEXT:    s_endpgm
11213;
11214; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
11215; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11216; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11217; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11218; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11219; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11220; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11221; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
11222; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11223; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11224; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11225; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
11226; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11227;
11228; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
11229; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11230; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11231; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11232; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11233; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11234; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11235; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
11236; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11237; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11238; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11239; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
11240; GFX940-TGSPLIT-NEXT:    s_endpgm
11241;
11242; GFX11-WGP-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
11243; GFX11-WGP:       ; %bb.0: ; %entry
11244; GFX11-WGP-NEXT:    s_clause 0x1
11245; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11246; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11247; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11248; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11249; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11250; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11251; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11252; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11253; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11254; GFX11-WGP-NEXT:    buffer_gl0_inv
11255; GFX11-WGP-NEXT:    buffer_gl1_inv
11256; GFX11-WGP-NEXT:    s_endpgm
11257;
11258; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
11259; GFX11-CU:       ; %bb.0: ; %entry
11260; GFX11-CU-NEXT:    s_clause 0x1
11261; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11262; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11263; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11264; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11265; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11266; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11267; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11268; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11269; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11270; GFX11-CU-NEXT:    buffer_gl0_inv
11271; GFX11-CU-NEXT:    buffer_gl1_inv
11272; GFX11-CU-NEXT:    s_endpgm
11273    i32* %out, i32 %in, i32 %old) {
11274entry:
11275  %gep = getelementptr i32, i32* %out, i32 4
11276  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst acquire
11277  ret void
11278}
11279
11280define amdgpu_kernel void @flat_agent_one_as_monotonic_seq_cst_cmpxchg(
11281; GFX7-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
11282; GFX7:       ; %bb.0: ; %entry
11283; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11284; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11285; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11286; GFX7-NEXT:    s_add_u32 s0, s0, 16
11287; GFX7-NEXT:    s_addc_u32 s1, s1, 0
11288; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11289; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11290; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11291; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11292; GFX7-NEXT:    s_waitcnt vmcnt(0)
11293; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11294; GFX7-NEXT:    s_waitcnt vmcnt(0)
11295; GFX7-NEXT:    buffer_wbinvl1_vol
11296; GFX7-NEXT:    s_endpgm
11297;
11298; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
11299; GFX10-WGP:       ; %bb.0: ; %entry
11300; GFX10-WGP-NEXT:    s_clause 0x1
11301; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11302; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11303; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11304; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
11305; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
11306; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11307; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11308; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11309; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11310; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11311; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11312; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11313; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11314; GFX10-WGP-NEXT:    buffer_gl0_inv
11315; GFX10-WGP-NEXT:    buffer_gl1_inv
11316; GFX10-WGP-NEXT:    s_endpgm
11317;
11318; GFX10-CU-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
11319; GFX10-CU:       ; %bb.0: ; %entry
11320; GFX10-CU-NEXT:    s_clause 0x1
11321; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11322; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11323; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11324; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
11325; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
11326; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11327; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11328; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11329; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11330; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11331; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11332; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11333; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11334; GFX10-CU-NEXT:    buffer_gl0_inv
11335; GFX10-CU-NEXT:    buffer_gl1_inv
11336; GFX10-CU-NEXT:    s_endpgm
11337;
11338; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
11339; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11340; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11341; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11342; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11343; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
11344; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
11345; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11346; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11347; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11348; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11349; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11350; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11351; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11352; SKIP-CACHE-INV-NEXT:    s_endpgm
11353;
11354; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
11355; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11356; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11357; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11358; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11359; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11360; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11361; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11362; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11363; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11364; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
11365; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11366;
11367; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
11368; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11369; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11370; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11371; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11372; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11373; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11374; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11375; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11376; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11377; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11378; GFX90A-TGSPLIT-NEXT:    s_endpgm
11379;
11380; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
11381; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11382; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11383; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11384; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11385; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11386; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11387; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
11388; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11389; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11390; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11391; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
11392; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11393;
11394; GFX940-TGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
11395; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11396; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11397; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11398; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11399; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11400; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11401; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
11402; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11403; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11404; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11405; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
11406; GFX940-TGSPLIT-NEXT:    s_endpgm
11407;
11408; GFX11-WGP-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
11409; GFX11-WGP:       ; %bb.0: ; %entry
11410; GFX11-WGP-NEXT:    s_clause 0x1
11411; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11412; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11413; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11414; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11415; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11416; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11417; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11418; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11419; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11420; GFX11-WGP-NEXT:    buffer_gl0_inv
11421; GFX11-WGP-NEXT:    buffer_gl1_inv
11422; GFX11-WGP-NEXT:    s_endpgm
11423;
11424; GFX11-CU-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
11425; GFX11-CU:       ; %bb.0: ; %entry
11426; GFX11-CU-NEXT:    s_clause 0x1
11427; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11428; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11429; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11430; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11431; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11432; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11433; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11434; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11435; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11436; GFX11-CU-NEXT:    buffer_gl0_inv
11437; GFX11-CU-NEXT:    buffer_gl1_inv
11438; GFX11-CU-NEXT:    s_endpgm
11439    i32* %out, i32 %in, i32 %old) {
11440entry:
11441  %gep = getelementptr i32, i32* %out, i32 4
11442  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") monotonic seq_cst
11443  ret void
11444}
11445
11446define amdgpu_kernel void @flat_agent_one_as_acquire_seq_cst_cmpxchg(
11447; GFX7-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
11448; GFX7:       ; %bb.0: ; %entry
11449; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11450; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11451; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11452; GFX7-NEXT:    s_add_u32 s0, s0, 16
11453; GFX7-NEXT:    s_addc_u32 s1, s1, 0
11454; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11455; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11456; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11457; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11458; GFX7-NEXT:    s_waitcnt vmcnt(0)
11459; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11460; GFX7-NEXT:    s_waitcnt vmcnt(0)
11461; GFX7-NEXT:    buffer_wbinvl1_vol
11462; GFX7-NEXT:    s_endpgm
11463;
11464; GFX10-WGP-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
11465; GFX10-WGP:       ; %bb.0: ; %entry
11466; GFX10-WGP-NEXT:    s_clause 0x1
11467; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11468; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11469; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11470; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
11471; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
11472; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11473; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11474; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11475; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11476; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11477; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11478; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11479; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11480; GFX10-WGP-NEXT:    buffer_gl0_inv
11481; GFX10-WGP-NEXT:    buffer_gl1_inv
11482; GFX10-WGP-NEXT:    s_endpgm
11483;
11484; GFX10-CU-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
11485; GFX10-CU:       ; %bb.0: ; %entry
11486; GFX10-CU-NEXT:    s_clause 0x1
11487; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11488; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11489; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11490; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
11491; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
11492; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11493; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11494; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11495; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11496; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11497; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11498; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11499; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11500; GFX10-CU-NEXT:    buffer_gl0_inv
11501; GFX10-CU-NEXT:    buffer_gl1_inv
11502; GFX10-CU-NEXT:    s_endpgm
11503;
11504; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
11505; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11506; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11507; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11508; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11509; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
11510; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
11511; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11512; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11513; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11514; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11515; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11516; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11517; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11518; SKIP-CACHE-INV-NEXT:    s_endpgm
11519;
11520; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
11521; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11522; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11523; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11524; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11525; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11526; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11527; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11528; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11529; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11530; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
11531; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11532;
11533; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
11534; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11535; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11536; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11537; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11538; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11539; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11540; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11541; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11542; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11543; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11544; GFX90A-TGSPLIT-NEXT:    s_endpgm
11545;
11546; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
11547; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11548; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11549; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11550; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11551; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11552; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11553; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
11554; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11555; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11556; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11557; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
11558; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11559;
11560; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
11561; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11562; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11563; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11564; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11565; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11566; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11567; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
11568; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11569; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11570; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11571; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
11572; GFX940-TGSPLIT-NEXT:    s_endpgm
11573;
11574; GFX11-WGP-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
11575; GFX11-WGP:       ; %bb.0: ; %entry
11576; GFX11-WGP-NEXT:    s_clause 0x1
11577; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11578; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11579; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11580; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11581; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11582; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11583; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11584; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11585; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11586; GFX11-WGP-NEXT:    buffer_gl0_inv
11587; GFX11-WGP-NEXT:    buffer_gl1_inv
11588; GFX11-WGP-NEXT:    s_endpgm
11589;
11590; GFX11-CU-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
11591; GFX11-CU:       ; %bb.0: ; %entry
11592; GFX11-CU-NEXT:    s_clause 0x1
11593; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11594; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11595; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11596; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11597; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11598; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11599; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11600; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11601; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11602; GFX11-CU-NEXT:    buffer_gl0_inv
11603; GFX11-CU-NEXT:    buffer_gl1_inv
11604; GFX11-CU-NEXT:    s_endpgm
11605    i32* %out, i32 %in, i32 %old) {
11606entry:
11607  %gep = getelementptr i32, i32* %out, i32 4
11608  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire seq_cst
11609  ret void
11610}
11611
11612define amdgpu_kernel void @flat_agent_one_as_release_seq_cst_cmpxchg(
11613; GFX7-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
11614; GFX7:       ; %bb.0: ; %entry
11615; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11616; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11617; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11618; GFX7-NEXT:    s_add_u32 s0, s0, 16
11619; GFX7-NEXT:    s_addc_u32 s1, s1, 0
11620; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11621; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11622; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11623; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11624; GFX7-NEXT:    s_waitcnt vmcnt(0)
11625; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11626; GFX7-NEXT:    s_waitcnt vmcnt(0)
11627; GFX7-NEXT:    buffer_wbinvl1_vol
11628; GFX7-NEXT:    s_endpgm
11629;
11630; GFX10-WGP-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
11631; GFX10-WGP:       ; %bb.0: ; %entry
11632; GFX10-WGP-NEXT:    s_clause 0x1
11633; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11634; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11635; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11636; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
11637; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
11638; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11639; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11640; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11641; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11642; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11643; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11644; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11645; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11646; GFX10-WGP-NEXT:    buffer_gl0_inv
11647; GFX10-WGP-NEXT:    buffer_gl1_inv
11648; GFX10-WGP-NEXT:    s_endpgm
11649;
11650; GFX10-CU-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
11651; GFX10-CU:       ; %bb.0: ; %entry
11652; GFX10-CU-NEXT:    s_clause 0x1
11653; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11654; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11655; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11656; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
11657; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
11658; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11659; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11660; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11661; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11662; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11663; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11664; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11665; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11666; GFX10-CU-NEXT:    buffer_gl0_inv
11667; GFX10-CU-NEXT:    buffer_gl1_inv
11668; GFX10-CU-NEXT:    s_endpgm
11669;
11670; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
11671; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11672; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11673; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11674; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11675; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
11676; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
11677; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11678; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11679; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11680; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11681; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11682; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11683; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11684; SKIP-CACHE-INV-NEXT:    s_endpgm
11685;
11686; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
11687; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11688; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11689; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11690; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11691; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11692; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11693; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11694; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11695; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11696; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
11697; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11698;
11699; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
11700; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11701; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11702; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11703; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11704; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11705; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11706; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11707; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11708; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11709; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11710; GFX90A-TGSPLIT-NEXT:    s_endpgm
11711;
11712; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
11713; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11714; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11715; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11716; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11717; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11718; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11719; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
11720; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11721; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11722; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11723; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
11724; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11725;
11726; GFX940-TGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
11727; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11728; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11729; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11730; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11731; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11732; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11733; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
11734; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11735; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11736; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11737; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
11738; GFX940-TGSPLIT-NEXT:    s_endpgm
11739;
11740; GFX11-WGP-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
11741; GFX11-WGP:       ; %bb.0: ; %entry
11742; GFX11-WGP-NEXT:    s_clause 0x1
11743; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11744; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11745; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11746; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11747; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11748; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11749; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11750; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11751; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11752; GFX11-WGP-NEXT:    buffer_gl0_inv
11753; GFX11-WGP-NEXT:    buffer_gl1_inv
11754; GFX11-WGP-NEXT:    s_endpgm
11755;
11756; GFX11-CU-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
11757; GFX11-CU:       ; %bb.0: ; %entry
11758; GFX11-CU-NEXT:    s_clause 0x1
11759; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11760; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11761; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11762; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11763; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11764; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11765; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11766; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11767; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11768; GFX11-CU-NEXT:    buffer_gl0_inv
11769; GFX11-CU-NEXT:    buffer_gl1_inv
11770; GFX11-CU-NEXT:    s_endpgm
11771    i32* %out, i32 %in, i32 %old) {
11772entry:
11773  %gep = getelementptr i32, i32* %out, i32 4
11774  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release seq_cst
11775  ret void
11776}
11777
11778define amdgpu_kernel void @flat_agent_one_as_acq_rel_seq_cst_cmpxchg(
11779; GFX7-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
11780; GFX7:       ; %bb.0: ; %entry
11781; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11782; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11783; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11784; GFX7-NEXT:    s_add_u32 s0, s0, 16
11785; GFX7-NEXT:    s_addc_u32 s1, s1, 0
11786; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11787; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11788; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11789; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11790; GFX7-NEXT:    s_waitcnt vmcnt(0)
11791; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11792; GFX7-NEXT:    s_waitcnt vmcnt(0)
11793; GFX7-NEXT:    buffer_wbinvl1_vol
11794; GFX7-NEXT:    s_endpgm
11795;
11796; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
11797; GFX10-WGP:       ; %bb.0: ; %entry
11798; GFX10-WGP-NEXT:    s_clause 0x1
11799; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11800; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11801; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11802; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
11803; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
11804; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11805; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11806; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11807; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11808; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11809; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11810; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11811; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11812; GFX10-WGP-NEXT:    buffer_gl0_inv
11813; GFX10-WGP-NEXT:    buffer_gl1_inv
11814; GFX10-WGP-NEXT:    s_endpgm
11815;
11816; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
11817; GFX10-CU:       ; %bb.0: ; %entry
11818; GFX10-CU-NEXT:    s_clause 0x1
11819; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11820; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11821; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11822; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
11823; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
11824; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11825; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11826; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11827; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11828; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11829; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11830; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11831; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11832; GFX10-CU-NEXT:    buffer_gl0_inv
11833; GFX10-CU-NEXT:    buffer_gl1_inv
11834; GFX10-CU-NEXT:    s_endpgm
11835;
11836; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
11837; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11838; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11839; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11840; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11841; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
11842; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
11843; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11844; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11845; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11846; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11847; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11848; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11849; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11850; SKIP-CACHE-INV-NEXT:    s_endpgm
11851;
11852; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
11853; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11854; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11855; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11856; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11857; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11858; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11859; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11860; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11861; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11862; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
11863; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11864;
11865; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
11866; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11867; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11868; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11869; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11870; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11871; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11872; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11873; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11874; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11875; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11876; GFX90A-TGSPLIT-NEXT:    s_endpgm
11877;
11878; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
11879; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11880; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11881; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11882; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11883; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11884; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11885; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
11886; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11887; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11888; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11889; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
11890; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11891;
11892; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
11893; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11894; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11895; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11896; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11897; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11898; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11899; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
11900; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11901; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11902; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11903; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
11904; GFX940-TGSPLIT-NEXT:    s_endpgm
11905;
11906; GFX11-WGP-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
11907; GFX11-WGP:       ; %bb.0: ; %entry
11908; GFX11-WGP-NEXT:    s_clause 0x1
11909; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11910; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11911; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11912; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11913; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11914; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11915; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11916; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11917; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11918; GFX11-WGP-NEXT:    buffer_gl0_inv
11919; GFX11-WGP-NEXT:    buffer_gl1_inv
11920; GFX11-WGP-NEXT:    s_endpgm
11921;
11922; GFX11-CU-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
11923; GFX11-CU:       ; %bb.0: ; %entry
11924; GFX11-CU-NEXT:    s_clause 0x1
11925; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11926; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11927; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11928; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
11929; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
11930; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11931; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11932; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11933; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11934; GFX11-CU-NEXT:    buffer_gl0_inv
11935; GFX11-CU-NEXT:    buffer_gl1_inv
11936; GFX11-CU-NEXT:    s_endpgm
11937    i32* %out, i32 %in, i32 %old) {
11938entry:
11939  %gep = getelementptr i32, i32* %out, i32 4
11940  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel seq_cst
11941  ret void
11942}
11943
11944define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_cmpxchg(
11945; GFX7-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
11946; GFX7:       ; %bb.0: ; %entry
11947; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11948; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11949; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11950; GFX7-NEXT:    s_add_u32 s0, s0, 16
11951; GFX7-NEXT:    s_addc_u32 s1, s1, 0
11952; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11953; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11954; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11955; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11956; GFX7-NEXT:    s_waitcnt vmcnt(0)
11957; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11958; GFX7-NEXT:    s_waitcnt vmcnt(0)
11959; GFX7-NEXT:    buffer_wbinvl1_vol
11960; GFX7-NEXT:    s_endpgm
11961;
11962; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
11963; GFX10-WGP:       ; %bb.0: ; %entry
11964; GFX10-WGP-NEXT:    s_clause 0x1
11965; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11966; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11967; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11968; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
11969; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
11970; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11971; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11972; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11973; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11974; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11975; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11976; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11977; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11978; GFX10-WGP-NEXT:    buffer_gl0_inv
11979; GFX10-WGP-NEXT:    buffer_gl1_inv
11980; GFX10-WGP-NEXT:    s_endpgm
11981;
11982; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
11983; GFX10-CU:       ; %bb.0: ; %entry
11984; GFX10-CU-NEXT:    s_clause 0x1
11985; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11986; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11987; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11988; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
11989; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
11990; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11991; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11992; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11993; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11994; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11995; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11996; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11997; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11998; GFX10-CU-NEXT:    buffer_gl0_inv
11999; GFX10-CU-NEXT:    buffer_gl1_inv
12000; GFX10-CU-NEXT:    s_endpgm
12001;
12002; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
12003; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12004; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12005; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12006; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12007; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
12008; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
12009; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12010; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12011; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12012; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12013; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12014; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
12015; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12016; SKIP-CACHE-INV-NEXT:    s_endpgm
12017;
12018; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
12019; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12020; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12021; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12022; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12023; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12024; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12025; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12026; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
12027; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12028; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
12029; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12030;
12031; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
12032; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12033; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12034; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12035; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12036; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12037; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12038; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12039; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
12040; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12041; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
12042; GFX90A-TGSPLIT-NEXT:    s_endpgm
12043;
12044; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
12045; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12046; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12047; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12048; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12049; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12050; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12051; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
12052; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12053; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
12054; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12055; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
12056; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12057;
12058; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
12059; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12060; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12061; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12062; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12063; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12064; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12065; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
12066; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12067; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
12068; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12069; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
12070; GFX940-TGSPLIT-NEXT:    s_endpgm
12071;
12072; GFX11-WGP-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
12073; GFX11-WGP:       ; %bb.0: ; %entry
12074; GFX11-WGP-NEXT:    s_clause 0x1
12075; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12076; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12077; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12078; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12079; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12080; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12081; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12082; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
12083; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12084; GFX11-WGP-NEXT:    buffer_gl0_inv
12085; GFX11-WGP-NEXT:    buffer_gl1_inv
12086; GFX11-WGP-NEXT:    s_endpgm
12087;
12088; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
12089; GFX11-CU:       ; %bb.0: ; %entry
12090; GFX11-CU-NEXT:    s_clause 0x1
12091; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12092; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12093; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12094; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12095; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12096; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
12097; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12098; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
12099; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12100; GFX11-CU-NEXT:    buffer_gl0_inv
12101; GFX11-CU-NEXT:    buffer_gl1_inv
12102; GFX11-CU-NEXT:    s_endpgm
12103    i32* %out, i32 %in, i32 %old) {
12104entry:
12105  %gep = getelementptr i32, i32* %out, i32 4
12106  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst seq_cst
12107  ret void
12108}
12109
12110define amdgpu_kernel void @flat_agent_one_as_monotonic_monotonic_ret_cmpxchg(
12111; GFX7-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
12112; GFX7:       ; %bb.0: ; %entry
12113; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12114; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12115; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12116; GFX7-NEXT:    s_add_u32 s4, s0, 16
12117; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12118; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12119; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12120; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12121; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12122; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12123; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12124; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12125; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12126; GFX7-NEXT:    flat_store_dword v[0:1], v2
12127; GFX7-NEXT:    s_endpgm
12128;
12129; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
12130; GFX10-WGP:       ; %bb.0: ; %entry
12131; GFX10-WGP-NEXT:    s_clause 0x1
12132; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12133; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12134; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12135; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12136; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12137; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12138; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12139; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12140; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12141; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12142; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12143; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12144; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12145; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12146; GFX10-WGP-NEXT:    s_endpgm
12147;
12148; GFX10-CU-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
12149; GFX10-CU:       ; %bb.0: ; %entry
12150; GFX10-CU-NEXT:    s_clause 0x1
12151; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12152; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12153; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12154; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12155; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12156; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12157; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12158; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12159; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12160; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12161; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12162; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12163; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12164; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12165; GFX10-CU-NEXT:    s_endpgm
12166;
12167; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
12168; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12169; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12170; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12171; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12172; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12173; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12174; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12175; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12176; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12177; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12178; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12179; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12180; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12181; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12182; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12183; SKIP-CACHE-INV-NEXT:    s_endpgm
12184;
12185; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
12186; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12187; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12188; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12189; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12190; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12191; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12192; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12193; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12194; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12195; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12196;
12197; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
12198; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12199; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12200; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12201; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12202; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12203; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12204; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12205; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12206; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12207; GFX90A-TGSPLIT-NEXT:    s_endpgm
12208;
12209; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
12210; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12211; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12212; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12213; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12214; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12215; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12216; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12217; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12218; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12219; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12220;
12221; GFX940-TGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
12222; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12223; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12224; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12225; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12226; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12227; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12228; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12229; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12230; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12231; GFX940-TGSPLIT-NEXT:    s_endpgm
12232;
12233; GFX11-WGP-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
12234; GFX11-WGP:       ; %bb.0: ; %entry
12235; GFX11-WGP-NEXT:    s_clause 0x1
12236; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12237; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12238; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12239; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12240; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12241; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12242; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12243; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12244; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12245; GFX11-WGP-NEXT:    s_endpgm
12246;
12247; GFX11-CU-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
12248; GFX11-CU:       ; %bb.0: ; %entry
12249; GFX11-CU-NEXT:    s_clause 0x1
12250; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12251; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12252; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12253; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12254; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12255; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12256; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12257; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12258; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12259; GFX11-CU-NEXT:    s_endpgm
12260    i32* %out, i32 %in, i32 %old) {
12261entry:
12262  %gep = getelementptr i32, i32* %out, i32 4
12263  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") monotonic monotonic
12264  %val0 = extractvalue { i32, i1 } %val, 0
12265  store i32 %val0, i32* %out, align 4
12266  ret void
12267}
12268
12269define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_ret_cmpxchg(
12270; GFX7-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
12271; GFX7:       ; %bb.0: ; %entry
12272; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12273; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12274; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12275; GFX7-NEXT:    s_add_u32 s4, s0, 16
12276; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12277; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12278; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12279; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12280; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12281; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12282; GFX7-NEXT:    s_waitcnt vmcnt(0)
12283; GFX7-NEXT:    buffer_wbinvl1_vol
12284; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12285; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12286; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12287; GFX7-NEXT:    flat_store_dword v[0:1], v2
12288; GFX7-NEXT:    s_endpgm
12289;
12290; GFX10-WGP-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
12291; GFX10-WGP:       ; %bb.0: ; %entry
12292; GFX10-WGP-NEXT:    s_clause 0x1
12293; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12294; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12295; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12296; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12297; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12298; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12299; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12300; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12301; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12302; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12303; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12304; GFX10-WGP-NEXT:    buffer_gl0_inv
12305; GFX10-WGP-NEXT:    buffer_gl1_inv
12306; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12307; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12308; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12309; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12310; GFX10-WGP-NEXT:    s_endpgm
12311;
12312; GFX10-CU-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
12313; GFX10-CU:       ; %bb.0: ; %entry
12314; GFX10-CU-NEXT:    s_clause 0x1
12315; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12316; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12317; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12318; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12319; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12320; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12321; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12322; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12323; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12324; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12325; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12326; GFX10-CU-NEXT:    buffer_gl0_inv
12327; GFX10-CU-NEXT:    buffer_gl1_inv
12328; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12329; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12330; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12331; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12332; GFX10-CU-NEXT:    s_endpgm
12333;
12334; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
12335; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12336; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12337; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12338; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12339; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12340; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12341; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12342; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12343; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12344; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12345; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12346; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12347; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12348; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12349; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12350; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12351; SKIP-CACHE-INV-NEXT:    s_endpgm
12352;
12353; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
12354; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12355; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12356; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12357; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12358; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12359; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12360; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12361; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12362; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
12363; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12364; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12365; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12366;
12367; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
12368; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12369; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12370; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12371; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12372; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12373; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12374; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12375; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12376; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
12377; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12378; GFX90A-TGSPLIT-NEXT:    s_endpgm
12379;
12380; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
12381; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12382; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12383; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12384; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12385; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12386; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12387; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12388; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12389; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
12390; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12391; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12392; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12393;
12394; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
12395; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12396; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12397; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12398; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12399; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12400; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12401; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12402; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12403; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
12404; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12405; GFX940-TGSPLIT-NEXT:    s_endpgm
12406;
12407; GFX11-WGP-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
12408; GFX11-WGP:       ; %bb.0: ; %entry
12409; GFX11-WGP-NEXT:    s_clause 0x1
12410; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12411; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12412; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12413; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12414; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12415; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12416; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12417; GFX11-WGP-NEXT:    buffer_gl0_inv
12418; GFX11-WGP-NEXT:    buffer_gl1_inv
12419; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12420; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12421; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12422; GFX11-WGP-NEXT:    s_endpgm
12423;
12424; GFX11-CU-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
12425; GFX11-CU:       ; %bb.0: ; %entry
12426; GFX11-CU-NEXT:    s_clause 0x1
12427; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12428; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12429; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12430; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12431; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12432; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12433; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
12434; GFX11-CU-NEXT:    buffer_gl0_inv
12435; GFX11-CU-NEXT:    buffer_gl1_inv
12436; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12437; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12438; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12439; GFX11-CU-NEXT:    s_endpgm
12440    i32* %out, i32 %in, i32 %old) {
12441entry:
12442  %gep = getelementptr i32, i32* %out, i32 4
12443  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire monotonic
12444  %val0 = extractvalue { i32, i1 } %val, 0
12445  store i32 %val0, i32* %out, align 4
12446  ret void
12447}
12448
12449define amdgpu_kernel void @flat_agent_one_as_release_monotonic_ret_cmpxchg(
12450; GFX7-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
12451; GFX7:       ; %bb.0: ; %entry
12452; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12453; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12454; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12455; GFX7-NEXT:    s_add_u32 s4, s0, 16
12456; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12457; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12458; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12459; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12460; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12461; GFX7-NEXT:    s_waitcnt vmcnt(0)
12462; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12463; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12464; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12465; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12466; GFX7-NEXT:    flat_store_dword v[0:1], v2
12467; GFX7-NEXT:    s_endpgm
12468;
12469; GFX10-WGP-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
12470; GFX10-WGP:       ; %bb.0: ; %entry
12471; GFX10-WGP-NEXT:    s_clause 0x1
12472; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12473; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12474; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12475; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12476; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12477; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12478; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12479; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12480; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12481; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12482; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12483; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12484; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12485; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12486; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12487; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12488; GFX10-WGP-NEXT:    s_endpgm
12489;
12490; GFX10-CU-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
12491; GFX10-CU:       ; %bb.0: ; %entry
12492; GFX10-CU-NEXT:    s_clause 0x1
12493; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12494; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12495; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12496; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12497; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12498; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12499; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12500; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12501; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12502; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12503; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12504; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12505; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12506; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12507; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12508; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12509; GFX10-CU-NEXT:    s_endpgm
12510;
12511; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
12512; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12513; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12514; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12515; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12516; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12517; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12518; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12519; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12520; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12521; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12522; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12523; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12524; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12525; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12526; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12527; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12528; SKIP-CACHE-INV-NEXT:    s_endpgm
12529;
12530; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
12531; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12532; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12533; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12534; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12535; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12536; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12537; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12538; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12539; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12540; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12541; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12542;
12543; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
12544; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12545; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12546; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12547; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12548; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12549; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12550; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12551; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12552; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12553; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12554; GFX90A-TGSPLIT-NEXT:    s_endpgm
12555;
12556; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
12557; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12558; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12559; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12560; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12561; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12562; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12563; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
12564; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12565; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12566; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12567; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12568; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12569;
12570; GFX940-TGSPLIT-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
12571; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12572; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12573; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12574; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12575; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12576; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12577; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
12578; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12579; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12580; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12581; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12582; GFX940-TGSPLIT-NEXT:    s_endpgm
12583;
12584; GFX11-WGP-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
12585; GFX11-WGP:       ; %bb.0: ; %entry
12586; GFX11-WGP-NEXT:    s_clause 0x1
12587; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12588; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12589; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12590; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12591; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12592; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12593; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12594; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12595; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12596; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12597; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12598; GFX11-WGP-NEXT:    s_endpgm
12599;
12600; GFX11-CU-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
12601; GFX11-CU:       ; %bb.0: ; %entry
12602; GFX11-CU-NEXT:    s_clause 0x1
12603; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12604; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12605; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12606; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12607; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12608; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
12609; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12610; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12611; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12612; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12613; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12614; GFX11-CU-NEXT:    s_endpgm
12615    i32* %out, i32 %in, i32 %old) {
12616entry:
12617  %gep = getelementptr i32, i32* %out, i32 4
12618  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release monotonic
12619  %val0 = extractvalue { i32, i1 } %val, 0
12620  store i32 %val0, i32* %out, align 4
12621  ret void
12622}
12623
12624define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg(
12625; GFX7-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
12626; GFX7:       ; %bb.0: ; %entry
12627; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12628; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12629; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12630; GFX7-NEXT:    s_add_u32 s4, s0, 16
12631; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12632; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12633; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12634; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12635; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12636; GFX7-NEXT:    s_waitcnt vmcnt(0)
12637; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12638; GFX7-NEXT:    s_waitcnt vmcnt(0)
12639; GFX7-NEXT:    buffer_wbinvl1_vol
12640; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12641; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12642; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12643; GFX7-NEXT:    flat_store_dword v[0:1], v2
12644; GFX7-NEXT:    s_endpgm
12645;
12646; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
12647; GFX10-WGP:       ; %bb.0: ; %entry
12648; GFX10-WGP-NEXT:    s_clause 0x1
12649; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12650; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12651; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12652; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12653; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12654; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12655; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12656; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12657; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12658; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12659; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12660; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12661; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12662; GFX10-WGP-NEXT:    buffer_gl0_inv
12663; GFX10-WGP-NEXT:    buffer_gl1_inv
12664; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12665; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12666; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12667; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12668; GFX10-WGP-NEXT:    s_endpgm
12669;
12670; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
12671; GFX10-CU:       ; %bb.0: ; %entry
12672; GFX10-CU-NEXT:    s_clause 0x1
12673; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12674; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12675; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12676; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12677; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12678; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12679; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12680; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12681; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12682; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12683; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12684; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12685; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12686; GFX10-CU-NEXT:    buffer_gl0_inv
12687; GFX10-CU-NEXT:    buffer_gl1_inv
12688; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12689; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12690; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12691; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12692; GFX10-CU-NEXT:    s_endpgm
12693;
12694; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
12695; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12696; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12697; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12698; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12699; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12700; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12701; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12702; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12703; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12704; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12705; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12706; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12707; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12708; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12709; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12710; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12711; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12712; SKIP-CACHE-INV-NEXT:    s_endpgm
12713;
12714; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
12715; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12716; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12717; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12718; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12719; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12720; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12721; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12722; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12723; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12724; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
12725; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12726; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12727; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12728;
12729; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
12730; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12731; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12732; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12733; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12734; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12735; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12736; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12737; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12738; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12739; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
12740; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12741; GFX90A-TGSPLIT-NEXT:    s_endpgm
12742;
12743; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
12744; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12745; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12746; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12747; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12748; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12749; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12750; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
12751; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12752; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12753; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12754; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
12755; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12756; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12757; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12758;
12759; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
12760; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12761; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12762; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12763; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12764; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12765; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12766; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
12767; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12768; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12769; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12770; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
12771; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12772; GFX940-TGSPLIT-NEXT:    s_endpgm
12773;
12774; GFX11-WGP-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
12775; GFX11-WGP:       ; %bb.0: ; %entry
12776; GFX11-WGP-NEXT:    s_clause 0x1
12777; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12778; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12779; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12780; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12781; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12782; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12783; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12784; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12785; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12786; GFX11-WGP-NEXT:    buffer_gl0_inv
12787; GFX11-WGP-NEXT:    buffer_gl1_inv
12788; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12789; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12790; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12791; GFX11-WGP-NEXT:    s_endpgm
12792;
12793; GFX11-CU-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
12794; GFX11-CU:       ; %bb.0: ; %entry
12795; GFX11-CU-NEXT:    s_clause 0x1
12796; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12797; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12798; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12799; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12800; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12801; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
12802; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12803; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12804; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
12805; GFX11-CU-NEXT:    buffer_gl0_inv
12806; GFX11-CU-NEXT:    buffer_gl1_inv
12807; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12808; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12809; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12810; GFX11-CU-NEXT:    s_endpgm
12811    i32* %out, i32 %in, i32 %old) {
12812entry:
12813  %gep = getelementptr i32, i32* %out, i32 4
12814  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel monotonic
12815  %val0 = extractvalue { i32, i1 } %val, 0
12816  store i32 %val0, i32* %out, align 4
12817  ret void
12818}
12819
12820define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg(
12821; GFX7-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
12822; GFX7:       ; %bb.0: ; %entry
12823; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12824; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12825; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12826; GFX7-NEXT:    s_add_u32 s4, s0, 16
12827; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12828; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12829; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12830; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12831; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12832; GFX7-NEXT:    s_waitcnt vmcnt(0)
12833; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12834; GFX7-NEXT:    s_waitcnt vmcnt(0)
12835; GFX7-NEXT:    buffer_wbinvl1_vol
12836; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12837; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12838; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12839; GFX7-NEXT:    flat_store_dword v[0:1], v2
12840; GFX7-NEXT:    s_endpgm
12841;
12842; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
12843; GFX10-WGP:       ; %bb.0: ; %entry
12844; GFX10-WGP-NEXT:    s_clause 0x1
12845; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12846; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12847; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12848; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12849; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12850; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12851; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12852; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12853; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12854; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12855; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12856; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12857; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12858; GFX10-WGP-NEXT:    buffer_gl0_inv
12859; GFX10-WGP-NEXT:    buffer_gl1_inv
12860; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12861; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12862; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12863; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12864; GFX10-WGP-NEXT:    s_endpgm
12865;
12866; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
12867; GFX10-CU:       ; %bb.0: ; %entry
12868; GFX10-CU-NEXT:    s_clause 0x1
12869; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12870; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12871; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12872; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12873; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12874; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12875; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12876; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12877; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12878; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12879; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12880; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12881; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12882; GFX10-CU-NEXT:    buffer_gl0_inv
12883; GFX10-CU-NEXT:    buffer_gl1_inv
12884; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12885; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12886; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12887; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12888; GFX10-CU-NEXT:    s_endpgm
12889;
12890; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
12891; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12892; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12893; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12894; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12895; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12896; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12897; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12898; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12899; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12900; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12901; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12902; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12903; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12904; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12905; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12906; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12907; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12908; SKIP-CACHE-INV-NEXT:    s_endpgm
12909;
12910; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
12911; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12912; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12913; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12914; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12915; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12916; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12917; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12918; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12919; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12920; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
12921; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12922; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12923; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12924;
12925; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
12926; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12927; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12928; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12929; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12930; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12931; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12932; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12933; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12934; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12935; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
12936; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12937; GFX90A-TGSPLIT-NEXT:    s_endpgm
12938;
12939; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
12940; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12941; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12942; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12943; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12944; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12945; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12946; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
12947; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12948; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12949; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12950; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
12951; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12952; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12953; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12954;
12955; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
12956; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12957; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12958; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12959; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12960; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12961; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12962; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
12963; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12964; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12965; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12966; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
12967; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12968; GFX940-TGSPLIT-NEXT:    s_endpgm
12969;
12970; GFX11-WGP-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
12971; GFX11-WGP:       ; %bb.0: ; %entry
12972; GFX11-WGP-NEXT:    s_clause 0x1
12973; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12974; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12975; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12976; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12977; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12978; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12979; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12980; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12981; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12982; GFX11-WGP-NEXT:    buffer_gl0_inv
12983; GFX11-WGP-NEXT:    buffer_gl1_inv
12984; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12985; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12986; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
12987; GFX11-WGP-NEXT:    s_endpgm
12988;
12989; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
12990; GFX11-CU:       ; %bb.0: ; %entry
12991; GFX11-CU-NEXT:    s_clause 0x1
12992; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12993; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12994; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12995; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
12996; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
12997; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
12998; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12999; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13000; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13001; GFX11-CU-NEXT:    buffer_gl0_inv
13002; GFX11-CU-NEXT:    buffer_gl1_inv
13003; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13004; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
13005; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
13006; GFX11-CU-NEXT:    s_endpgm
13007    i32* %out, i32 %in, i32 %old) {
13008entry:
13009  %gep = getelementptr i32, i32* %out, i32 4
13010  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst monotonic
13011  %val0 = extractvalue { i32, i1 } %val, 0
13012  store i32 %val0, i32* %out, align 4
13013  ret void
13014}
13015
13016define amdgpu_kernel void @flat_agent_one_as_monotonic_acquire_ret_cmpxchg(
13017; GFX7-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
13018; GFX7:       ; %bb.0: ; %entry
13019; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13020; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
13021; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13022; GFX7-NEXT:    s_add_u32 s4, s0, 16
13023; GFX7-NEXT:    s_addc_u32 s5, s1, 0
13024; GFX7-NEXT:    v_mov_b32_e32 v0, s4
13025; GFX7-NEXT:    v_mov_b32_e32 v2, s2
13026; GFX7-NEXT:    v_mov_b32_e32 v1, s5
13027; GFX7-NEXT:    v_mov_b32_e32 v3, s3
13028; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13029; GFX7-NEXT:    s_waitcnt vmcnt(0)
13030; GFX7-NEXT:    buffer_wbinvl1_vol
13031; GFX7-NEXT:    v_mov_b32_e32 v0, s0
13032; GFX7-NEXT:    v_mov_b32_e32 v1, s1
13033; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13034; GFX7-NEXT:    flat_store_dword v[0:1], v2
13035; GFX7-NEXT:    s_endpgm
13036;
13037; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
13038; GFX10-WGP:       ; %bb.0: ; %entry
13039; GFX10-WGP-NEXT:    s_clause 0x1
13040; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13041; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13042; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13043; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
13044; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
13045; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
13046; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
13047; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
13048; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
13049; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13050; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13051; GFX10-WGP-NEXT:    buffer_gl0_inv
13052; GFX10-WGP-NEXT:    buffer_gl1_inv
13053; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
13054; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
13055; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13056; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
13057; GFX10-WGP-NEXT:    s_endpgm
13058;
13059; GFX10-CU-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
13060; GFX10-CU:       ; %bb.0: ; %entry
13061; GFX10-CU-NEXT:    s_clause 0x1
13062; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13063; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13064; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13065; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
13066; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
13067; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
13068; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
13069; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
13070; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
13071; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13072; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13073; GFX10-CU-NEXT:    buffer_gl0_inv
13074; GFX10-CU-NEXT:    buffer_gl1_inv
13075; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
13076; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
13077; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13078; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
13079; GFX10-CU-NEXT:    s_endpgm
13080;
13081; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
13082; SKIP-CACHE-INV:       ; %bb.0: ; %entry
13083; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13084; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
13085; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13086; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
13087; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
13088; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
13089; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
13090; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
13091; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
13092; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13093; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
13094; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
13095; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
13096; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13097; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
13098; SKIP-CACHE-INV-NEXT:    s_endpgm
13099;
13100; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
13101; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
13102; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13103; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13104; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13105; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13106; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13107; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13108; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13109; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
13110; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13111; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13112; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
13113;
13114; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
13115; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
13116; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13117; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13118; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13119; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13120; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13121; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13122; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13123; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
13124; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13125; GFX90A-TGSPLIT-NEXT:    s_endpgm
13126;
13127; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
13128; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
13129; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13130; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13131; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13132; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13133; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13134; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
13135; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13136; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
13137; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13138; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13139; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
13140;
13141; GFX940-TGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
13142; GFX940-TGSPLIT:       ; %bb.0: ; %entry
13143; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13144; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13145; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13146; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13147; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13148; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
13149; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13150; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
13151; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13152; GFX940-TGSPLIT-NEXT:    s_endpgm
13153;
13154; GFX11-WGP-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
13155; GFX11-WGP:       ; %bb.0: ; %entry
13156; GFX11-WGP-NEXT:    s_clause 0x1
13157; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13158; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13159; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13160; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
13161; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
13162; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13163; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13164; GFX11-WGP-NEXT:    buffer_gl0_inv
13165; GFX11-WGP-NEXT:    buffer_gl1_inv
13166; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13167; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
13168; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
13169; GFX11-WGP-NEXT:    s_endpgm
13170;
13171; GFX11-CU-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
13172; GFX11-CU:       ; %bb.0: ; %entry
13173; GFX11-CU-NEXT:    s_clause 0x1
13174; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13175; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13176; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13177; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
13178; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
13179; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13180; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13181; GFX11-CU-NEXT:    buffer_gl0_inv
13182; GFX11-CU-NEXT:    buffer_gl1_inv
13183; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13184; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
13185; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
13186; GFX11-CU-NEXT:    s_endpgm
13187    i32* %out, i32 %in, i32 %old) {
13188entry:
13189  %gep = getelementptr i32, i32* %out, i32 4
13190  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") monotonic acquire
13191  %val0 = extractvalue { i32, i1 } %val, 0
13192  store i32 %val0, i32* %out, align 4
13193  ret void
13194}
13195
13196define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_ret_cmpxchg(
13197; GFX7-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
13198; GFX7:       ; %bb.0: ; %entry
13199; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13200; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
13201; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13202; GFX7-NEXT:    s_add_u32 s4, s0, 16
13203; GFX7-NEXT:    s_addc_u32 s5, s1, 0
13204; GFX7-NEXT:    v_mov_b32_e32 v0, s4
13205; GFX7-NEXT:    v_mov_b32_e32 v2, s2
13206; GFX7-NEXT:    v_mov_b32_e32 v1, s5
13207; GFX7-NEXT:    v_mov_b32_e32 v3, s3
13208; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13209; GFX7-NEXT:    s_waitcnt vmcnt(0)
13210; GFX7-NEXT:    buffer_wbinvl1_vol
13211; GFX7-NEXT:    v_mov_b32_e32 v0, s0
13212; GFX7-NEXT:    v_mov_b32_e32 v1, s1
13213; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13214; GFX7-NEXT:    flat_store_dword v[0:1], v2
13215; GFX7-NEXT:    s_endpgm
13216;
13217; GFX10-WGP-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
13218; GFX10-WGP:       ; %bb.0: ; %entry
13219; GFX10-WGP-NEXT:    s_clause 0x1
13220; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13221; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13222; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13223; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
13224; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
13225; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
13226; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
13227; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
13228; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
13229; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13230; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13231; GFX10-WGP-NEXT:    buffer_gl0_inv
13232; GFX10-WGP-NEXT:    buffer_gl1_inv
13233; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
13234; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
13235; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13236; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
13237; GFX10-WGP-NEXT:    s_endpgm
13238;
13239; GFX10-CU-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
13240; GFX10-CU:       ; %bb.0: ; %entry
13241; GFX10-CU-NEXT:    s_clause 0x1
13242; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13243; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13244; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13245; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
13246; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
13247; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
13248; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
13249; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
13250; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
13251; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13252; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13253; GFX10-CU-NEXT:    buffer_gl0_inv
13254; GFX10-CU-NEXT:    buffer_gl1_inv
13255; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
13256; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
13257; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13258; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
13259; GFX10-CU-NEXT:    s_endpgm
13260;
13261; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
13262; SKIP-CACHE-INV:       ; %bb.0: ; %entry
13263; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13264; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
13265; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13266; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
13267; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
13268; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
13269; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
13270; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
13271; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
13272; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13273; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
13274; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
13275; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
13276; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13277; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
13278; SKIP-CACHE-INV-NEXT:    s_endpgm
13279;
13280; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
13281; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
13282; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13283; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13284; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13285; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13286; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13287; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13288; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13289; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
13290; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13291; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13292; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
13293;
13294; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
13295; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
13296; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13297; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13298; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13299; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13300; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13301; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13302; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13303; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
13304; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13305; GFX90A-TGSPLIT-NEXT:    s_endpgm
13306;
13307; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
13308; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
13309; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13310; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13311; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13312; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13313; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13314; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
13315; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13316; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
13317; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13318; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13319; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
13320;
13321; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
13322; GFX940-TGSPLIT:       ; %bb.0: ; %entry
13323; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13324; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13325; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13326; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13327; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13328; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
13329; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13330; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
13331; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13332; GFX940-TGSPLIT-NEXT:    s_endpgm
13333;
13334; GFX11-WGP-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
13335; GFX11-WGP:       ; %bb.0: ; %entry
13336; GFX11-WGP-NEXT:    s_clause 0x1
13337; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13338; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13339; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13340; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
13341; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
13342; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13343; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13344; GFX11-WGP-NEXT:    buffer_gl0_inv
13345; GFX11-WGP-NEXT:    buffer_gl1_inv
13346; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13347; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
13348; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
13349; GFX11-WGP-NEXT:    s_endpgm
13350;
13351; GFX11-CU-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
13352; GFX11-CU:       ; %bb.0: ; %entry
13353; GFX11-CU-NEXT:    s_clause 0x1
13354; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13355; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13356; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13357; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
13358; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
13359; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13360; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13361; GFX11-CU-NEXT:    buffer_gl0_inv
13362; GFX11-CU-NEXT:    buffer_gl1_inv
13363; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13364; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
13365; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
13366; GFX11-CU-NEXT:    s_endpgm
13367    i32* %out, i32 %in, i32 %old) {
13368entry:
13369  %gep = getelementptr i32, i32* %out, i32 4
13370  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire acquire
13371  %val0 = extractvalue { i32, i1 } %val, 0
13372  store i32 %val0, i32* %out, align 4
13373  ret void
13374}
13375
13376define amdgpu_kernel void @flat_agent_one_as_release_acquire_ret_cmpxchg(
13377; GFX7-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
13378; GFX7:       ; %bb.0: ; %entry
13379; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13380; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
13381; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13382; GFX7-NEXT:    s_add_u32 s4, s0, 16
13383; GFX7-NEXT:    s_addc_u32 s5, s1, 0
13384; GFX7-NEXT:    v_mov_b32_e32 v0, s4
13385; GFX7-NEXT:    v_mov_b32_e32 v2, s2
13386; GFX7-NEXT:    v_mov_b32_e32 v1, s5
13387; GFX7-NEXT:    v_mov_b32_e32 v3, s3
13388; GFX7-NEXT:    s_waitcnt vmcnt(0)
13389; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13390; GFX7-NEXT:    s_waitcnt vmcnt(0)
13391; GFX7-NEXT:    buffer_wbinvl1_vol
13392; GFX7-NEXT:    v_mov_b32_e32 v0, s0
13393; GFX7-NEXT:    v_mov_b32_e32 v1, s1
13394; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13395; GFX7-NEXT:    flat_store_dword v[0:1], v2
13396; GFX7-NEXT:    s_endpgm
13397;
13398; GFX10-WGP-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
13399; GFX10-WGP:       ; %bb.0: ; %entry
13400; GFX10-WGP-NEXT:    s_clause 0x1
13401; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13402; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13403; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13404; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
13405; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
13406; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
13407; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
13408; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
13409; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
13410; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13411; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
13412; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13413; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13414; GFX10-WGP-NEXT:    buffer_gl0_inv
13415; GFX10-WGP-NEXT:    buffer_gl1_inv
13416; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
13417; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
13418; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13419; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
13420; GFX10-WGP-NEXT:    s_endpgm
13421;
13422; GFX10-CU-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
13423; GFX10-CU:       ; %bb.0: ; %entry
13424; GFX10-CU-NEXT:    s_clause 0x1
13425; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13426; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13427; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13428; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
13429; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
13430; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
13431; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
13432; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
13433; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
13434; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13435; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
13436; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13437; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13438; GFX10-CU-NEXT:    buffer_gl0_inv
13439; GFX10-CU-NEXT:    buffer_gl1_inv
13440; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
13441; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
13442; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13443; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
13444; GFX10-CU-NEXT:    s_endpgm
13445;
13446; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
13447; SKIP-CACHE-INV:       ; %bb.0: ; %entry
13448; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13449; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
13450; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13451; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
13452; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
13453; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
13454; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
13455; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
13456; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
13457; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
13458; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13459; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
13460; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
13461; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
13462; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13463; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
13464; SKIP-CACHE-INV-NEXT:    s_endpgm
13465;
13466; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
13467; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
13468; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13469; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13470; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13471; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13472; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13473; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13474; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13475; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13476; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
13477; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13478; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13479; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
13480;
13481; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
13482; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
13483; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13484; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13485; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13486; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13487; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13488; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13489; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13490; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13491; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
13492; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13493; GFX90A-TGSPLIT-NEXT:    s_endpgm
13494;
13495; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
13496; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
13497; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13498; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13499; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13500; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13501; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13502; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
13503; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13504; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
13505; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13506; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
13507; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13508; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13509; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
13510;
13511; GFX940-TGSPLIT-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
13512; GFX940-TGSPLIT:       ; %bb.0: ; %entry
13513; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13514; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13515; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13516; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13517; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13518; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
13519; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13520; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
13521; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13522; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
13523; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13524; GFX940-TGSPLIT-NEXT:    s_endpgm
13525;
13526; GFX11-WGP-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
13527; GFX11-WGP:       ; %bb.0: ; %entry
13528; GFX11-WGP-NEXT:    s_clause 0x1
13529; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13530; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13531; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13532; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
13533; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
13534; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13535; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
13536; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13537; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13538; GFX11-WGP-NEXT:    buffer_gl0_inv
13539; GFX11-WGP-NEXT:    buffer_gl1_inv
13540; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13541; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
13542; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
13543; GFX11-WGP-NEXT:    s_endpgm
13544;
13545; GFX11-CU-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
13546; GFX11-CU:       ; %bb.0: ; %entry
13547; GFX11-CU-NEXT:    s_clause 0x1
13548; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13549; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13550; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13551; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
13552; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
13553; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13554; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
13555; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13556; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13557; GFX11-CU-NEXT:    buffer_gl0_inv
13558; GFX11-CU-NEXT:    buffer_gl1_inv
13559; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13560; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
13561; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
13562; GFX11-CU-NEXT:    s_endpgm
13563    i32* %out, i32 %in, i32 %old) {
13564entry:
13565  %gep = getelementptr i32, i32* %out, i32 4
13566  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release acquire
13567  %val0 = extractvalue { i32, i1 } %val, 0
13568  store i32 %val0, i32* %out, align 4
13569  ret void
13570}
13571
13572define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_ret_cmpxchg(
13573; GFX7-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
13574; GFX7:       ; %bb.0: ; %entry
13575; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13576; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
13577; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13578; GFX7-NEXT:    s_add_u32 s4, s0, 16
13579; GFX7-NEXT:    s_addc_u32 s5, s1, 0
13580; GFX7-NEXT:    v_mov_b32_e32 v0, s4
13581; GFX7-NEXT:    v_mov_b32_e32 v2, s2
13582; GFX7-NEXT:    v_mov_b32_e32 v1, s5
13583; GFX7-NEXT:    v_mov_b32_e32 v3, s3
13584; GFX7-NEXT:    s_waitcnt vmcnt(0)
13585; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13586; GFX7-NEXT:    s_waitcnt vmcnt(0)
13587; GFX7-NEXT:    buffer_wbinvl1_vol
13588; GFX7-NEXT:    v_mov_b32_e32 v0, s0
13589; GFX7-NEXT:    v_mov_b32_e32 v1, s1
13590; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13591; GFX7-NEXT:    flat_store_dword v[0:1], v2
13592; GFX7-NEXT:    s_endpgm
13593;
13594; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
13595; GFX10-WGP:       ; %bb.0: ; %entry
13596; GFX10-WGP-NEXT:    s_clause 0x1
13597; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13598; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13599; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13600; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
13601; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
13602; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
13603; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
13604; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
13605; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
13606; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13607; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
13608; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13609; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13610; GFX10-WGP-NEXT:    buffer_gl0_inv
13611; GFX10-WGP-NEXT:    buffer_gl1_inv
13612; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
13613; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
13614; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13615; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
13616; GFX10-WGP-NEXT:    s_endpgm
13617;
13618; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
13619; GFX10-CU:       ; %bb.0: ; %entry
13620; GFX10-CU-NEXT:    s_clause 0x1
13621; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13622; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13623; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13624; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
13625; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
13626; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
13627; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
13628; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
13629; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
13630; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13631; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
13632; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13633; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13634; GFX10-CU-NEXT:    buffer_gl0_inv
13635; GFX10-CU-NEXT:    buffer_gl1_inv
13636; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
13637; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
13638; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13639; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
13640; GFX10-CU-NEXT:    s_endpgm
13641;
13642; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
13643; SKIP-CACHE-INV:       ; %bb.0: ; %entry
13644; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13645; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
13646; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13647; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
13648; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
13649; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
13650; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
13651; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
13652; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
13653; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
13654; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13655; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
13656; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
13657; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
13658; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13659; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
13660; SKIP-CACHE-INV-NEXT:    s_endpgm
13661;
13662; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
13663; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
13664; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13665; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13666; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13667; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13668; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13669; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13670; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13671; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13672; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
13673; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13674; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13675; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
13676;
13677; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
13678; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
13679; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13680; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13681; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13682; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13683; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13684; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13685; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13686; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13687; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
13688; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13689; GFX90A-TGSPLIT-NEXT:    s_endpgm
13690;
13691; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
13692; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
13693; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13694; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13695; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13696; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13697; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13698; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
13699; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13700; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
13701; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13702; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
13703; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13704; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13705; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
13706;
13707; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
13708; GFX940-TGSPLIT:       ; %bb.0: ; %entry
13709; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13710; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13711; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13712; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13713; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13714; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
13715; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13716; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
13717; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13718; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
13719; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13720; GFX940-TGSPLIT-NEXT:    s_endpgm
13721;
13722; GFX11-WGP-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
13723; GFX11-WGP:       ; %bb.0: ; %entry
13724; GFX11-WGP-NEXT:    s_clause 0x1
13725; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13726; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13727; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13728; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
13729; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
13730; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13731; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
13732; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13733; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13734; GFX11-WGP-NEXT:    buffer_gl0_inv
13735; GFX11-WGP-NEXT:    buffer_gl1_inv
13736; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13737; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
13738; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
13739; GFX11-WGP-NEXT:    s_endpgm
13740;
13741; GFX11-CU-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
13742; GFX11-CU:       ; %bb.0: ; %entry
13743; GFX11-CU-NEXT:    s_clause 0x1
13744; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13745; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13746; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13747; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
13748; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
13749; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13750; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
13751; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13752; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13753; GFX11-CU-NEXT:    buffer_gl0_inv
13754; GFX11-CU-NEXT:    buffer_gl1_inv
13755; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13756; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
13757; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
13758; GFX11-CU-NEXT:    s_endpgm
13759    i32* %out, i32 %in, i32 %old) {
13760entry:
13761  %gep = getelementptr i32, i32* %out, i32 4
13762  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel acquire
13763  %val0 = extractvalue { i32, i1 } %val, 0
13764  store i32 %val0, i32* %out, align 4
13765  ret void
13766}
13767
13768define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_ret_cmpxchg(
13769; GFX7-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
13770; GFX7:       ; %bb.0: ; %entry
13771; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13772; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
13773; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13774; GFX7-NEXT:    s_add_u32 s4, s0, 16
13775; GFX7-NEXT:    s_addc_u32 s5, s1, 0
13776; GFX7-NEXT:    v_mov_b32_e32 v0, s4
13777; GFX7-NEXT:    v_mov_b32_e32 v2, s2
13778; GFX7-NEXT:    v_mov_b32_e32 v1, s5
13779; GFX7-NEXT:    v_mov_b32_e32 v3, s3
13780; GFX7-NEXT:    s_waitcnt vmcnt(0)
13781; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13782; GFX7-NEXT:    s_waitcnt vmcnt(0)
13783; GFX7-NEXT:    buffer_wbinvl1_vol
13784; GFX7-NEXT:    v_mov_b32_e32 v0, s0
13785; GFX7-NEXT:    v_mov_b32_e32 v1, s1
13786; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13787; GFX7-NEXT:    flat_store_dword v[0:1], v2
13788; GFX7-NEXT:    s_endpgm
13789;
13790; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
13791; GFX10-WGP:       ; %bb.0: ; %entry
13792; GFX10-WGP-NEXT:    s_clause 0x1
13793; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13794; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13795; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13796; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
13797; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
13798; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
13799; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
13800; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
13801; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
13802; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13803; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
13804; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13805; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13806; GFX10-WGP-NEXT:    buffer_gl0_inv
13807; GFX10-WGP-NEXT:    buffer_gl1_inv
13808; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
13809; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
13810; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13811; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
13812; GFX10-WGP-NEXT:    s_endpgm
13813;
13814; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
13815; GFX10-CU:       ; %bb.0: ; %entry
13816; GFX10-CU-NEXT:    s_clause 0x1
13817; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13818; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13819; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13820; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
13821; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
13822; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
13823; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
13824; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
13825; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
13826; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13827; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
13828; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13829; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13830; GFX10-CU-NEXT:    buffer_gl0_inv
13831; GFX10-CU-NEXT:    buffer_gl1_inv
13832; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
13833; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
13834; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13835; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
13836; GFX10-CU-NEXT:    s_endpgm
13837;
13838; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
13839; SKIP-CACHE-INV:       ; %bb.0: ; %entry
13840; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13841; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
13842; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13843; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
13844; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
13845; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
13846; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
13847; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
13848; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
13849; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
13850; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13851; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
13852; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
13853; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
13854; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13855; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
13856; SKIP-CACHE-INV-NEXT:    s_endpgm
13857;
13858; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
13859; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
13860; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13861; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13862; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13863; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13864; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13865; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13866; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13867; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13868; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
13869; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13870; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13871; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
13872;
13873; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
13874; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
13875; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13876; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13877; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13878; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13879; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13880; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13881; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13882; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13883; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
13884; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13885; GFX90A-TGSPLIT-NEXT:    s_endpgm
13886;
13887; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
13888; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
13889; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13890; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13891; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13892; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13893; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13894; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
13895; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13896; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
13897; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13898; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
13899; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13900; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13901; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
13902;
13903; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
13904; GFX940-TGSPLIT:       ; %bb.0: ; %entry
13905; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13906; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13907; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13908; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13909; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13910; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
13911; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13912; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
13913; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13914; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
13915; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13916; GFX940-TGSPLIT-NEXT:    s_endpgm
13917;
13918; GFX11-WGP-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
13919; GFX11-WGP:       ; %bb.0: ; %entry
13920; GFX11-WGP-NEXT:    s_clause 0x1
13921; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13922; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13923; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13924; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
13925; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
13926; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13927; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
13928; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13929; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13930; GFX11-WGP-NEXT:    buffer_gl0_inv
13931; GFX11-WGP-NEXT:    buffer_gl1_inv
13932; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13933; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
13934; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
13935; GFX11-WGP-NEXT:    s_endpgm
13936;
13937; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
13938; GFX11-CU:       ; %bb.0: ; %entry
13939; GFX11-CU-NEXT:    s_clause 0x1
13940; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13941; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13942; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13943; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
13944; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
13945; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13946; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
13947; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13948; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13949; GFX11-CU-NEXT:    buffer_gl0_inv
13950; GFX11-CU-NEXT:    buffer_gl1_inv
13951; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13952; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
13953; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
13954; GFX11-CU-NEXT:    s_endpgm
13955    i32* %out, i32 %in, i32 %old) {
13956entry:
13957  %gep = getelementptr i32, i32* %out, i32 4
13958  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst acquire
13959  %val0 = extractvalue { i32, i1 } %val, 0
13960  store i32 %val0, i32* %out, align 4
13961  ret void
13962}
13963
13964define amdgpu_kernel void @flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg(
13965; GFX7-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
13966; GFX7:       ; %bb.0: ; %entry
13967; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13968; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
13969; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13970; GFX7-NEXT:    s_add_u32 s4, s0, 16
13971; GFX7-NEXT:    s_addc_u32 s5, s1, 0
13972; GFX7-NEXT:    v_mov_b32_e32 v0, s4
13973; GFX7-NEXT:    v_mov_b32_e32 v2, s2
13974; GFX7-NEXT:    v_mov_b32_e32 v1, s5
13975; GFX7-NEXT:    v_mov_b32_e32 v3, s3
13976; GFX7-NEXT:    s_waitcnt vmcnt(0)
13977; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13978; GFX7-NEXT:    s_waitcnt vmcnt(0)
13979; GFX7-NEXT:    buffer_wbinvl1_vol
13980; GFX7-NEXT:    v_mov_b32_e32 v0, s0
13981; GFX7-NEXT:    v_mov_b32_e32 v1, s1
13982; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13983; GFX7-NEXT:    flat_store_dword v[0:1], v2
13984; GFX7-NEXT:    s_endpgm
13985;
13986; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
13987; GFX10-WGP:       ; %bb.0: ; %entry
13988; GFX10-WGP-NEXT:    s_clause 0x1
13989; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13990; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13991; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13992; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
13993; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
13994; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
13995; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
13996; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
13997; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
13998; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13999; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14000; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14001; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14002; GFX10-WGP-NEXT:    buffer_gl0_inv
14003; GFX10-WGP-NEXT:    buffer_gl1_inv
14004; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
14005; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
14006; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14007; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
14008; GFX10-WGP-NEXT:    s_endpgm
14009;
14010; GFX10-CU-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
14011; GFX10-CU:       ; %bb.0: ; %entry
14012; GFX10-CU-NEXT:    s_clause 0x1
14013; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14014; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14015; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14016; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
14017; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
14018; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
14019; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
14020; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
14021; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
14022; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14023; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14024; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14025; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14026; GFX10-CU-NEXT:    buffer_gl0_inv
14027; GFX10-CU-NEXT:    buffer_gl1_inv
14028; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
14029; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
14030; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14031; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
14032; GFX10-CU-NEXT:    s_endpgm
14033;
14034; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
14035; SKIP-CACHE-INV:       ; %bb.0: ; %entry
14036; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14037; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
14038; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14039; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
14040; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
14041; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
14042; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
14043; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
14044; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
14045; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14046; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14047; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14048; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
14049; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
14050; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14051; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
14052; SKIP-CACHE-INV-NEXT:    s_endpgm
14053;
14054; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
14055; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
14056; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14057; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14058; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14059; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14060; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14061; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14062; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14063; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14064; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
14065; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14066; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14067; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
14068;
14069; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
14070; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
14071; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14072; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14073; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14074; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14075; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14076; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14077; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14078; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14079; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
14080; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14081; GFX90A-TGSPLIT-NEXT:    s_endpgm
14082;
14083; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
14084; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
14085; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14086; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14087; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14088; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14089; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14090; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
14091; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14092; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
14093; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14094; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
14095; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14096; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14097; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
14098;
14099; GFX940-TGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
14100; GFX940-TGSPLIT:       ; %bb.0: ; %entry
14101; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14102; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14103; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14104; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14105; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14106; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
14107; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14108; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
14109; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14110; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
14111; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14112; GFX940-TGSPLIT-NEXT:    s_endpgm
14113;
14114; GFX11-WGP-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
14115; GFX11-WGP:       ; %bb.0: ; %entry
14116; GFX11-WGP-NEXT:    s_clause 0x1
14117; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14118; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14119; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14120; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
14121; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
14122; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14123; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14124; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14125; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14126; GFX11-WGP-NEXT:    buffer_gl0_inv
14127; GFX11-WGP-NEXT:    buffer_gl1_inv
14128; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14129; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
14130; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
14131; GFX11-WGP-NEXT:    s_endpgm
14132;
14133; GFX11-CU-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
14134; GFX11-CU:       ; %bb.0: ; %entry
14135; GFX11-CU-NEXT:    s_clause 0x1
14136; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14137; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14138; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14139; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
14140; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
14141; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14142; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14143; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14144; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14145; GFX11-CU-NEXT:    buffer_gl0_inv
14146; GFX11-CU-NEXT:    buffer_gl1_inv
14147; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14148; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
14149; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
14150; GFX11-CU-NEXT:    s_endpgm
14151    i32* %out, i32 %in, i32 %old) {
14152entry:
14153  %gep = getelementptr i32, i32* %out, i32 4
14154  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") monotonic seq_cst
14155  %val0 = extractvalue { i32, i1 } %val, 0
14156  store i32 %val0, i32* %out, align 4
14157  ret void
14158}
14159
14160define amdgpu_kernel void @flat_agent_one_as_acquire_seq_cst_ret_cmpxchg(
14161; GFX7-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
14162; GFX7:       ; %bb.0: ; %entry
14163; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14164; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
14165; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14166; GFX7-NEXT:    s_add_u32 s4, s0, 16
14167; GFX7-NEXT:    s_addc_u32 s5, s1, 0
14168; GFX7-NEXT:    v_mov_b32_e32 v0, s4
14169; GFX7-NEXT:    v_mov_b32_e32 v2, s2
14170; GFX7-NEXT:    v_mov_b32_e32 v1, s5
14171; GFX7-NEXT:    v_mov_b32_e32 v3, s3
14172; GFX7-NEXT:    s_waitcnt vmcnt(0)
14173; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14174; GFX7-NEXT:    s_waitcnt vmcnt(0)
14175; GFX7-NEXT:    buffer_wbinvl1_vol
14176; GFX7-NEXT:    v_mov_b32_e32 v0, s0
14177; GFX7-NEXT:    v_mov_b32_e32 v1, s1
14178; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14179; GFX7-NEXT:    flat_store_dword v[0:1], v2
14180; GFX7-NEXT:    s_endpgm
14181;
14182; GFX10-WGP-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
14183; GFX10-WGP:       ; %bb.0: ; %entry
14184; GFX10-WGP-NEXT:    s_clause 0x1
14185; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14186; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14187; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14188; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
14189; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
14190; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
14191; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
14192; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
14193; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
14194; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14195; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14196; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14197; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14198; GFX10-WGP-NEXT:    buffer_gl0_inv
14199; GFX10-WGP-NEXT:    buffer_gl1_inv
14200; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
14201; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
14202; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14203; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
14204; GFX10-WGP-NEXT:    s_endpgm
14205;
14206; GFX10-CU-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
14207; GFX10-CU:       ; %bb.0: ; %entry
14208; GFX10-CU-NEXT:    s_clause 0x1
14209; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14210; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14211; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14212; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
14213; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
14214; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
14215; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
14216; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
14217; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
14218; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14219; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14220; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14221; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14222; GFX10-CU-NEXT:    buffer_gl0_inv
14223; GFX10-CU-NEXT:    buffer_gl1_inv
14224; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
14225; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
14226; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14227; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
14228; GFX10-CU-NEXT:    s_endpgm
14229;
14230; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
14231; SKIP-CACHE-INV:       ; %bb.0: ; %entry
14232; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14233; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
14234; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14235; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
14236; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
14237; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
14238; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
14239; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
14240; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
14241; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14242; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14243; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14244; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
14245; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
14246; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14247; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
14248; SKIP-CACHE-INV-NEXT:    s_endpgm
14249;
14250; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
14251; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
14252; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14253; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14254; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14255; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14256; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14257; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14258; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14259; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14260; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
14261; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14262; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14263; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
14264;
14265; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
14266; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
14267; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14268; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14269; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14270; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14271; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14272; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14273; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14274; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14275; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
14276; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14277; GFX90A-TGSPLIT-NEXT:    s_endpgm
14278;
14279; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
14280; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
14281; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14282; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14283; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14284; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14285; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14286; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
14287; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14288; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
14289; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14290; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
14291; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14292; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14293; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
14294;
14295; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
14296; GFX940-TGSPLIT:       ; %bb.0: ; %entry
14297; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14298; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14299; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14300; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14301; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14302; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
14303; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14304; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
14305; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14306; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
14307; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14308; GFX940-TGSPLIT-NEXT:    s_endpgm
14309;
14310; GFX11-WGP-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
14311; GFX11-WGP:       ; %bb.0: ; %entry
14312; GFX11-WGP-NEXT:    s_clause 0x1
14313; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14314; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14315; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14316; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
14317; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
14318; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14319; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14320; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14321; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14322; GFX11-WGP-NEXT:    buffer_gl0_inv
14323; GFX11-WGP-NEXT:    buffer_gl1_inv
14324; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14325; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
14326; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
14327; GFX11-WGP-NEXT:    s_endpgm
14328;
14329; GFX11-CU-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
14330; GFX11-CU:       ; %bb.0: ; %entry
14331; GFX11-CU-NEXT:    s_clause 0x1
14332; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14333; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14334; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14335; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
14336; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
14337; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14338; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14339; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14340; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14341; GFX11-CU-NEXT:    buffer_gl0_inv
14342; GFX11-CU-NEXT:    buffer_gl1_inv
14343; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14344; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
14345; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
14346; GFX11-CU-NEXT:    s_endpgm
14347    i32* %out, i32 %in, i32 %old) {
14348entry:
14349  %gep = getelementptr i32, i32* %out, i32 4
14350  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire seq_cst
14351  %val0 = extractvalue { i32, i1 } %val, 0
14352  store i32 %val0, i32* %out, align 4
14353  ret void
14354}
14355
14356define amdgpu_kernel void @flat_agent_one_as_release_seq_cst_ret_cmpxchg(
14357; GFX7-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
14358; GFX7:       ; %bb.0: ; %entry
14359; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14360; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
14361; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14362; GFX7-NEXT:    s_add_u32 s4, s0, 16
14363; GFX7-NEXT:    s_addc_u32 s5, s1, 0
14364; GFX7-NEXT:    v_mov_b32_e32 v0, s4
14365; GFX7-NEXT:    v_mov_b32_e32 v2, s2
14366; GFX7-NEXT:    v_mov_b32_e32 v1, s5
14367; GFX7-NEXT:    v_mov_b32_e32 v3, s3
14368; GFX7-NEXT:    s_waitcnt vmcnt(0)
14369; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14370; GFX7-NEXT:    s_waitcnt vmcnt(0)
14371; GFX7-NEXT:    buffer_wbinvl1_vol
14372; GFX7-NEXT:    v_mov_b32_e32 v0, s0
14373; GFX7-NEXT:    v_mov_b32_e32 v1, s1
14374; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14375; GFX7-NEXT:    flat_store_dword v[0:1], v2
14376; GFX7-NEXT:    s_endpgm
14377;
14378; GFX10-WGP-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
14379; GFX10-WGP:       ; %bb.0: ; %entry
14380; GFX10-WGP-NEXT:    s_clause 0x1
14381; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14382; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14383; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14384; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
14385; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
14386; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
14387; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
14388; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
14389; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
14390; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14391; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14392; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14393; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14394; GFX10-WGP-NEXT:    buffer_gl0_inv
14395; GFX10-WGP-NEXT:    buffer_gl1_inv
14396; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
14397; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
14398; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14399; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
14400; GFX10-WGP-NEXT:    s_endpgm
14401;
14402; GFX10-CU-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
14403; GFX10-CU:       ; %bb.0: ; %entry
14404; GFX10-CU-NEXT:    s_clause 0x1
14405; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14406; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14407; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14408; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
14409; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
14410; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
14411; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
14412; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
14413; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
14414; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14415; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14416; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14417; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14418; GFX10-CU-NEXT:    buffer_gl0_inv
14419; GFX10-CU-NEXT:    buffer_gl1_inv
14420; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
14421; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
14422; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14423; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
14424; GFX10-CU-NEXT:    s_endpgm
14425;
14426; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
14427; SKIP-CACHE-INV:       ; %bb.0: ; %entry
14428; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14429; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
14430; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14431; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
14432; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
14433; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
14434; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
14435; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
14436; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
14437; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14438; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14439; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14440; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
14441; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
14442; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14443; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
14444; SKIP-CACHE-INV-NEXT:    s_endpgm
14445;
14446; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
14447; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
14448; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14449; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14450; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14451; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14452; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14453; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14454; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14455; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14456; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
14457; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14458; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14459; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
14460;
14461; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
14462; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
14463; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14464; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14465; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14466; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14467; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14468; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14469; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14470; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14471; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
14472; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14473; GFX90A-TGSPLIT-NEXT:    s_endpgm
14474;
14475; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
14476; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
14477; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14478; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14479; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14480; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14481; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14482; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
14483; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14484; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
14485; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14486; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
14487; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14488; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14489; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
14490;
14491; GFX940-TGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
14492; GFX940-TGSPLIT:       ; %bb.0: ; %entry
14493; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14494; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14495; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14496; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14497; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14498; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
14499; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14500; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
14501; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14502; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
14503; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14504; GFX940-TGSPLIT-NEXT:    s_endpgm
14505;
14506; GFX11-WGP-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
14507; GFX11-WGP:       ; %bb.0: ; %entry
14508; GFX11-WGP-NEXT:    s_clause 0x1
14509; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14510; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14511; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14512; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
14513; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
14514; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14515; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14516; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14517; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14518; GFX11-WGP-NEXT:    buffer_gl0_inv
14519; GFX11-WGP-NEXT:    buffer_gl1_inv
14520; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14521; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
14522; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
14523; GFX11-WGP-NEXT:    s_endpgm
14524;
14525; GFX11-CU-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
14526; GFX11-CU:       ; %bb.0: ; %entry
14527; GFX11-CU-NEXT:    s_clause 0x1
14528; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14529; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14530; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14531; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
14532; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
14533; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14534; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14535; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14536; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14537; GFX11-CU-NEXT:    buffer_gl0_inv
14538; GFX11-CU-NEXT:    buffer_gl1_inv
14539; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14540; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
14541; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
14542; GFX11-CU-NEXT:    s_endpgm
14543    i32* %out, i32 %in, i32 %old) {
14544entry:
14545  %gep = getelementptr i32, i32* %out, i32 4
14546  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release seq_cst
14547  %val0 = extractvalue { i32, i1 } %val, 0
14548  store i32 %val0, i32* %out, align 4
14549  ret void
14550}
14551
14552define amdgpu_kernel void @flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg(
14553; GFX7-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
14554; GFX7:       ; %bb.0: ; %entry
14555; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14556; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
14557; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14558; GFX7-NEXT:    s_add_u32 s4, s0, 16
14559; GFX7-NEXT:    s_addc_u32 s5, s1, 0
14560; GFX7-NEXT:    v_mov_b32_e32 v0, s4
14561; GFX7-NEXT:    v_mov_b32_e32 v2, s2
14562; GFX7-NEXT:    v_mov_b32_e32 v1, s5
14563; GFX7-NEXT:    v_mov_b32_e32 v3, s3
14564; GFX7-NEXT:    s_waitcnt vmcnt(0)
14565; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14566; GFX7-NEXT:    s_waitcnt vmcnt(0)
14567; GFX7-NEXT:    buffer_wbinvl1_vol
14568; GFX7-NEXT:    v_mov_b32_e32 v0, s0
14569; GFX7-NEXT:    v_mov_b32_e32 v1, s1
14570; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14571; GFX7-NEXT:    flat_store_dword v[0:1], v2
14572; GFX7-NEXT:    s_endpgm
14573;
14574; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
14575; GFX10-WGP:       ; %bb.0: ; %entry
14576; GFX10-WGP-NEXT:    s_clause 0x1
14577; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14578; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14579; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14580; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
14581; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
14582; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
14583; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
14584; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
14585; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
14586; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14587; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14588; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14589; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14590; GFX10-WGP-NEXT:    buffer_gl0_inv
14591; GFX10-WGP-NEXT:    buffer_gl1_inv
14592; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
14593; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
14594; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14595; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
14596; GFX10-WGP-NEXT:    s_endpgm
14597;
14598; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
14599; GFX10-CU:       ; %bb.0: ; %entry
14600; GFX10-CU-NEXT:    s_clause 0x1
14601; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14602; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14603; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14604; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
14605; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
14606; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
14607; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
14608; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
14609; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
14610; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14611; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14612; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14613; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14614; GFX10-CU-NEXT:    buffer_gl0_inv
14615; GFX10-CU-NEXT:    buffer_gl1_inv
14616; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
14617; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
14618; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14619; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
14620; GFX10-CU-NEXT:    s_endpgm
14621;
14622; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
14623; SKIP-CACHE-INV:       ; %bb.0: ; %entry
14624; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14625; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
14626; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14627; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
14628; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
14629; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
14630; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
14631; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
14632; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
14633; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14634; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14635; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14636; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
14637; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
14638; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14639; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
14640; SKIP-CACHE-INV-NEXT:    s_endpgm
14641;
14642; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
14643; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
14644; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14645; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14646; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14647; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14648; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14649; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14650; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14651; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14652; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
14653; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14654; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14655; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
14656;
14657; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
14658; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
14659; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14660; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14661; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14662; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14663; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14664; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14665; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14666; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14667; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
14668; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14669; GFX90A-TGSPLIT-NEXT:    s_endpgm
14670;
14671; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
14672; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
14673; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14674; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14675; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14676; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14677; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14678; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
14679; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14680; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
14681; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14682; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
14683; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14684; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14685; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
14686;
14687; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
14688; GFX940-TGSPLIT:       ; %bb.0: ; %entry
14689; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14690; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14691; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14692; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14693; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14694; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
14695; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14696; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
14697; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14698; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
14699; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14700; GFX940-TGSPLIT-NEXT:    s_endpgm
14701;
14702; GFX11-WGP-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
14703; GFX11-WGP:       ; %bb.0: ; %entry
14704; GFX11-WGP-NEXT:    s_clause 0x1
14705; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14706; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14707; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14708; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
14709; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
14710; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14711; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14712; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14713; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14714; GFX11-WGP-NEXT:    buffer_gl0_inv
14715; GFX11-WGP-NEXT:    buffer_gl1_inv
14716; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14717; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
14718; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
14719; GFX11-WGP-NEXT:    s_endpgm
14720;
14721; GFX11-CU-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
14722; GFX11-CU:       ; %bb.0: ; %entry
14723; GFX11-CU-NEXT:    s_clause 0x1
14724; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14725; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14726; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14727; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
14728; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
14729; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14730; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14731; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14732; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14733; GFX11-CU-NEXT:    buffer_gl0_inv
14734; GFX11-CU-NEXT:    buffer_gl1_inv
14735; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14736; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
14737; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
14738; GFX11-CU-NEXT:    s_endpgm
14739    i32* %out, i32 %in, i32 %old) {
14740entry:
14741  %gep = getelementptr i32, i32* %out, i32 4
14742  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel seq_cst
14743  %val0 = extractvalue { i32, i1 } %val, 0
14744  store i32 %val0, i32* %out, align 4
14745  ret void
14746}
14747
14748define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg(
14749; GFX7-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
14750; GFX7:       ; %bb.0: ; %entry
14751; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14752; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
14753; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14754; GFX7-NEXT:    s_add_u32 s4, s0, 16
14755; GFX7-NEXT:    s_addc_u32 s5, s1, 0
14756; GFX7-NEXT:    v_mov_b32_e32 v0, s4
14757; GFX7-NEXT:    v_mov_b32_e32 v2, s2
14758; GFX7-NEXT:    v_mov_b32_e32 v1, s5
14759; GFX7-NEXT:    v_mov_b32_e32 v3, s3
14760; GFX7-NEXT:    s_waitcnt vmcnt(0)
14761; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14762; GFX7-NEXT:    s_waitcnt vmcnt(0)
14763; GFX7-NEXT:    buffer_wbinvl1_vol
14764; GFX7-NEXT:    v_mov_b32_e32 v0, s0
14765; GFX7-NEXT:    v_mov_b32_e32 v1, s1
14766; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14767; GFX7-NEXT:    flat_store_dword v[0:1], v2
14768; GFX7-NEXT:    s_endpgm
14769;
14770; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
14771; GFX10-WGP:       ; %bb.0: ; %entry
14772; GFX10-WGP-NEXT:    s_clause 0x1
14773; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14774; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14775; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14776; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
14777; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
14778; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
14779; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
14780; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
14781; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
14782; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14783; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14784; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14785; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14786; GFX10-WGP-NEXT:    buffer_gl0_inv
14787; GFX10-WGP-NEXT:    buffer_gl1_inv
14788; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
14789; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
14790; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14791; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
14792; GFX10-WGP-NEXT:    s_endpgm
14793;
14794; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
14795; GFX10-CU:       ; %bb.0: ; %entry
14796; GFX10-CU-NEXT:    s_clause 0x1
14797; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14798; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14799; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14800; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
14801; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
14802; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
14803; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
14804; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
14805; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
14806; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14807; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14808; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14809; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14810; GFX10-CU-NEXT:    buffer_gl0_inv
14811; GFX10-CU-NEXT:    buffer_gl1_inv
14812; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
14813; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
14814; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14815; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
14816; GFX10-CU-NEXT:    s_endpgm
14817;
14818; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
14819; SKIP-CACHE-INV:       ; %bb.0: ; %entry
14820; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14821; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
14822; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14823; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
14824; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
14825; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
14826; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
14827; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
14828; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
14829; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14830; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14831; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14832; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
14833; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
14834; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14835; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
14836; SKIP-CACHE-INV-NEXT:    s_endpgm
14837;
14838; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
14839; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
14840; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14841; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14842; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14843; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14844; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14845; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14846; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14847; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14848; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
14849; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14850; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14851; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
14852;
14853; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
14854; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
14855; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14856; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14857; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14858; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14859; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14860; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14861; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14862; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14863; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
14864; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14865; GFX90A-TGSPLIT-NEXT:    s_endpgm
14866;
14867; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
14868; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
14869; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14870; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14871; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14872; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14873; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14874; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
14875; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14876; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
14877; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14878; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
14879; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14880; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14881; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
14882;
14883; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
14884; GFX940-TGSPLIT:       ; %bb.0: ; %entry
14885; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14886; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14887; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14888; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14889; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14890; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
14891; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14892; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
14893; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14894; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
14895; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14896; GFX940-TGSPLIT-NEXT:    s_endpgm
14897;
14898; GFX11-WGP-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
14899; GFX11-WGP:       ; %bb.0: ; %entry
14900; GFX11-WGP-NEXT:    s_clause 0x1
14901; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14902; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14903; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14904; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
14905; GFX11-WGP-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
14906; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14907; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14908; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14909; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14910; GFX11-WGP-NEXT:    buffer_gl0_inv
14911; GFX11-WGP-NEXT:    buffer_gl1_inv
14912; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14913; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
14914; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
14915; GFX11-WGP-NEXT:    s_endpgm
14916;
14917; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
14918; GFX11-CU:       ; %bb.0: ; %entry
14919; GFX11-CU-NEXT:    s_clause 0x1
14920; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14921; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14922; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14923; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
14924; GFX11-CU-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
14925; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14926; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14927; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14928; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14929; GFX11-CU-NEXT:    buffer_gl0_inv
14930; GFX11-CU-NEXT:    buffer_gl1_inv
14931; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14932; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
14933; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
14934; GFX11-CU-NEXT:    s_endpgm
14935    i32* %out, i32 %in, i32 %old) {
14936entry:
14937  %gep = getelementptr i32, i32* %out, i32 4
14938  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst seq_cst
14939  %val0 = extractvalue { i32, i1 } %val, 0
14940  store i32 %val0, i32* %out, align 4
14941  ret void
14942}
14943
14944