1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
3; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
4; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
5; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
6; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-NOTTGSPLIT %s
7; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-TGSPLIT %s
8; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-NOTTGSPLIT %s
9; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
10; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11-WGP %s
11; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11-CU %s
12
13define amdgpu_kernel void @flat_agent_unordered_load(
14; GFX7-LABEL: flat_agent_unordered_load:
15; GFX7:       ; %bb.0: ; %entry
16; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
17; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
18; GFX7-NEXT:    v_mov_b32_e32 v0, s0
19; GFX7-NEXT:    v_mov_b32_e32 v1, s1
20; GFX7-NEXT:    flat_load_dword v2, v[0:1]
21; GFX7-NEXT:    v_mov_b32_e32 v0, s2
22; GFX7-NEXT:    v_mov_b32_e32 v1, s3
23; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
24; GFX7-NEXT:    flat_store_dword v[0:1], v2
25; GFX7-NEXT:    s_endpgm
26;
27; GFX10-WGP-LABEL: flat_agent_unordered_load:
28; GFX10-WGP:       ; %bb.0: ; %entry
29; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
30; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
31; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
32; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
33; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
34; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
35; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
36; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
37; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
38; GFX10-WGP-NEXT:    s_endpgm
39;
40; GFX10-CU-LABEL: flat_agent_unordered_load:
41; GFX10-CU:       ; %bb.0: ; %entry
42; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
43; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
44; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
45; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
46; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
47; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
48; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
49; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
50; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
51; GFX10-CU-NEXT:    s_endpgm
52;
53; SKIP-CACHE-INV-LABEL: flat_agent_unordered_load:
54; SKIP-CACHE-INV:       ; %bb.0: ; %entry
55; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
56; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
57; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
58; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
59; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
60; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
61; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
62; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
63; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
64; SKIP-CACHE-INV-NEXT:    s_endpgm
65;
66; GFX90A-NOTTGSPLIT-LABEL: flat_agent_unordered_load:
67; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
68; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
69; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
70; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
71; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
72; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
73; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
74; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
75; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
76; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
77; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
78;
79; GFX90A-TGSPLIT-LABEL: flat_agent_unordered_load:
80; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
81; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
82; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
83; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
84; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
85; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
86; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
87; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
88; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
89; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
90; GFX90A-TGSPLIT-NEXT:    s_endpgm
91;
92; GFX940-NOTTGSPLIT-LABEL: flat_agent_unordered_load:
93; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
94; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
95; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
96; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
97; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
98; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
99; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
100; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
101; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
102; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
103; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
104;
105; GFX940-TGSPLIT-LABEL: flat_agent_unordered_load:
106; GFX940-TGSPLIT:       ; %bb.0: ; %entry
107; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
108; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
109; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
110; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
111; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
112; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
113; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
114; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
115; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
116; GFX940-TGSPLIT-NEXT:    s_endpgm
117;
118; GFX11-WGP-LABEL: flat_agent_unordered_load:
119; GFX11-WGP:       ; %bb.0: ; %entry
120; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
121; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
122; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
123; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
124; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1]
125; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
126; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
127; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
128; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
129; GFX11-WGP-NEXT:    s_endpgm
130;
131; GFX11-CU-LABEL: flat_agent_unordered_load:
132; GFX11-CU:       ; %bb.0: ; %entry
133; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
134; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
135; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
136; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
137; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
138; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
139; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
140; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
141; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
142; GFX11-CU-NEXT:    s_endpgm
143    i32* %in, i32* %out) {
144entry:
145  %val = load atomic i32, i32* %in syncscope("agent") unordered, align 4
146  store i32 %val, i32* %out
147  ret void
148}
149
150define amdgpu_kernel void @flat_agent_monotonic_load(
151; GFX7-LABEL: flat_agent_monotonic_load:
152; GFX7:       ; %bb.0: ; %entry
153; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
154; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
155; GFX7-NEXT:    v_mov_b32_e32 v0, s0
156; GFX7-NEXT:    v_mov_b32_e32 v1, s1
157; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
158; GFX7-NEXT:    v_mov_b32_e32 v0, s2
159; GFX7-NEXT:    v_mov_b32_e32 v1, s3
160; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
161; GFX7-NEXT:    flat_store_dword v[0:1], v2
162; GFX7-NEXT:    s_endpgm
163;
164; GFX10-WGP-LABEL: flat_agent_monotonic_load:
165; GFX10-WGP:       ; %bb.0: ; %entry
166; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
167; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
168; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
169; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
170; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
171; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
172; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
173; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
174; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
175; GFX10-WGP-NEXT:    s_endpgm
176;
177; GFX10-CU-LABEL: flat_agent_monotonic_load:
178; GFX10-CU:       ; %bb.0: ; %entry
179; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
180; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
181; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
182; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
183; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
184; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
185; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
186; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
187; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
188; GFX10-CU-NEXT:    s_endpgm
189;
190; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_load:
191; SKIP-CACHE-INV:       ; %bb.0: ; %entry
192; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
193; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
194; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
195; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
196; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
197; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
198; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
199; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
200; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
201; SKIP-CACHE-INV-NEXT:    s_endpgm
202;
203; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_load:
204; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
205; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
206; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
207; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
208; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
209; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
210; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
211; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
212; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
213; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
214; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
215;
216; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_load:
217; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
218; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
219; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
220; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
221; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
222; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
223; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
224; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
225; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
226; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
227; GFX90A-TGSPLIT-NEXT:    s_endpgm
228;
229; GFX940-NOTTGSPLIT-LABEL: flat_agent_monotonic_load:
230; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
231; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
232; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
233; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
234; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
235; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc1
236; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
237; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
238; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
239; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
240; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
241;
242; GFX940-TGSPLIT-LABEL: flat_agent_monotonic_load:
243; GFX940-TGSPLIT:       ; %bb.0: ; %entry
244; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
245; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
246; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
247; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
248; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc1
249; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
250; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
251; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
252; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
253; GFX940-TGSPLIT-NEXT:    s_endpgm
254;
255; GFX11-WGP-LABEL: flat_agent_monotonic_load:
256; GFX11-WGP:       ; %bb.0: ; %entry
257; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
258; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
259; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
260; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
261; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc
262; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
263; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
264; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
265; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
266; GFX11-WGP-NEXT:    s_endpgm
267;
268; GFX11-CU-LABEL: flat_agent_monotonic_load:
269; GFX11-CU:       ; %bb.0: ; %entry
270; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
271; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
272; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
273; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
274; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1] glc
275; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
276; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
277; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
278; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
279; GFX11-CU-NEXT:    s_endpgm
280    i32* %in, i32* %out) {
281entry:
282  %val = load atomic i32, i32* %in syncscope("agent") monotonic, align 4
283  store i32 %val, i32* %out
284  ret void
285}
286
287define amdgpu_kernel void @flat_agent_acquire_load(
288; GFX7-LABEL: flat_agent_acquire_load:
289; GFX7:       ; %bb.0: ; %entry
290; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
291; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
292; GFX7-NEXT:    v_mov_b32_e32 v0, s0
293; GFX7-NEXT:    v_mov_b32_e32 v1, s1
294; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
295; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
296; GFX7-NEXT:    buffer_wbinvl1_vol
297; GFX7-NEXT:    v_mov_b32_e32 v0, s2
298; GFX7-NEXT:    v_mov_b32_e32 v1, s3
299; GFX7-NEXT:    flat_store_dword v[0:1], v2
300; GFX7-NEXT:    s_endpgm
301;
302; GFX10-WGP-LABEL: flat_agent_acquire_load:
303; GFX10-WGP:       ; %bb.0: ; %entry
304; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
305; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
306; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
307; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
308; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
309; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
310; GFX10-WGP-NEXT:    buffer_gl0_inv
311; GFX10-WGP-NEXT:    buffer_gl1_inv
312; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
313; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
314; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
315; GFX10-WGP-NEXT:    s_endpgm
316;
317; GFX10-CU-LABEL: flat_agent_acquire_load:
318; GFX10-CU:       ; %bb.0: ; %entry
319; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
320; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
321; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
322; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
323; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
324; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
325; GFX10-CU-NEXT:    buffer_gl0_inv
326; GFX10-CU-NEXT:    buffer_gl1_inv
327; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
328; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
329; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
330; GFX10-CU-NEXT:    s_endpgm
331;
332; SKIP-CACHE-INV-LABEL: flat_agent_acquire_load:
333; SKIP-CACHE-INV:       ; %bb.0: ; %entry
334; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
335; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
336; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
337; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
338; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
339; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
340; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
341; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
342; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
343; SKIP-CACHE-INV-NEXT:    s_endpgm
344;
345; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_load:
346; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
347; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
348; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
349; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
350; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
351; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
352; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
353; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
354; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
355; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
356; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
357; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
358;
359; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_load:
360; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
361; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
362; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
363; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
364; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
365; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
366; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
367; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
368; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
369; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
370; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
371; GFX90A-TGSPLIT-NEXT:    s_endpgm
372;
373; GFX940-NOTTGSPLIT-LABEL: flat_agent_acquire_load:
374; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
375; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
376; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
377; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
378; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
379; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc1
380; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
381; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
382; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
383; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
384; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
385; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
386;
387; GFX940-TGSPLIT-LABEL: flat_agent_acquire_load:
388; GFX940-TGSPLIT:       ; %bb.0: ; %entry
389; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
390; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
391; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
392; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
393; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc1
394; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
395; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
396; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
397; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
398; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
399; GFX940-TGSPLIT-NEXT:    s_endpgm
400;
401; GFX11-WGP-LABEL: flat_agent_acquire_load:
402; GFX11-WGP:       ; %bb.0: ; %entry
403; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
404; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
405; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
406; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
407; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc
408; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
409; GFX11-WGP-NEXT:    buffer_gl0_inv
410; GFX11-WGP-NEXT:    buffer_gl1_inv
411; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
412; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
413; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
414; GFX11-WGP-NEXT:    s_endpgm
415;
416; GFX11-CU-LABEL: flat_agent_acquire_load:
417; GFX11-CU:       ; %bb.0: ; %entry
418; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
419; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
420; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
421; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
422; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1] glc
423; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
424; GFX11-CU-NEXT:    buffer_gl0_inv
425; GFX11-CU-NEXT:    buffer_gl1_inv
426; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
427; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
428; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
429; GFX11-CU-NEXT:    s_endpgm
430    i32* %in, i32* %out) {
431entry:
432  %val = load atomic i32, i32* %in syncscope("agent") acquire, align 4
433  store i32 %val, i32* %out
434  ret void
435}
436
437define amdgpu_kernel void @flat_agent_seq_cst_load(
438; GFX7-LABEL: flat_agent_seq_cst_load:
439; GFX7:       ; %bb.0: ; %entry
440; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
441; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
442; GFX7-NEXT:    v_mov_b32_e32 v0, s0
443; GFX7-NEXT:    v_mov_b32_e32 v1, s1
444; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
445; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
446; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
447; GFX7-NEXT:    buffer_wbinvl1_vol
448; GFX7-NEXT:    v_mov_b32_e32 v0, s2
449; GFX7-NEXT:    v_mov_b32_e32 v1, s3
450; GFX7-NEXT:    flat_store_dword v[0:1], v2
451; GFX7-NEXT:    s_endpgm
452;
453; GFX10-WGP-LABEL: flat_agent_seq_cst_load:
454; GFX10-WGP:       ; %bb.0: ; %entry
455; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
456; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
457; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
458; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
459; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
460; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
461; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
462; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
463; GFX10-WGP-NEXT:    buffer_gl0_inv
464; GFX10-WGP-NEXT:    buffer_gl1_inv
465; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
466; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
467; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
468; GFX10-WGP-NEXT:    s_endpgm
469;
470; GFX10-CU-LABEL: flat_agent_seq_cst_load:
471; GFX10-CU:       ; %bb.0: ; %entry
472; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
473; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
474; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
475; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
476; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
477; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
478; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
479; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
480; GFX10-CU-NEXT:    buffer_gl0_inv
481; GFX10-CU-NEXT:    buffer_gl1_inv
482; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
483; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
484; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
485; GFX10-CU-NEXT:    s_endpgm
486;
487; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_load:
488; SKIP-CACHE-INV:       ; %bb.0: ; %entry
489; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
490; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
491; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
492; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
493; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
494; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
495; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
496; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
497; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
498; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
499; SKIP-CACHE-INV-NEXT:    s_endpgm
500;
501; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_load:
502; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
503; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
504; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
505; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
506; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
507; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
508; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
509; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
510; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
511; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
512; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
513; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
514; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
515;
516; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_load:
517; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
518; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
519; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
520; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
521; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
522; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
523; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
524; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
525; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
526; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
527; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
528; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
529; GFX90A-TGSPLIT-NEXT:    s_endpgm
530;
531; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_load:
532; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
533; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
534; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
535; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
536; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
537; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
538; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc1
539; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
540; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
541; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
542; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
543; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
544; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
545;
546; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_load:
547; GFX940-TGSPLIT:       ; %bb.0: ; %entry
548; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
549; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
550; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
551; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
552; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
553; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc1
554; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
555; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
556; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
557; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
558; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
559; GFX940-TGSPLIT-NEXT:    s_endpgm
560;
561; GFX11-WGP-LABEL: flat_agent_seq_cst_load:
562; GFX11-WGP:       ; %bb.0: ; %entry
563; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
564; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
565; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
566; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
567; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
568; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
569; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc
570; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
571; GFX11-WGP-NEXT:    buffer_gl0_inv
572; GFX11-WGP-NEXT:    buffer_gl1_inv
573; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
574; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
575; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
576; GFX11-WGP-NEXT:    s_endpgm
577;
578; GFX11-CU-LABEL: flat_agent_seq_cst_load:
579; GFX11-CU:       ; %bb.0: ; %entry
580; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
581; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
582; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
583; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
584; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
585; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
586; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1] glc
587; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
588; GFX11-CU-NEXT:    buffer_gl0_inv
589; GFX11-CU-NEXT:    buffer_gl1_inv
590; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
591; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
592; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
593; GFX11-CU-NEXT:    s_endpgm
594    i32* %in, i32* %out) {
595entry:
596  %val = load atomic i32, i32* %in syncscope("agent") seq_cst, align 4
597  store i32 %val, i32* %out
598  ret void
599}
600
601define amdgpu_kernel void @flat_agent_unordered_store(
602; GFX7-LABEL: flat_agent_unordered_store:
603; GFX7:       ; %bb.0: ; %entry
604; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
605; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
606; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
607; GFX7-NEXT:    v_mov_b32_e32 v0, s0
608; GFX7-NEXT:    v_mov_b32_e32 v1, s1
609; GFX7-NEXT:    v_mov_b32_e32 v2, s2
610; GFX7-NEXT:    flat_store_dword v[0:1], v2
611; GFX7-NEXT:    s_endpgm
612;
613; GFX10-WGP-LABEL: flat_agent_unordered_store:
614; GFX10-WGP:       ; %bb.0: ; %entry
615; GFX10-WGP-NEXT:    s_clause 0x1
616; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
617; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
618; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
619; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
620; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
621; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
622; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
623; GFX10-WGP-NEXT:    s_endpgm
624;
625; GFX10-CU-LABEL: flat_agent_unordered_store:
626; GFX10-CU:       ; %bb.0: ; %entry
627; GFX10-CU-NEXT:    s_clause 0x1
628; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
629; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
630; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
631; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
632; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
633; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
634; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
635; GFX10-CU-NEXT:    s_endpgm
636;
637; SKIP-CACHE-INV-LABEL: flat_agent_unordered_store:
638; SKIP-CACHE-INV:       ; %bb.0: ; %entry
639; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
640; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
641; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
642; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
643; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
644; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
645; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
646; SKIP-CACHE-INV-NEXT:    s_endpgm
647;
648; GFX90A-NOTTGSPLIT-LABEL: flat_agent_unordered_store:
649; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
650; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
651; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
652; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
653; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
654; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
655; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
656; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
657;
658; GFX90A-TGSPLIT-LABEL: flat_agent_unordered_store:
659; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
660; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
661; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
662; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
663; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
664; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
665; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
666; GFX90A-TGSPLIT-NEXT:    s_endpgm
667;
668; GFX940-NOTTGSPLIT-LABEL: flat_agent_unordered_store:
669; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
670; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
671; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
672; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
673; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
674; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
675; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
676; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
677;
678; GFX940-TGSPLIT-LABEL: flat_agent_unordered_store:
679; GFX940-TGSPLIT:       ; %bb.0: ; %entry
680; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
681; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
682; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
683; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
684; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
685; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
686; GFX940-TGSPLIT-NEXT:    s_endpgm
687;
688; GFX11-WGP-LABEL: flat_agent_unordered_store:
689; GFX11-WGP:       ; %bb.0: ; %entry
690; GFX11-WGP-NEXT:    s_clause 0x1
691; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
692; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
693; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
694; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
695; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
696; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
697; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
698; GFX11-WGP-NEXT:    s_endpgm
699;
700; GFX11-CU-LABEL: flat_agent_unordered_store:
701; GFX11-CU:       ; %bb.0: ; %entry
702; GFX11-CU-NEXT:    s_clause 0x1
703; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
704; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
705; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
706; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
707; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
708; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
709; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
710; GFX11-CU-NEXT:    s_endpgm
711    i32 %in, i32* %out) {
712entry:
713  store atomic i32 %in, i32* %out syncscope("agent") unordered, align 4
714  ret void
715}
716
717define amdgpu_kernel void @flat_agent_monotonic_store(
718; GFX7-LABEL: flat_agent_monotonic_store:
719; GFX7:       ; %bb.0: ; %entry
720; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
721; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
722; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
723; GFX7-NEXT:    v_mov_b32_e32 v0, s0
724; GFX7-NEXT:    v_mov_b32_e32 v1, s1
725; GFX7-NEXT:    v_mov_b32_e32 v2, s2
726; GFX7-NEXT:    flat_store_dword v[0:1], v2
727; GFX7-NEXT:    s_endpgm
728;
729; GFX10-WGP-LABEL: flat_agent_monotonic_store:
730; GFX10-WGP:       ; %bb.0: ; %entry
731; GFX10-WGP-NEXT:    s_clause 0x1
732; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
733; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
734; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
735; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
736; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
737; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
738; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
739; GFX10-WGP-NEXT:    s_endpgm
740;
741; GFX10-CU-LABEL: flat_agent_monotonic_store:
742; GFX10-CU:       ; %bb.0: ; %entry
743; GFX10-CU-NEXT:    s_clause 0x1
744; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
745; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
746; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
747; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
748; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
749; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
750; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
751; GFX10-CU-NEXT:    s_endpgm
752;
753; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_store:
754; SKIP-CACHE-INV:       ; %bb.0: ; %entry
755; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
756; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
757; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
758; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
759; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
760; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
761; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
762; SKIP-CACHE-INV-NEXT:    s_endpgm
763;
764; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_store:
765; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
766; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
767; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
768; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
769; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
770; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
771; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
772; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
773;
774; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_store:
775; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
776; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
777; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
778; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
779; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
780; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
781; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
782; GFX90A-TGSPLIT-NEXT:    s_endpgm
783;
784; GFX940-NOTTGSPLIT-LABEL: flat_agent_monotonic_store:
785; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
786; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
787; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
788; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
789; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
790; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
791; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc1
792; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
793;
794; GFX940-TGSPLIT-LABEL: flat_agent_monotonic_store:
795; GFX940-TGSPLIT:       ; %bb.0: ; %entry
796; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
797; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
798; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
799; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
800; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
801; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc1
802; GFX940-TGSPLIT-NEXT:    s_endpgm
803;
804; GFX11-WGP-LABEL: flat_agent_monotonic_store:
805; GFX11-WGP:       ; %bb.0: ; %entry
806; GFX11-WGP-NEXT:    s_clause 0x1
807; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
808; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
809; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
810; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
811; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
812; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
813; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
814; GFX11-WGP-NEXT:    s_endpgm
815;
816; GFX11-CU-LABEL: flat_agent_monotonic_store:
817; GFX11-CU:       ; %bb.0: ; %entry
818; GFX11-CU-NEXT:    s_clause 0x1
819; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
820; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
821; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
822; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
823; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
824; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
825; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
826; GFX11-CU-NEXT:    s_endpgm
827    i32 %in, i32* %out) {
828entry:
829  store atomic i32 %in, i32* %out syncscope("agent") monotonic, align 4
830  ret void
831}
832
833define amdgpu_kernel void @flat_agent_release_store(
834; GFX7-LABEL: flat_agent_release_store:
835; GFX7:       ; %bb.0: ; %entry
836; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
837; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
838; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
839; GFX7-NEXT:    v_mov_b32_e32 v0, s0
840; GFX7-NEXT:    v_mov_b32_e32 v1, s1
841; GFX7-NEXT:    v_mov_b32_e32 v2, s2
842; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
843; GFX7-NEXT:    flat_store_dword v[0:1], v2
844; GFX7-NEXT:    s_endpgm
845;
846; GFX10-WGP-LABEL: flat_agent_release_store:
847; GFX10-WGP:       ; %bb.0: ; %entry
848; GFX10-WGP-NEXT:    s_clause 0x1
849; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
850; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
851; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
852; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
853; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
854; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
855; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
856; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
857; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
858; GFX10-WGP-NEXT:    s_endpgm
859;
860; GFX10-CU-LABEL: flat_agent_release_store:
861; GFX10-CU:       ; %bb.0: ; %entry
862; GFX10-CU-NEXT:    s_clause 0x1
863; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
864; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
865; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
866; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
867; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
868; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
869; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
870; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
871; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
872; GFX10-CU-NEXT:    s_endpgm
873;
874; SKIP-CACHE-INV-LABEL: flat_agent_release_store:
875; SKIP-CACHE-INV:       ; %bb.0: ; %entry
876; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
877; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
878; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
879; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
880; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
881; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
882; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
883; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
884; SKIP-CACHE-INV-NEXT:    s_endpgm
885;
886; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_store:
887; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
888; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
889; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
890; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
891; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
892; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
893; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
894; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
895; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
896;
897; GFX90A-TGSPLIT-LABEL: flat_agent_release_store:
898; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
899; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
900; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
901; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
902; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
903; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
904; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
905; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
906; GFX90A-TGSPLIT-NEXT:    s_endpgm
907;
908; GFX940-NOTTGSPLIT-LABEL: flat_agent_release_store:
909; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
910; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
911; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
912; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
913; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
914; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
915; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
916; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
917; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc1
918; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
919;
920; GFX940-TGSPLIT-LABEL: flat_agent_release_store:
921; GFX940-TGSPLIT:       ; %bb.0: ; %entry
922; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
923; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
924; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
925; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
926; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
927; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
928; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
929; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc1
930; GFX940-TGSPLIT-NEXT:    s_endpgm
931;
932; GFX11-WGP-LABEL: flat_agent_release_store:
933; GFX11-WGP:       ; %bb.0: ; %entry
934; GFX11-WGP-NEXT:    s_clause 0x1
935; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
936; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
937; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
938; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
939; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
940; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
941; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
942; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
943; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
944; GFX11-WGP-NEXT:    s_endpgm
945;
946; GFX11-CU-LABEL: flat_agent_release_store:
947; GFX11-CU:       ; %bb.0: ; %entry
948; GFX11-CU-NEXT:    s_clause 0x1
949; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
950; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
951; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
952; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
953; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
954; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
955; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
956; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
957; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
958; GFX11-CU-NEXT:    s_endpgm
959    i32 %in, i32* %out) {
960entry:
961  store atomic i32 %in, i32* %out syncscope("agent") release, align 4
962  ret void
963}
964
965define amdgpu_kernel void @flat_agent_seq_cst_store(
966; GFX7-LABEL: flat_agent_seq_cst_store:
967; GFX7:       ; %bb.0: ; %entry
968; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
969; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
970; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
971; GFX7-NEXT:    v_mov_b32_e32 v0, s0
972; GFX7-NEXT:    v_mov_b32_e32 v1, s1
973; GFX7-NEXT:    v_mov_b32_e32 v2, s2
974; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
975; GFX7-NEXT:    flat_store_dword v[0:1], v2
976; GFX7-NEXT:    s_endpgm
977;
978; GFX10-WGP-LABEL: flat_agent_seq_cst_store:
979; GFX10-WGP:       ; %bb.0: ; %entry
980; GFX10-WGP-NEXT:    s_clause 0x1
981; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
982; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
983; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
984; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
985; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
986; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
987; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
988; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
989; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
990; GFX10-WGP-NEXT:    s_endpgm
991;
992; GFX10-CU-LABEL: flat_agent_seq_cst_store:
993; GFX10-CU:       ; %bb.0: ; %entry
994; GFX10-CU-NEXT:    s_clause 0x1
995; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
996; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
997; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
998; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
999; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1000; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1001; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1002; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1003; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1004; GFX10-CU-NEXT:    s_endpgm
1005;
1006; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_store:
1007; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1008; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
1009; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
1010; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1011; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1012; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1013; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1014; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1015; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1016; SKIP-CACHE-INV-NEXT:    s_endpgm
1017;
1018; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_store:
1019; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1020; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1021; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
1022; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1023; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1024; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1025; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1026; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1027; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1028;
1029; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_store:
1030; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1031; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1032; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
1033; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1034; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1035; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1036; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1037; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1038; GFX90A-TGSPLIT-NEXT:    s_endpgm
1039;
1040; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_store:
1041; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1042; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
1043; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
1044; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1045; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1046; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1047; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
1048; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1049; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc1
1050; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1051;
1052; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_store:
1053; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1054; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
1055; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
1056; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1057; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1058; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1059; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
1060; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1061; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc1
1062; GFX940-TGSPLIT-NEXT:    s_endpgm
1063;
1064; GFX11-WGP-LABEL: flat_agent_seq_cst_store:
1065; GFX11-WGP:       ; %bb.0: ; %entry
1066; GFX11-WGP-NEXT:    s_clause 0x1
1067; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
1068; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
1069; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1070; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1071; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1072; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1073; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1074; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1075; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
1076; GFX11-WGP-NEXT:    s_endpgm
1077;
1078; GFX11-CU-LABEL: flat_agent_seq_cst_store:
1079; GFX11-CU:       ; %bb.0: ; %entry
1080; GFX11-CU-NEXT:    s_clause 0x1
1081; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
1082; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
1083; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1084; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
1085; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
1086; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1087; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1088; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1089; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
1090; GFX11-CU-NEXT:    s_endpgm
1091    i32 %in, i32* %out) {
1092entry:
1093  store atomic i32 %in, i32* %out syncscope("agent") seq_cst, align 4
1094  ret void
1095}
1096
1097define amdgpu_kernel void @flat_agent_monotonic_atomicrmw(
1098; GFX7-LABEL: flat_agent_monotonic_atomicrmw:
1099; GFX7:       ; %bb.0: ; %entry
1100; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1101; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1102; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1103; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1104; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1105; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1106; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1107; GFX7-NEXT:    s_endpgm
1108;
1109; GFX10-WGP-LABEL: flat_agent_monotonic_atomicrmw:
1110; GFX10-WGP:       ; %bb.0: ; %entry
1111; GFX10-WGP-NEXT:    s_clause 0x1
1112; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1113; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1114; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1115; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1116; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1117; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1118; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1119; GFX10-WGP-NEXT:    s_endpgm
1120;
1121; GFX10-CU-LABEL: flat_agent_monotonic_atomicrmw:
1122; GFX10-CU:       ; %bb.0: ; %entry
1123; GFX10-CU-NEXT:    s_clause 0x1
1124; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1125; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1126; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1127; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1128; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1129; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1130; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1131; GFX10-CU-NEXT:    s_endpgm
1132;
1133; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_atomicrmw:
1134; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1135; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1136; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1137; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1138; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1139; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1140; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1141; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1142; SKIP-CACHE-INV-NEXT:    s_endpgm
1143;
1144; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_atomicrmw:
1145; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1146; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1147; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1148; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1149; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1150; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1151; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1152; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1153;
1154; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_atomicrmw:
1155; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1156; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1157; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1158; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1159; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1160; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1161; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1162; GFX90A-TGSPLIT-NEXT:    s_endpgm
1163;
1164; GFX940-NOTTGSPLIT-LABEL: flat_agent_monotonic_atomicrmw:
1165; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1166; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1167; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1168; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1169; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1170; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1171; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1172; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1173;
1174; GFX940-TGSPLIT-LABEL: flat_agent_monotonic_atomicrmw:
1175; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1176; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1177; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1178; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1179; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1180; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1181; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1182; GFX940-TGSPLIT-NEXT:    s_endpgm
1183;
1184; GFX11-WGP-LABEL: flat_agent_monotonic_atomicrmw:
1185; GFX11-WGP:       ; %bb.0: ; %entry
1186; GFX11-WGP-NEXT:    s_clause 0x1
1187; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1188; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1189; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1190; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1191; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1192; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1193; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1194; GFX11-WGP-NEXT:    s_endpgm
1195;
1196; GFX11-CU-LABEL: flat_agent_monotonic_atomicrmw:
1197; GFX11-CU:       ; %bb.0: ; %entry
1198; GFX11-CU-NEXT:    s_clause 0x1
1199; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1200; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1201; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1202; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
1203; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
1204; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1205; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1206; GFX11-CU-NEXT:    s_endpgm
1207    i32* %out, i32 %in) {
1208entry:
1209  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") monotonic
1210  ret void
1211}
1212
1213define amdgpu_kernel void @flat_agent_acquire_atomicrmw(
1214; GFX7-LABEL: flat_agent_acquire_atomicrmw:
1215; GFX7:       ; %bb.0: ; %entry
1216; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1217; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1218; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1219; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1220; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1221; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1222; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1223; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1224; GFX7-NEXT:    buffer_wbinvl1_vol
1225; GFX7-NEXT:    s_endpgm
1226;
1227; GFX10-WGP-LABEL: flat_agent_acquire_atomicrmw:
1228; GFX10-WGP:       ; %bb.0: ; %entry
1229; GFX10-WGP-NEXT:    s_clause 0x1
1230; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1231; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1232; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1233; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1234; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1235; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1236; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1237; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1238; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1239; GFX10-WGP-NEXT:    buffer_gl0_inv
1240; GFX10-WGP-NEXT:    buffer_gl1_inv
1241; GFX10-WGP-NEXT:    s_endpgm
1242;
1243; GFX10-CU-LABEL: flat_agent_acquire_atomicrmw:
1244; GFX10-CU:       ; %bb.0: ; %entry
1245; GFX10-CU-NEXT:    s_clause 0x1
1246; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1247; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1248; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1249; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1250; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1251; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1252; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1253; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1254; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1255; GFX10-CU-NEXT:    buffer_gl0_inv
1256; GFX10-CU-NEXT:    buffer_gl1_inv
1257; GFX10-CU-NEXT:    s_endpgm
1258;
1259; SKIP-CACHE-INV-LABEL: flat_agent_acquire_atomicrmw:
1260; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1261; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1262; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1263; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1264; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1265; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1266; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1267; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1268; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1269; SKIP-CACHE-INV-NEXT:    s_endpgm
1270;
1271; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_atomicrmw:
1272; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1273; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1274; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1275; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1276; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1277; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1278; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1279; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1280; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1281; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1282;
1283; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_atomicrmw:
1284; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1285; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1286; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1287; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1288; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1289; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1290; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1291; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1292; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1293; GFX90A-TGSPLIT-NEXT:    s_endpgm
1294;
1295; GFX940-NOTTGSPLIT-LABEL: flat_agent_acquire_atomicrmw:
1296; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1297; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1298; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1299; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1300; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1301; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1302; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1303; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1304; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
1305; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1306;
1307; GFX940-TGSPLIT-LABEL: flat_agent_acquire_atomicrmw:
1308; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1309; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1310; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1311; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1312; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1313; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1314; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1315; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1316; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
1317; GFX940-TGSPLIT-NEXT:    s_endpgm
1318;
1319; GFX11-WGP-LABEL: flat_agent_acquire_atomicrmw:
1320; GFX11-WGP:       ; %bb.0: ; %entry
1321; GFX11-WGP-NEXT:    s_clause 0x1
1322; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1323; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1324; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1325; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1326; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1327; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1328; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1329; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1330; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1331; GFX11-WGP-NEXT:    buffer_gl0_inv
1332; GFX11-WGP-NEXT:    buffer_gl1_inv
1333; GFX11-WGP-NEXT:    s_endpgm
1334;
1335; GFX11-CU-LABEL: flat_agent_acquire_atomicrmw:
1336; GFX11-CU:       ; %bb.0: ; %entry
1337; GFX11-CU-NEXT:    s_clause 0x1
1338; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1339; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1340; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1341; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
1342; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
1343; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1344; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1345; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1346; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1347; GFX11-CU-NEXT:    buffer_gl0_inv
1348; GFX11-CU-NEXT:    buffer_gl1_inv
1349; GFX11-CU-NEXT:    s_endpgm
1350    i32* %out, i32 %in) {
1351entry:
1352  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") acquire
1353  ret void
1354}
1355
1356define amdgpu_kernel void @flat_agent_release_atomicrmw(
1357; GFX7-LABEL: flat_agent_release_atomicrmw:
1358; GFX7:       ; %bb.0: ; %entry
1359; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1360; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1361; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1362; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1363; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1364; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1365; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1366; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1367; GFX7-NEXT:    s_endpgm
1368;
1369; GFX10-WGP-LABEL: flat_agent_release_atomicrmw:
1370; GFX10-WGP:       ; %bb.0: ; %entry
1371; GFX10-WGP-NEXT:    s_clause 0x1
1372; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1373; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1374; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1375; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1376; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1377; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1378; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1379; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1380; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1381; GFX10-WGP-NEXT:    s_endpgm
1382;
1383; GFX10-CU-LABEL: flat_agent_release_atomicrmw:
1384; GFX10-CU:       ; %bb.0: ; %entry
1385; GFX10-CU-NEXT:    s_clause 0x1
1386; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1387; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1388; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1389; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1390; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1391; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1392; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1393; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1394; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1395; GFX10-CU-NEXT:    s_endpgm
1396;
1397; SKIP-CACHE-INV-LABEL: flat_agent_release_atomicrmw:
1398; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1399; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1400; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1401; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1402; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1403; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1404; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1405; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1406; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1407; SKIP-CACHE-INV-NEXT:    s_endpgm
1408;
1409; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_atomicrmw:
1410; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1411; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1412; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1413; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1414; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1415; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1416; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1417; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1418; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1419;
1420; GFX90A-TGSPLIT-LABEL: flat_agent_release_atomicrmw:
1421; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1422; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1423; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1424; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1425; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1426; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1427; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1428; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1429; GFX90A-TGSPLIT-NEXT:    s_endpgm
1430;
1431; GFX940-NOTTGSPLIT-LABEL: flat_agent_release_atomicrmw:
1432; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1433; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1434; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1435; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1436; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1437; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1438; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
1439; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1440; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1441; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1442;
1443; GFX940-TGSPLIT-LABEL: flat_agent_release_atomicrmw:
1444; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1445; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1446; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1447; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1448; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1449; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1450; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
1451; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1452; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1453; GFX940-TGSPLIT-NEXT:    s_endpgm
1454;
1455; GFX11-WGP-LABEL: flat_agent_release_atomicrmw:
1456; GFX11-WGP:       ; %bb.0: ; %entry
1457; GFX11-WGP-NEXT:    s_clause 0x1
1458; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1459; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1460; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1461; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1462; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1463; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1464; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1465; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1466; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1467; GFX11-WGP-NEXT:    s_endpgm
1468;
1469; GFX11-CU-LABEL: flat_agent_release_atomicrmw:
1470; GFX11-CU:       ; %bb.0: ; %entry
1471; GFX11-CU-NEXT:    s_clause 0x1
1472; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1473; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1474; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1475; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
1476; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
1477; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1478; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1479; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1480; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1481; GFX11-CU-NEXT:    s_endpgm
1482    i32* %out, i32 %in) {
1483entry:
1484  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") release
1485  ret void
1486}
1487
1488define amdgpu_kernel void @flat_agent_acq_rel_atomicrmw(
1489; GFX7-LABEL: flat_agent_acq_rel_atomicrmw:
1490; GFX7:       ; %bb.0: ; %entry
1491; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1492; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1493; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1494; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1495; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1496; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1497; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1498; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1499; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1500; GFX7-NEXT:    buffer_wbinvl1_vol
1501; GFX7-NEXT:    s_endpgm
1502;
1503; GFX10-WGP-LABEL: flat_agent_acq_rel_atomicrmw:
1504; GFX10-WGP:       ; %bb.0: ; %entry
1505; GFX10-WGP-NEXT:    s_clause 0x1
1506; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1507; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1508; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1509; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1510; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1511; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1512; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1513; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1514; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1515; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1516; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1517; GFX10-WGP-NEXT:    buffer_gl0_inv
1518; GFX10-WGP-NEXT:    buffer_gl1_inv
1519; GFX10-WGP-NEXT:    s_endpgm
1520;
1521; GFX10-CU-LABEL: flat_agent_acq_rel_atomicrmw:
1522; GFX10-CU:       ; %bb.0: ; %entry
1523; GFX10-CU-NEXT:    s_clause 0x1
1524; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1525; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1526; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1527; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1528; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1529; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1530; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1531; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1532; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1533; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1534; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1535; GFX10-CU-NEXT:    buffer_gl0_inv
1536; GFX10-CU-NEXT:    buffer_gl1_inv
1537; GFX10-CU-NEXT:    s_endpgm
1538;
1539; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_atomicrmw:
1540; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1541; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1542; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1543; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1544; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1545; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1546; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1547; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1548; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1549; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1550; SKIP-CACHE-INV-NEXT:    s_endpgm
1551;
1552; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_atomicrmw:
1553; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1554; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1555; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1556; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1557; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1558; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1559; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1560; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1561; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1562; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1563; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1564;
1565; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_atomicrmw:
1566; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1567; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1568; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1569; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1570; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1571; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1572; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1573; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1574; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1575; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1576; GFX90A-TGSPLIT-NEXT:    s_endpgm
1577;
1578; GFX940-NOTTGSPLIT-LABEL: flat_agent_acq_rel_atomicrmw:
1579; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1580; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1581; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1582; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1583; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1584; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1585; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
1586; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1587; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1588; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1589; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
1590; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1591;
1592; GFX940-TGSPLIT-LABEL: flat_agent_acq_rel_atomicrmw:
1593; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1594; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1595; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1596; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1597; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1598; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1599; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
1600; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1601; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1602; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1603; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
1604; GFX940-TGSPLIT-NEXT:    s_endpgm
1605;
1606; GFX11-WGP-LABEL: flat_agent_acq_rel_atomicrmw:
1607; GFX11-WGP:       ; %bb.0: ; %entry
1608; GFX11-WGP-NEXT:    s_clause 0x1
1609; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1610; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1611; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1612; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1613; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1614; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1615; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1616; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1617; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1618; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1619; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1620; GFX11-WGP-NEXT:    buffer_gl0_inv
1621; GFX11-WGP-NEXT:    buffer_gl1_inv
1622; GFX11-WGP-NEXT:    s_endpgm
1623;
1624; GFX11-CU-LABEL: flat_agent_acq_rel_atomicrmw:
1625; GFX11-CU:       ; %bb.0: ; %entry
1626; GFX11-CU-NEXT:    s_clause 0x1
1627; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1628; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1629; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1630; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
1631; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
1632; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1633; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1634; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1635; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1636; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1637; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1638; GFX11-CU-NEXT:    buffer_gl0_inv
1639; GFX11-CU-NEXT:    buffer_gl1_inv
1640; GFX11-CU-NEXT:    s_endpgm
1641    i32* %out, i32 %in) {
1642entry:
1643  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") acq_rel
1644  ret void
1645}
1646
1647define amdgpu_kernel void @flat_agent_seq_cst_atomicrmw(
1648; GFX7-LABEL: flat_agent_seq_cst_atomicrmw:
1649; GFX7:       ; %bb.0: ; %entry
1650; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1651; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1652; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1653; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1654; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1655; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1656; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1657; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1658; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1659; GFX7-NEXT:    buffer_wbinvl1_vol
1660; GFX7-NEXT:    s_endpgm
1661;
1662; GFX10-WGP-LABEL: flat_agent_seq_cst_atomicrmw:
1663; GFX10-WGP:       ; %bb.0: ; %entry
1664; GFX10-WGP-NEXT:    s_clause 0x1
1665; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1666; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1667; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1668; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1669; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1670; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1671; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1672; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1673; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1674; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1675; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1676; GFX10-WGP-NEXT:    buffer_gl0_inv
1677; GFX10-WGP-NEXT:    buffer_gl1_inv
1678; GFX10-WGP-NEXT:    s_endpgm
1679;
1680; GFX10-CU-LABEL: flat_agent_seq_cst_atomicrmw:
1681; GFX10-CU:       ; %bb.0: ; %entry
1682; GFX10-CU-NEXT:    s_clause 0x1
1683; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1684; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1685; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1686; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1687; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1688; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1689; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1690; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1691; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1692; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1693; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1694; GFX10-CU-NEXT:    buffer_gl0_inv
1695; GFX10-CU-NEXT:    buffer_gl1_inv
1696; GFX10-CU-NEXT:    s_endpgm
1697;
1698; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_atomicrmw:
1699; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1700; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1701; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1702; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1703; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1704; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1705; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1706; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1707; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1708; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1709; SKIP-CACHE-INV-NEXT:    s_endpgm
1710;
1711; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_atomicrmw:
1712; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1713; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1714; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1715; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1716; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1717; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1718; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1719; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1720; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1721; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1722; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1723;
1724; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_atomicrmw:
1725; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1726; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1727; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1728; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1729; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1730; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1731; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1732; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1733; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1734; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1735; GFX90A-TGSPLIT-NEXT:    s_endpgm
1736;
1737; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_atomicrmw:
1738; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1739; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1740; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1741; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1742; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1743; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1744; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
1745; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1746; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1747; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1748; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
1749; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1750;
1751; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_atomicrmw:
1752; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1753; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1754; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1755; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1756; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1757; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1758; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
1759; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1760; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1761; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1762; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
1763; GFX940-TGSPLIT-NEXT:    s_endpgm
1764;
1765; GFX11-WGP-LABEL: flat_agent_seq_cst_atomicrmw:
1766; GFX11-WGP:       ; %bb.0: ; %entry
1767; GFX11-WGP-NEXT:    s_clause 0x1
1768; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1769; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1770; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1771; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1772; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1773; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1774; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1775; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1776; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1777; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1778; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1779; GFX11-WGP-NEXT:    buffer_gl0_inv
1780; GFX11-WGP-NEXT:    buffer_gl1_inv
1781; GFX11-WGP-NEXT:    s_endpgm
1782;
1783; GFX11-CU-LABEL: flat_agent_seq_cst_atomicrmw:
1784; GFX11-CU:       ; %bb.0: ; %entry
1785; GFX11-CU-NEXT:    s_clause 0x1
1786; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1787; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1788; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1789; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
1790; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
1791; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1792; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1793; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1794; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
1795; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1796; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1797; GFX11-CU-NEXT:    buffer_gl0_inv
1798; GFX11-CU-NEXT:    buffer_gl1_inv
1799; GFX11-CU-NEXT:    s_endpgm
1800    i32* %out, i32 %in) {
1801entry:
1802  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") seq_cst
1803  ret void
1804}
1805
1806define amdgpu_kernel void @flat_agent_acquire_ret_atomicrmw(
1807; GFX7-LABEL: flat_agent_acquire_ret_atomicrmw:
1808; GFX7:       ; %bb.0: ; %entry
1809; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1810; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1811; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1812; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1813; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1814; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1815; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1816; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1817; GFX7-NEXT:    buffer_wbinvl1_vol
1818; GFX7-NEXT:    flat_store_dword v[0:1], v2
1819; GFX7-NEXT:    s_endpgm
1820;
1821; GFX10-WGP-LABEL: flat_agent_acquire_ret_atomicrmw:
1822; GFX10-WGP:       ; %bb.0: ; %entry
1823; GFX10-WGP-NEXT:    s_clause 0x1
1824; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1825; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1826; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1827; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1828; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1829; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1830; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1831; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1832; GFX10-WGP-NEXT:    buffer_gl0_inv
1833; GFX10-WGP-NEXT:    buffer_gl1_inv
1834; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1835; GFX10-WGP-NEXT:    s_endpgm
1836;
1837; GFX10-CU-LABEL: flat_agent_acquire_ret_atomicrmw:
1838; GFX10-CU:       ; %bb.0: ; %entry
1839; GFX10-CU-NEXT:    s_clause 0x1
1840; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1841; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1842; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1843; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1844; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1845; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1846; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1847; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1848; GFX10-CU-NEXT:    buffer_gl0_inv
1849; GFX10-CU-NEXT:    buffer_gl1_inv
1850; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1851; GFX10-CU-NEXT:    s_endpgm
1852;
1853; SKIP-CACHE-INV-LABEL: flat_agent_acquire_ret_atomicrmw:
1854; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1855; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1856; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1857; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1858; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1859; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1860; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1861; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1862; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1863; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1864; SKIP-CACHE-INV-NEXT:    s_endpgm
1865;
1866; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_ret_atomicrmw:
1867; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1868; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1869; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1870; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1871; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1872; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1873; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1874; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1875; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1876; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1877; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1878;
1879; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_ret_atomicrmw:
1880; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1881; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1882; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1883; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1884; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1885; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1886; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1887; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1888; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1889; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1890; GFX90A-TGSPLIT-NEXT:    s_endpgm
1891;
1892; GFX940-NOTTGSPLIT-LABEL: flat_agent_acquire_ret_atomicrmw:
1893; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1894; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1895; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1896; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1897; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1898; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1899; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
1900; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1901; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
1902; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1903; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1904;
1905; GFX940-TGSPLIT-LABEL: flat_agent_acquire_ret_atomicrmw:
1906; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1907; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1908; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1909; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1910; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1911; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1912; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
1913; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1914; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
1915; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1916; GFX940-TGSPLIT-NEXT:    s_endpgm
1917;
1918; GFX11-WGP-LABEL: flat_agent_acquire_ret_atomicrmw:
1919; GFX11-WGP:       ; %bb.0: ; %entry
1920; GFX11-WGP-NEXT:    s_clause 0x1
1921; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1922; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
1923; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1924; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
1925; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
1926; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
1927; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
1928; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1929; GFX11-WGP-NEXT:    buffer_gl0_inv
1930; GFX11-WGP-NEXT:    buffer_gl1_inv
1931; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
1932; GFX11-WGP-NEXT:    s_endpgm
1933;
1934; GFX11-CU-LABEL: flat_agent_acquire_ret_atomicrmw:
1935; GFX11-CU:       ; %bb.0: ; %entry
1936; GFX11-CU-NEXT:    s_clause 0x1
1937; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
1938; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
1939; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
1940; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
1941; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
1942; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
1943; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
1944; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1945; GFX11-CU-NEXT:    buffer_gl0_inv
1946; GFX11-CU-NEXT:    buffer_gl1_inv
1947; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
1948; GFX11-CU-NEXT:    s_endpgm
1949    i32* %out, i32 %in) {
1950entry:
1951  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") acquire
1952  store i32 %val, i32* %out, align 4
1953  ret void
1954}
1955
1956define amdgpu_kernel void @flat_agent_acq_rel_ret_atomicrmw(
1957; GFX7-LABEL: flat_agent_acq_rel_ret_atomicrmw:
1958; GFX7:       ; %bb.0: ; %entry
1959; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1960; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1961; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1962; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1963; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1964; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1965; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1966; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1967; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1968; GFX7-NEXT:    buffer_wbinvl1_vol
1969; GFX7-NEXT:    flat_store_dword v[0:1], v2
1970; GFX7-NEXT:    s_endpgm
1971;
1972; GFX10-WGP-LABEL: flat_agent_acq_rel_ret_atomicrmw:
1973; GFX10-WGP:       ; %bb.0: ; %entry
1974; GFX10-WGP-NEXT:    s_clause 0x1
1975; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1976; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1977; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1978; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1979; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1980; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1981; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1982; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1983; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1984; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1985; GFX10-WGP-NEXT:    buffer_gl0_inv
1986; GFX10-WGP-NEXT:    buffer_gl1_inv
1987; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1988; GFX10-WGP-NEXT:    s_endpgm
1989;
1990; GFX10-CU-LABEL: flat_agent_acq_rel_ret_atomicrmw:
1991; GFX10-CU:       ; %bb.0: ; %entry
1992; GFX10-CU-NEXT:    s_clause 0x1
1993; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1994; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1995; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1996; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1997; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1998; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1999; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2000; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2001; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2002; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2003; GFX10-CU-NEXT:    buffer_gl0_inv
2004; GFX10-CU-NEXT:    buffer_gl1_inv
2005; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
2006; GFX10-CU-NEXT:    s_endpgm
2007;
2008; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_ret_atomicrmw:
2009; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2010; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2011; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
2012; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2013; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2014; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2015; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2016; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2017; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2018; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2019; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
2020; SKIP-CACHE-INV-NEXT:    s_endpgm
2021;
2022; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_ret_atomicrmw:
2023; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2024; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2025; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
2026; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2027; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2028; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
2029; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2030; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2031; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2032; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2033; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2034; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2035;
2036; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_ret_atomicrmw:
2037; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2038; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2039; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
2040; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2041; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2042; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
2043; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2044; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2045; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2046; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2047; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2048; GFX90A-TGSPLIT-NEXT:    s_endpgm
2049;
2050; GFX940-NOTTGSPLIT-LABEL: flat_agent_acq_rel_ret_atomicrmw:
2051; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2052; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2053; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
2054; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2055; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2056; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
2057; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
2058; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2059; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
2060; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2061; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
2062; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2063; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2064;
2065; GFX940-TGSPLIT-LABEL: flat_agent_acq_rel_ret_atomicrmw:
2066; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2067; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2068; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
2069; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2070; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2071; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
2072; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
2073; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2074; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
2075; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2076; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
2077; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2078; GFX940-TGSPLIT-NEXT:    s_endpgm
2079;
2080; GFX11-WGP-LABEL: flat_agent_acq_rel_ret_atomicrmw:
2081; GFX11-WGP:       ; %bb.0: ; %entry
2082; GFX11-WGP-NEXT:    s_clause 0x1
2083; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2084; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
2085; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2086; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2087; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2088; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2089; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2090; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2091; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
2092; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2093; GFX11-WGP-NEXT:    buffer_gl0_inv
2094; GFX11-WGP-NEXT:    buffer_gl1_inv
2095; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
2096; GFX11-WGP-NEXT:    s_endpgm
2097;
2098; GFX11-CU-LABEL: flat_agent_acq_rel_ret_atomicrmw:
2099; GFX11-CU:       ; %bb.0: ; %entry
2100; GFX11-CU-NEXT:    s_clause 0x1
2101; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2102; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
2103; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2104; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2105; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2106; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2107; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2108; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2109; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
2110; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2111; GFX11-CU-NEXT:    buffer_gl0_inv
2112; GFX11-CU-NEXT:    buffer_gl1_inv
2113; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
2114; GFX11-CU-NEXT:    s_endpgm
2115    i32* %out, i32 %in) {
2116entry:
2117  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") acq_rel
2118  store i32 %val, i32* %out, align 4
2119  ret void
2120}
2121
2122define amdgpu_kernel void @flat_agent_seq_cst_ret_atomicrmw(
2123; GFX7-LABEL: flat_agent_seq_cst_ret_atomicrmw:
2124; GFX7:       ; %bb.0: ; %entry
2125; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2126; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
2127; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2128; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2129; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2130; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2131; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2132; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2133; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2134; GFX7-NEXT:    buffer_wbinvl1_vol
2135; GFX7-NEXT:    flat_store_dword v[0:1], v2
2136; GFX7-NEXT:    s_endpgm
2137;
2138; GFX10-WGP-LABEL: flat_agent_seq_cst_ret_atomicrmw:
2139; GFX10-WGP:       ; %bb.0: ; %entry
2140; GFX10-WGP-NEXT:    s_clause 0x1
2141; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2142; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
2143; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2144; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2145; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2146; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2147; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2148; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2149; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2150; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2151; GFX10-WGP-NEXT:    buffer_gl0_inv
2152; GFX10-WGP-NEXT:    buffer_gl1_inv
2153; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
2154; GFX10-WGP-NEXT:    s_endpgm
2155;
2156; GFX10-CU-LABEL: flat_agent_seq_cst_ret_atomicrmw:
2157; GFX10-CU:       ; %bb.0: ; %entry
2158; GFX10-CU-NEXT:    s_clause 0x1
2159; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2160; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
2161; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2162; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2163; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2164; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2165; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2166; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2167; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2168; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2169; GFX10-CU-NEXT:    buffer_gl0_inv
2170; GFX10-CU-NEXT:    buffer_gl1_inv
2171; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
2172; GFX10-CU-NEXT:    s_endpgm
2173;
2174; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_ret_atomicrmw:
2175; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2176; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2177; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
2178; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2179; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2180; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2181; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2182; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2183; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2184; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2185; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
2186; SKIP-CACHE-INV-NEXT:    s_endpgm
2187;
2188; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_ret_atomicrmw:
2189; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2190; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2191; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
2192; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2193; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2194; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
2195; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2196; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2197; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2198; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2199; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2200; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2201;
2202; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_ret_atomicrmw:
2203; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2204; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2205; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
2206; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2207; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2208; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
2209; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2210; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
2211; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2212; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2213; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2214; GFX90A-TGSPLIT-NEXT:    s_endpgm
2215;
2216; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_ret_atomicrmw:
2217; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2218; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2219; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
2220; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2221; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2222; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
2223; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
2224; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2225; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
2226; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2227; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
2228; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2229; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2230;
2231; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_ret_atomicrmw:
2232; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2233; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2234; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
2235; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2236; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2237; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
2238; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
2239; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2240; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
2241; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2242; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
2243; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2244; GFX940-TGSPLIT-NEXT:    s_endpgm
2245;
2246; GFX11-WGP-LABEL: flat_agent_seq_cst_ret_atomicrmw:
2247; GFX11-WGP:       ; %bb.0: ; %entry
2248; GFX11-WGP-NEXT:    s_clause 0x1
2249; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2250; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
2251; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2252; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2253; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2254; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2255; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2256; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2257; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
2258; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2259; GFX11-WGP-NEXT:    buffer_gl0_inv
2260; GFX11-WGP-NEXT:    buffer_gl1_inv
2261; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
2262; GFX11-WGP-NEXT:    s_endpgm
2263;
2264; GFX11-CU-LABEL: flat_agent_seq_cst_ret_atomicrmw:
2265; GFX11-CU:       ; %bb.0: ; %entry
2266; GFX11-CU-NEXT:    s_clause 0x1
2267; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2268; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
2269; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2270; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2271; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2272; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2273; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2274; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2275; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
2276; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2277; GFX11-CU-NEXT:    buffer_gl0_inv
2278; GFX11-CU-NEXT:    buffer_gl1_inv
2279; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
2280; GFX11-CU-NEXT:    s_endpgm
2281    i32* %out, i32 %in) {
2282entry:
2283  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") seq_cst
2284  store i32 %val, i32* %out, align 4
2285  ret void
2286}
2287
2288define amdgpu_kernel void @flat_agent_monotonic_monotonic_cmpxchg(
2289; GFX7-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
2290; GFX7:       ; %bb.0: ; %entry
2291; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2292; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2293; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2294; GFX7-NEXT:    s_add_u32 s0, s0, 16
2295; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2296; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2297; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2298; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2299; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2300; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2301; GFX7-NEXT:    s_endpgm
2302;
2303; GFX10-WGP-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
2304; GFX10-WGP:       ; %bb.0: ; %entry
2305; GFX10-WGP-NEXT:    s_clause 0x1
2306; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2307; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2308; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2309; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2310; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2311; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2312; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2313; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2314; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2315; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2316; GFX10-WGP-NEXT:    s_endpgm
2317;
2318; GFX10-CU-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
2319; GFX10-CU:       ; %bb.0: ; %entry
2320; GFX10-CU-NEXT:    s_clause 0x1
2321; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2322; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2323; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2324; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2325; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2326; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2327; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2328; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2329; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2330; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2331; GFX10-CU-NEXT:    s_endpgm
2332;
2333; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
2334; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2335; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2336; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2337; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2338; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2339; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2340; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2341; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2342; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2343; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2344; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2345; SKIP-CACHE-INV-NEXT:    s_endpgm
2346;
2347; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
2348; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2349; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2350; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2351; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2352; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2353; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2354; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2355; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2356;
2357; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
2358; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2359; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2360; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2361; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2362; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2363; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2364; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2365; GFX90A-TGSPLIT-NEXT:    s_endpgm
2366;
2367; GFX940-NOTTGSPLIT-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
2368; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2369; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2370; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2371; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2372; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2373; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2374; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2375; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2376;
2377; GFX940-TGSPLIT-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
2378; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2379; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2380; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2381; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2382; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2383; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2384; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2385; GFX940-TGSPLIT-NEXT:    s_endpgm
2386;
2387; GFX11-WGP-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
2388; GFX11-WGP:       ; %bb.0: ; %entry
2389; GFX11-WGP-NEXT:    s_clause 0x1
2390; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2391; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2392; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2393; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2394; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
2395; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2396; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2397; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2398; GFX11-WGP-NEXT:    s_endpgm
2399;
2400; GFX11-CU-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
2401; GFX11-CU:       ; %bb.0: ; %entry
2402; GFX11-CU-NEXT:    s_clause 0x1
2403; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2404; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2405; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2406; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2407; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
2408; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2409; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2410; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2411; GFX11-CU-NEXT:    s_endpgm
2412    i32* %out, i32 %in, i32 %old) {
2413entry:
2414  %gep = getelementptr i32, i32* %out, i32 4
2415  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") monotonic monotonic
2416  ret void
2417}
2418
2419define amdgpu_kernel void @flat_agent_acquire_monotonic_cmpxchg(
2420; GFX7-LABEL: flat_agent_acquire_monotonic_cmpxchg:
2421; GFX7:       ; %bb.0: ; %entry
2422; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2423; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2424; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2425; GFX7-NEXT:    s_add_u32 s0, s0, 16
2426; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2427; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2428; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2429; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2430; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2431; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2432; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2433; GFX7-NEXT:    buffer_wbinvl1_vol
2434; GFX7-NEXT:    s_endpgm
2435;
2436; GFX10-WGP-LABEL: flat_agent_acquire_monotonic_cmpxchg:
2437; GFX10-WGP:       ; %bb.0: ; %entry
2438; GFX10-WGP-NEXT:    s_clause 0x1
2439; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2440; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2441; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2442; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2443; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2444; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2445; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2446; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2447; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2448; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2449; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2450; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2451; GFX10-WGP-NEXT:    buffer_gl0_inv
2452; GFX10-WGP-NEXT:    buffer_gl1_inv
2453; GFX10-WGP-NEXT:    s_endpgm
2454;
2455; GFX10-CU-LABEL: flat_agent_acquire_monotonic_cmpxchg:
2456; GFX10-CU:       ; %bb.0: ; %entry
2457; GFX10-CU-NEXT:    s_clause 0x1
2458; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2459; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2460; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2461; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2462; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2463; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2464; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2465; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2466; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2467; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2468; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2469; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2470; GFX10-CU-NEXT:    buffer_gl0_inv
2471; GFX10-CU-NEXT:    buffer_gl1_inv
2472; GFX10-CU-NEXT:    s_endpgm
2473;
2474; SKIP-CACHE-INV-LABEL: flat_agent_acquire_monotonic_cmpxchg:
2475; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2476; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2477; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2478; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2479; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2480; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2481; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2482; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2483; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2484; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2485; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2486; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2487; SKIP-CACHE-INV-NEXT:    s_endpgm
2488;
2489; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_monotonic_cmpxchg:
2490; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2491; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2492; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2493; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2494; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2495; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2496; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2497; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2498; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2499; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2500;
2501; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_monotonic_cmpxchg:
2502; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2503; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2504; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2505; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2506; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2507; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2508; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2509; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2510; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2511; GFX90A-TGSPLIT-NEXT:    s_endpgm
2512;
2513; GFX940-NOTTGSPLIT-LABEL: flat_agent_acquire_monotonic_cmpxchg:
2514; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2515; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2516; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2517; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2518; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2519; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2520; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2521; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2522; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
2523; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2524;
2525; GFX940-TGSPLIT-LABEL: flat_agent_acquire_monotonic_cmpxchg:
2526; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2527; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2528; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2529; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2530; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2531; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2532; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2533; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2534; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
2535; GFX940-TGSPLIT-NEXT:    s_endpgm
2536;
2537; GFX11-WGP-LABEL: flat_agent_acquire_monotonic_cmpxchg:
2538; GFX11-WGP:       ; %bb.0: ; %entry
2539; GFX11-WGP-NEXT:    s_clause 0x1
2540; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2541; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2542; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2543; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2544; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
2545; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2546; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2547; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2548; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2549; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2550; GFX11-WGP-NEXT:    buffer_gl0_inv
2551; GFX11-WGP-NEXT:    buffer_gl1_inv
2552; GFX11-WGP-NEXT:    s_endpgm
2553;
2554; GFX11-CU-LABEL: flat_agent_acquire_monotonic_cmpxchg:
2555; GFX11-CU:       ; %bb.0: ; %entry
2556; GFX11-CU-NEXT:    s_clause 0x1
2557; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2558; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2559; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2560; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2561; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
2562; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2563; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2564; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2565; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2566; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2567; GFX11-CU-NEXT:    buffer_gl0_inv
2568; GFX11-CU-NEXT:    buffer_gl1_inv
2569; GFX11-CU-NEXT:    s_endpgm
2570    i32* %out, i32 %in, i32 %old) {
2571entry:
2572  %gep = getelementptr i32, i32* %out, i32 4
2573  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire monotonic
2574  ret void
2575}
2576
2577define amdgpu_kernel void @flat_agent_release_monotonic_cmpxchg(
2578; GFX7-LABEL: flat_agent_release_monotonic_cmpxchg:
2579; GFX7:       ; %bb.0: ; %entry
2580; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2581; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2582; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2583; GFX7-NEXT:    s_add_u32 s0, s0, 16
2584; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2585; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2586; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2587; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2588; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2589; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2590; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2591; GFX7-NEXT:    s_endpgm
2592;
2593; GFX10-WGP-LABEL: flat_agent_release_monotonic_cmpxchg:
2594; GFX10-WGP:       ; %bb.0: ; %entry
2595; GFX10-WGP-NEXT:    s_clause 0x1
2596; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2597; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2598; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2599; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2600; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2601; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2602; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2603; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2604; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2605; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2606; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2607; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2608; GFX10-WGP-NEXT:    s_endpgm
2609;
2610; GFX10-CU-LABEL: flat_agent_release_monotonic_cmpxchg:
2611; GFX10-CU:       ; %bb.0: ; %entry
2612; GFX10-CU-NEXT:    s_clause 0x1
2613; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2614; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2615; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2616; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2617; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2618; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2619; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2620; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2621; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2622; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2623; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2624; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2625; GFX10-CU-NEXT:    s_endpgm
2626;
2627; SKIP-CACHE-INV-LABEL: flat_agent_release_monotonic_cmpxchg:
2628; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2629; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2630; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2631; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2632; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2633; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2634; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2635; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2636; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2637; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2638; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2639; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2640; SKIP-CACHE-INV-NEXT:    s_endpgm
2641;
2642; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_monotonic_cmpxchg:
2643; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2644; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2645; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2646; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2647; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2648; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2649; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2650; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2651; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2652;
2653; GFX90A-TGSPLIT-LABEL: flat_agent_release_monotonic_cmpxchg:
2654; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2655; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2656; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2657; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2658; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2659; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2660; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2661; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2662; GFX90A-TGSPLIT-NEXT:    s_endpgm
2663;
2664; GFX940-NOTTGSPLIT-LABEL: flat_agent_release_monotonic_cmpxchg:
2665; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2666; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2667; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2668; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2669; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2670; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2671; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
2672; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2673; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2674; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2675;
2676; GFX940-TGSPLIT-LABEL: flat_agent_release_monotonic_cmpxchg:
2677; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2678; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2679; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2680; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2681; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2682; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2683; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
2684; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2685; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2686; GFX940-TGSPLIT-NEXT:    s_endpgm
2687;
2688; GFX11-WGP-LABEL: flat_agent_release_monotonic_cmpxchg:
2689; GFX11-WGP:       ; %bb.0: ; %entry
2690; GFX11-WGP-NEXT:    s_clause 0x1
2691; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2692; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2693; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2694; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2695; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
2696; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2697; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2698; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2699; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2700; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2701; GFX11-WGP-NEXT:    s_endpgm
2702;
2703; GFX11-CU-LABEL: flat_agent_release_monotonic_cmpxchg:
2704; GFX11-CU:       ; %bb.0: ; %entry
2705; GFX11-CU-NEXT:    s_clause 0x1
2706; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2707; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2708; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2709; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2710; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
2711; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2712; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2713; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2714; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2715; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2716; GFX11-CU-NEXT:    s_endpgm
2717    i32* %out, i32 %in, i32 %old) {
2718entry:
2719  %gep = getelementptr i32, i32* %out, i32 4
2720  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release monotonic
2721  ret void
2722}
2723
2724define amdgpu_kernel void @flat_agent_acq_rel_monotonic_cmpxchg(
2725; GFX7-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
2726; GFX7:       ; %bb.0: ; %entry
2727; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2728; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2729; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2730; GFX7-NEXT:    s_add_u32 s0, s0, 16
2731; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2732; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2733; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2734; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2735; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2736; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2737; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2738; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2739; GFX7-NEXT:    buffer_wbinvl1_vol
2740; GFX7-NEXT:    s_endpgm
2741;
2742; GFX10-WGP-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
2743; GFX10-WGP:       ; %bb.0: ; %entry
2744; GFX10-WGP-NEXT:    s_clause 0x1
2745; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2746; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2747; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2748; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2749; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2750; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2751; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2752; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2753; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2754; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2755; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2756; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2757; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2758; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2759; GFX10-WGP-NEXT:    buffer_gl0_inv
2760; GFX10-WGP-NEXT:    buffer_gl1_inv
2761; GFX10-WGP-NEXT:    s_endpgm
2762;
2763; GFX10-CU-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
2764; GFX10-CU:       ; %bb.0: ; %entry
2765; GFX10-CU-NEXT:    s_clause 0x1
2766; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2767; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2768; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2769; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2770; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2771; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2772; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2773; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2774; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2775; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2776; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2777; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2778; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2779; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2780; GFX10-CU-NEXT:    buffer_gl0_inv
2781; GFX10-CU-NEXT:    buffer_gl1_inv
2782; GFX10-CU-NEXT:    s_endpgm
2783;
2784; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
2785; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2786; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2787; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2788; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2789; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2790; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2791; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2792; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2793; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2794; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2795; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2796; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2797; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2798; SKIP-CACHE-INV-NEXT:    s_endpgm
2799;
2800; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
2801; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2802; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2803; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2804; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2805; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2806; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2807; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2808; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2809; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2810; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2811; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2812;
2813; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
2814; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2815; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2816; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2817; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2818; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2819; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2820; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2821; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2822; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2823; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2824; GFX90A-TGSPLIT-NEXT:    s_endpgm
2825;
2826; GFX940-NOTTGSPLIT-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
2827; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2828; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2829; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2830; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2831; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2832; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2833; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
2834; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2835; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2836; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2837; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
2838; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2839;
2840; GFX940-TGSPLIT-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
2841; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2842; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2843; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2844; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2845; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2846; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2847; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
2848; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2849; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2850; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2851; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
2852; GFX940-TGSPLIT-NEXT:    s_endpgm
2853;
2854; GFX11-WGP-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
2855; GFX11-WGP:       ; %bb.0: ; %entry
2856; GFX11-WGP-NEXT:    s_clause 0x1
2857; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2858; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2859; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2860; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
2861; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
2862; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
2863; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
2864; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2865; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2866; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2867; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2868; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2869; GFX11-WGP-NEXT:    buffer_gl0_inv
2870; GFX11-WGP-NEXT:    buffer_gl1_inv
2871; GFX11-WGP-NEXT:    s_endpgm
2872;
2873; GFX11-CU-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
2874; GFX11-CU:       ; %bb.0: ; %entry
2875; GFX11-CU-NEXT:    s_clause 0x1
2876; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
2877; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
2878; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2879; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
2880; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
2881; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
2882; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
2883; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2884; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2885; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2886; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
2887; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2888; GFX11-CU-NEXT:    buffer_gl0_inv
2889; GFX11-CU-NEXT:    buffer_gl1_inv
2890; GFX11-CU-NEXT:    s_endpgm
2891    i32* %out, i32 %in, i32 %old) {
2892entry:
2893  %gep = getelementptr i32, i32* %out, i32 4
2894  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel monotonic
2895  ret void
2896}
2897
2898define amdgpu_kernel void @flat_agent_seq_cst_monotonic_cmpxchg(
2899; GFX7-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
2900; GFX7:       ; %bb.0: ; %entry
2901; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2902; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2903; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2904; GFX7-NEXT:    s_add_u32 s0, s0, 16
2905; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2906; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2907; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2908; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2909; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2910; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2911; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2912; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2913; GFX7-NEXT:    buffer_wbinvl1_vol
2914; GFX7-NEXT:    s_endpgm
2915;
2916; GFX10-WGP-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
2917; GFX10-WGP:       ; %bb.0: ; %entry
2918; GFX10-WGP-NEXT:    s_clause 0x1
2919; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2920; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2921; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2922; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2923; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2924; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2925; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2926; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2927; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2928; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2929; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2930; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2931; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2932; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2933; GFX10-WGP-NEXT:    buffer_gl0_inv
2934; GFX10-WGP-NEXT:    buffer_gl1_inv
2935; GFX10-WGP-NEXT:    s_endpgm
2936;
2937; GFX10-CU-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
2938; GFX10-CU:       ; %bb.0: ; %entry
2939; GFX10-CU-NEXT:    s_clause 0x1
2940; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2941; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2942; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2943; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2944; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2945; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2946; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2947; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2948; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2949; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2950; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2951; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2952; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2953; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2954; GFX10-CU-NEXT:    buffer_gl0_inv
2955; GFX10-CU-NEXT:    buffer_gl1_inv
2956; GFX10-CU-NEXT:    s_endpgm
2957;
2958; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
2959; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2960; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2961; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2962; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2963; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2964; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2965; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2966; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2967; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2968; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2969; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2970; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2971; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2972; SKIP-CACHE-INV-NEXT:    s_endpgm
2973;
2974; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
2975; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2976; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2977; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2978; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2979; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2980; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2981; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2982; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2983; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2984; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2985; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2986;
2987; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
2988; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2989; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2990; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2991; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2992; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2993; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2994; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2995; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2996; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2997; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2998; GFX90A-TGSPLIT-NEXT:    s_endpgm
2999;
3000; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
3001; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3002; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3003; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3004; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3005; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3006; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3007; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
3008; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3009; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3010; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3011; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
3012; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3013;
3014; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
3015; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3016; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3017; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3018; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3019; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3020; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3021; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
3022; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3023; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3024; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3025; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
3026; GFX940-TGSPLIT-NEXT:    s_endpgm
3027;
3028; GFX11-WGP-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
3029; GFX11-WGP:       ; %bb.0: ; %entry
3030; GFX11-WGP-NEXT:    s_clause 0x1
3031; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3032; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3033; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3034; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3035; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
3036; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3037; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
3038; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3039; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3040; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3041; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3042; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3043; GFX11-WGP-NEXT:    buffer_gl0_inv
3044; GFX11-WGP-NEXT:    buffer_gl1_inv
3045; GFX11-WGP-NEXT:    s_endpgm
3046;
3047; GFX11-CU-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
3048; GFX11-CU:       ; %bb.0: ; %entry
3049; GFX11-CU-NEXT:    s_clause 0x1
3050; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3051; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3052; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3053; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3054; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
3055; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3056; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
3057; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3058; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3059; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3060; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3061; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3062; GFX11-CU-NEXT:    buffer_gl0_inv
3063; GFX11-CU-NEXT:    buffer_gl1_inv
3064; GFX11-CU-NEXT:    s_endpgm
3065    i32* %out, i32 %in, i32 %old) {
3066entry:
3067  %gep = getelementptr i32, i32* %out, i32 4
3068  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst monotonic
3069  ret void
3070}
3071
3072define amdgpu_kernel void @flat_agent_monotonic_acquire_cmpxchg(
3073; GFX7-LABEL: flat_agent_monotonic_acquire_cmpxchg:
3074; GFX7:       ; %bb.0: ; %entry
3075; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3076; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3077; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3078; GFX7-NEXT:    s_add_u32 s0, s0, 16
3079; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3080; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3081; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3082; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3083; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3084; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3085; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3086; GFX7-NEXT:    buffer_wbinvl1_vol
3087; GFX7-NEXT:    s_endpgm
3088;
3089; GFX10-WGP-LABEL: flat_agent_monotonic_acquire_cmpxchg:
3090; GFX10-WGP:       ; %bb.0: ; %entry
3091; GFX10-WGP-NEXT:    s_clause 0x1
3092; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3093; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3094; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3095; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3096; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3097; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3098; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3099; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3100; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3101; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3102; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3103; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3104; GFX10-WGP-NEXT:    buffer_gl0_inv
3105; GFX10-WGP-NEXT:    buffer_gl1_inv
3106; GFX10-WGP-NEXT:    s_endpgm
3107;
3108; GFX10-CU-LABEL: flat_agent_monotonic_acquire_cmpxchg:
3109; GFX10-CU:       ; %bb.0: ; %entry
3110; GFX10-CU-NEXT:    s_clause 0x1
3111; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3112; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3113; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3114; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3115; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3116; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3117; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3118; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3119; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3120; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3121; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3122; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3123; GFX10-CU-NEXT:    buffer_gl0_inv
3124; GFX10-CU-NEXT:    buffer_gl1_inv
3125; GFX10-CU-NEXT:    s_endpgm
3126;
3127; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_acquire_cmpxchg:
3128; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3129; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3130; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3131; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3132; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3133; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3134; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3135; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3136; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3137; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3138; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3139; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3140; SKIP-CACHE-INV-NEXT:    s_endpgm
3141;
3142; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_acquire_cmpxchg:
3143; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3144; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3145; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3146; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3147; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3148; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3149; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3150; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3151; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3152; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3153;
3154; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_acquire_cmpxchg:
3155; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3156; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3157; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3158; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3159; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3160; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3161; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3162; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3163; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3164; GFX90A-TGSPLIT-NEXT:    s_endpgm
3165;
3166; GFX940-NOTTGSPLIT-LABEL: flat_agent_monotonic_acquire_cmpxchg:
3167; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3168; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3169; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3170; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3171; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3172; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3173; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3174; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3175; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
3176; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3177;
3178; GFX940-TGSPLIT-LABEL: flat_agent_monotonic_acquire_cmpxchg:
3179; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3180; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3181; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3182; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3183; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3184; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3185; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3186; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3187; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
3188; GFX940-TGSPLIT-NEXT:    s_endpgm
3189;
3190; GFX11-WGP-LABEL: flat_agent_monotonic_acquire_cmpxchg:
3191; GFX11-WGP:       ; %bb.0: ; %entry
3192; GFX11-WGP-NEXT:    s_clause 0x1
3193; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3194; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3195; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3196; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3197; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
3198; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3199; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
3200; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3201; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3202; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3203; GFX11-WGP-NEXT:    buffer_gl0_inv
3204; GFX11-WGP-NEXT:    buffer_gl1_inv
3205; GFX11-WGP-NEXT:    s_endpgm
3206;
3207; GFX11-CU-LABEL: flat_agent_monotonic_acquire_cmpxchg:
3208; GFX11-CU:       ; %bb.0: ; %entry
3209; GFX11-CU-NEXT:    s_clause 0x1
3210; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3211; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3212; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3213; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3214; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
3215; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3216; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
3217; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3218; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3219; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3220; GFX11-CU-NEXT:    buffer_gl0_inv
3221; GFX11-CU-NEXT:    buffer_gl1_inv
3222; GFX11-CU-NEXT:    s_endpgm
3223    i32* %out, i32 %in, i32 %old) {
3224entry:
3225  %gep = getelementptr i32, i32* %out, i32 4
3226  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") monotonic acquire
3227  ret void
3228}
3229
3230define amdgpu_kernel void @flat_agent_acquire_acquire_cmpxchg(
3231; GFX7-LABEL: flat_agent_acquire_acquire_cmpxchg:
3232; GFX7:       ; %bb.0: ; %entry
3233; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3234; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3235; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3236; GFX7-NEXT:    s_add_u32 s0, s0, 16
3237; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3238; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3239; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3240; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3241; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3242; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3243; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3244; GFX7-NEXT:    buffer_wbinvl1_vol
3245; GFX7-NEXT:    s_endpgm
3246;
3247; GFX10-WGP-LABEL: flat_agent_acquire_acquire_cmpxchg:
3248; GFX10-WGP:       ; %bb.0: ; %entry
3249; GFX10-WGP-NEXT:    s_clause 0x1
3250; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3251; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3252; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3253; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3254; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3255; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3256; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3257; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3258; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3259; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3260; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3261; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3262; GFX10-WGP-NEXT:    buffer_gl0_inv
3263; GFX10-WGP-NEXT:    buffer_gl1_inv
3264; GFX10-WGP-NEXT:    s_endpgm
3265;
3266; GFX10-CU-LABEL: flat_agent_acquire_acquire_cmpxchg:
3267; GFX10-CU:       ; %bb.0: ; %entry
3268; GFX10-CU-NEXT:    s_clause 0x1
3269; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3270; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3271; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3272; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3273; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3274; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3275; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3276; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3277; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3278; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3279; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3280; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3281; GFX10-CU-NEXT:    buffer_gl0_inv
3282; GFX10-CU-NEXT:    buffer_gl1_inv
3283; GFX10-CU-NEXT:    s_endpgm
3284;
3285; SKIP-CACHE-INV-LABEL: flat_agent_acquire_acquire_cmpxchg:
3286; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3287; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3288; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3289; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3290; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3291; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3292; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3293; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3294; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3295; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3296; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3297; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3298; SKIP-CACHE-INV-NEXT:    s_endpgm
3299;
3300; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_acquire_cmpxchg:
3301; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3302; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3303; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3304; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3305; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3306; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3307; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3308; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3309; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3310; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3311;
3312; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_acquire_cmpxchg:
3313; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3314; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3315; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3316; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3317; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3318; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3319; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3320; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3321; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3322; GFX90A-TGSPLIT-NEXT:    s_endpgm
3323;
3324; GFX940-NOTTGSPLIT-LABEL: flat_agent_acquire_acquire_cmpxchg:
3325; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3326; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3327; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3328; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3329; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3330; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3331; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3332; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3333; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
3334; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3335;
3336; GFX940-TGSPLIT-LABEL: flat_agent_acquire_acquire_cmpxchg:
3337; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3338; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3339; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3340; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3341; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3342; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3343; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3344; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3345; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
3346; GFX940-TGSPLIT-NEXT:    s_endpgm
3347;
3348; GFX11-WGP-LABEL: flat_agent_acquire_acquire_cmpxchg:
3349; GFX11-WGP:       ; %bb.0: ; %entry
3350; GFX11-WGP-NEXT:    s_clause 0x1
3351; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3352; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3353; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3354; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3355; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
3356; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3357; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
3358; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3359; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3360; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3361; GFX11-WGP-NEXT:    buffer_gl0_inv
3362; GFX11-WGP-NEXT:    buffer_gl1_inv
3363; GFX11-WGP-NEXT:    s_endpgm
3364;
3365; GFX11-CU-LABEL: flat_agent_acquire_acquire_cmpxchg:
3366; GFX11-CU:       ; %bb.0: ; %entry
3367; GFX11-CU-NEXT:    s_clause 0x1
3368; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3369; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3370; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3371; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3372; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
3373; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3374; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
3375; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3376; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3377; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3378; GFX11-CU-NEXT:    buffer_gl0_inv
3379; GFX11-CU-NEXT:    buffer_gl1_inv
3380; GFX11-CU-NEXT:    s_endpgm
3381    i32* %out, i32 %in, i32 %old) {
3382entry:
3383  %gep = getelementptr i32, i32* %out, i32 4
3384  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire acquire
3385  ret void
3386}
3387
3388define amdgpu_kernel void @flat_agent_release_acquire_cmpxchg(
3389; GFX7-LABEL: flat_agent_release_acquire_cmpxchg:
3390; GFX7:       ; %bb.0: ; %entry
3391; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3392; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3393; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3394; GFX7-NEXT:    s_add_u32 s0, s0, 16
3395; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3396; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3397; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3398; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3399; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3400; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3401; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3402; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3403; GFX7-NEXT:    buffer_wbinvl1_vol
3404; GFX7-NEXT:    s_endpgm
3405;
3406; GFX10-WGP-LABEL: flat_agent_release_acquire_cmpxchg:
3407; GFX10-WGP:       ; %bb.0: ; %entry
3408; GFX10-WGP-NEXT:    s_clause 0x1
3409; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3410; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3411; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3412; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3413; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3414; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3415; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3416; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3417; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3418; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3419; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3420; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3421; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3422; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3423; GFX10-WGP-NEXT:    buffer_gl0_inv
3424; GFX10-WGP-NEXT:    buffer_gl1_inv
3425; GFX10-WGP-NEXT:    s_endpgm
3426;
3427; GFX10-CU-LABEL: flat_agent_release_acquire_cmpxchg:
3428; GFX10-CU:       ; %bb.0: ; %entry
3429; GFX10-CU-NEXT:    s_clause 0x1
3430; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3431; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3432; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3433; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3434; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3435; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3436; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3437; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3438; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3439; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3440; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3441; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3442; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3443; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3444; GFX10-CU-NEXT:    buffer_gl0_inv
3445; GFX10-CU-NEXT:    buffer_gl1_inv
3446; GFX10-CU-NEXT:    s_endpgm
3447;
3448; SKIP-CACHE-INV-LABEL: flat_agent_release_acquire_cmpxchg:
3449; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3450; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3451; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3452; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3453; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3454; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3455; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3456; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3457; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3458; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3459; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3460; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3461; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3462; SKIP-CACHE-INV-NEXT:    s_endpgm
3463;
3464; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_acquire_cmpxchg:
3465; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3466; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3467; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3468; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3469; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3470; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3471; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3472; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3473; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3474; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3475; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3476;
3477; GFX90A-TGSPLIT-LABEL: flat_agent_release_acquire_cmpxchg:
3478; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3479; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3480; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3481; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3482; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3483; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3484; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3485; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3486; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3487; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3488; GFX90A-TGSPLIT-NEXT:    s_endpgm
3489;
3490; GFX940-NOTTGSPLIT-LABEL: flat_agent_release_acquire_cmpxchg:
3491; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3492; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3493; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3494; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3495; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3496; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3497; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
3498; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3499; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3500; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3501; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
3502; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3503;
3504; GFX940-TGSPLIT-LABEL: flat_agent_release_acquire_cmpxchg:
3505; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3506; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3507; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3508; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3509; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3510; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3511; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
3512; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3513; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3514; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3515; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
3516; GFX940-TGSPLIT-NEXT:    s_endpgm
3517;
3518; GFX11-WGP-LABEL: flat_agent_release_acquire_cmpxchg:
3519; GFX11-WGP:       ; %bb.0: ; %entry
3520; GFX11-WGP-NEXT:    s_clause 0x1
3521; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3522; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3523; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3524; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3525; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
3526; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3527; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
3528; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3529; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3530; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3531; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3532; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3533; GFX11-WGP-NEXT:    buffer_gl0_inv
3534; GFX11-WGP-NEXT:    buffer_gl1_inv
3535; GFX11-WGP-NEXT:    s_endpgm
3536;
3537; GFX11-CU-LABEL: flat_agent_release_acquire_cmpxchg:
3538; GFX11-CU:       ; %bb.0: ; %entry
3539; GFX11-CU-NEXT:    s_clause 0x1
3540; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3541; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3542; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3543; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3544; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
3545; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3546; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
3547; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3548; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3549; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3550; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3551; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3552; GFX11-CU-NEXT:    buffer_gl0_inv
3553; GFX11-CU-NEXT:    buffer_gl1_inv
3554; GFX11-CU-NEXT:    s_endpgm
3555    i32* %out, i32 %in, i32 %old) {
3556entry:
3557  %gep = getelementptr i32, i32* %out, i32 4
3558  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release acquire
3559  ret void
3560}
3561
3562define amdgpu_kernel void @flat_agent_acq_rel_acquire_cmpxchg(
3563; GFX7-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
3564; GFX7:       ; %bb.0: ; %entry
3565; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3566; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3567; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3568; GFX7-NEXT:    s_add_u32 s0, s0, 16
3569; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3570; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3571; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3572; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3573; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3574; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3575; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3576; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3577; GFX7-NEXT:    buffer_wbinvl1_vol
3578; GFX7-NEXT:    s_endpgm
3579;
3580; GFX10-WGP-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
3581; GFX10-WGP:       ; %bb.0: ; %entry
3582; GFX10-WGP-NEXT:    s_clause 0x1
3583; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3584; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3585; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3586; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3587; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3588; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3589; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3590; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3591; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3592; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3593; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3594; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3595; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3596; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3597; GFX10-WGP-NEXT:    buffer_gl0_inv
3598; GFX10-WGP-NEXT:    buffer_gl1_inv
3599; GFX10-WGP-NEXT:    s_endpgm
3600;
3601; GFX10-CU-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
3602; GFX10-CU:       ; %bb.0: ; %entry
3603; GFX10-CU-NEXT:    s_clause 0x1
3604; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3605; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3606; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3607; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3608; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3609; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3610; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3611; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3612; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3613; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3614; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3615; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3616; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3617; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3618; GFX10-CU-NEXT:    buffer_gl0_inv
3619; GFX10-CU-NEXT:    buffer_gl1_inv
3620; GFX10-CU-NEXT:    s_endpgm
3621;
3622; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
3623; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3624; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3625; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3626; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3627; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3628; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3629; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3630; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3631; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3632; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3633; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3634; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3635; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3636; SKIP-CACHE-INV-NEXT:    s_endpgm
3637;
3638; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
3639; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3640; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3641; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3642; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3643; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3644; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3645; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3646; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3647; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3648; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3649; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3650;
3651; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
3652; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3653; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3654; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3655; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3656; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3657; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3658; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3659; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3660; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3661; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3662; GFX90A-TGSPLIT-NEXT:    s_endpgm
3663;
3664; GFX940-NOTTGSPLIT-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
3665; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3666; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3667; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3668; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3669; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3670; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3671; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
3672; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3673; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3674; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3675; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
3676; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3677;
3678; GFX940-TGSPLIT-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
3679; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3680; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3681; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3682; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3683; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3684; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3685; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
3686; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3687; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3688; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3689; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
3690; GFX940-TGSPLIT-NEXT:    s_endpgm
3691;
3692; GFX11-WGP-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
3693; GFX11-WGP:       ; %bb.0: ; %entry
3694; GFX11-WGP-NEXT:    s_clause 0x1
3695; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3696; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3697; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3698; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3699; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
3700; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3701; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
3702; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3703; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3704; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3705; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3706; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3707; GFX11-WGP-NEXT:    buffer_gl0_inv
3708; GFX11-WGP-NEXT:    buffer_gl1_inv
3709; GFX11-WGP-NEXT:    s_endpgm
3710;
3711; GFX11-CU-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
3712; GFX11-CU:       ; %bb.0: ; %entry
3713; GFX11-CU-NEXT:    s_clause 0x1
3714; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3715; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3716; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3717; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3718; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
3719; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3720; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
3721; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3722; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3723; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3724; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3725; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3726; GFX11-CU-NEXT:    buffer_gl0_inv
3727; GFX11-CU-NEXT:    buffer_gl1_inv
3728; GFX11-CU-NEXT:    s_endpgm
3729    i32* %out, i32 %in, i32 %old) {
3730entry:
3731  %gep = getelementptr i32, i32* %out, i32 4
3732  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel acquire
3733  ret void
3734}
3735
3736define amdgpu_kernel void @flat_agent_seq_cst_acquire_cmpxchg(
3737; GFX7-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
3738; GFX7:       ; %bb.0: ; %entry
3739; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3740; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3741; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3742; GFX7-NEXT:    s_add_u32 s0, s0, 16
3743; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3744; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3745; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3746; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3747; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3748; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3749; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3750; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3751; GFX7-NEXT:    buffer_wbinvl1_vol
3752; GFX7-NEXT:    s_endpgm
3753;
3754; GFX10-WGP-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
3755; GFX10-WGP:       ; %bb.0: ; %entry
3756; GFX10-WGP-NEXT:    s_clause 0x1
3757; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3758; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3759; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3760; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3761; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3762; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3763; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3764; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3765; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3766; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3767; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3768; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3769; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3770; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3771; GFX10-WGP-NEXT:    buffer_gl0_inv
3772; GFX10-WGP-NEXT:    buffer_gl1_inv
3773; GFX10-WGP-NEXT:    s_endpgm
3774;
3775; GFX10-CU-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
3776; GFX10-CU:       ; %bb.0: ; %entry
3777; GFX10-CU-NEXT:    s_clause 0x1
3778; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3779; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3780; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3781; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3782; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3783; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3784; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3785; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3786; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3787; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3788; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3789; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3790; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3791; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3792; GFX10-CU-NEXT:    buffer_gl0_inv
3793; GFX10-CU-NEXT:    buffer_gl1_inv
3794; GFX10-CU-NEXT:    s_endpgm
3795;
3796; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
3797; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3798; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3799; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3800; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3801; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3802; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3803; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3804; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3805; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3806; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3807; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3808; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3809; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3810; SKIP-CACHE-INV-NEXT:    s_endpgm
3811;
3812; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
3813; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3814; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3815; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3816; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3817; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3818; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3819; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3820; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3821; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3822; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3823; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3824;
3825; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
3826; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3827; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3828; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3829; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3830; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3831; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3832; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3833; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3834; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3835; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3836; GFX90A-TGSPLIT-NEXT:    s_endpgm
3837;
3838; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
3839; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3840; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3841; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3842; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3843; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3844; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3845; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
3846; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3847; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3848; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3849; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
3850; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3851;
3852; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
3853; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3854; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3855; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3856; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3857; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3858; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3859; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
3860; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3861; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3862; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3863; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
3864; GFX940-TGSPLIT-NEXT:    s_endpgm
3865;
3866; GFX11-WGP-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
3867; GFX11-WGP:       ; %bb.0: ; %entry
3868; GFX11-WGP-NEXT:    s_clause 0x1
3869; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3870; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3871; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3872; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
3873; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
3874; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
3875; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
3876; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3877; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3878; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3879; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3880; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3881; GFX11-WGP-NEXT:    buffer_gl0_inv
3882; GFX11-WGP-NEXT:    buffer_gl1_inv
3883; GFX11-WGP-NEXT:    s_endpgm
3884;
3885; GFX11-CU-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
3886; GFX11-CU:       ; %bb.0: ; %entry
3887; GFX11-CU-NEXT:    s_clause 0x1
3888; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
3889; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
3890; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3891; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
3892; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
3893; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
3894; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
3895; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3896; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3897; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3898; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
3899; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3900; GFX11-CU-NEXT:    buffer_gl0_inv
3901; GFX11-CU-NEXT:    buffer_gl1_inv
3902; GFX11-CU-NEXT:    s_endpgm
3903    i32* %out, i32 %in, i32 %old) {
3904entry:
3905  %gep = getelementptr i32, i32* %out, i32 4
3906  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst acquire
3907  ret void
3908}
3909
3910define amdgpu_kernel void @flat_agent_monotonic_seq_cst_cmpxchg(
3911; GFX7-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
3912; GFX7:       ; %bb.0: ; %entry
3913; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3914; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3915; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3916; GFX7-NEXT:    s_add_u32 s0, s0, 16
3917; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3918; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3919; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3920; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3921; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3922; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3923; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3924; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3925; GFX7-NEXT:    buffer_wbinvl1_vol
3926; GFX7-NEXT:    s_endpgm
3927;
3928; GFX10-WGP-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
3929; GFX10-WGP:       ; %bb.0: ; %entry
3930; GFX10-WGP-NEXT:    s_clause 0x1
3931; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3932; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3933; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3934; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3935; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3936; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3937; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3938; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3939; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3940; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3941; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3942; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3943; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3944; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3945; GFX10-WGP-NEXT:    buffer_gl0_inv
3946; GFX10-WGP-NEXT:    buffer_gl1_inv
3947; GFX10-WGP-NEXT:    s_endpgm
3948;
3949; GFX10-CU-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
3950; GFX10-CU:       ; %bb.0: ; %entry
3951; GFX10-CU-NEXT:    s_clause 0x1
3952; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3953; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3954; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3955; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3956; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3957; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3958; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3959; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3960; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3961; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3962; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3963; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3964; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3965; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3966; GFX10-CU-NEXT:    buffer_gl0_inv
3967; GFX10-CU-NEXT:    buffer_gl1_inv
3968; GFX10-CU-NEXT:    s_endpgm
3969;
3970; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
3971; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3972; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3973; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3974; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3975; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3976; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3977; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3978; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3979; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3980; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3981; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3982; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3983; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3984; SKIP-CACHE-INV-NEXT:    s_endpgm
3985;
3986; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
3987; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3988; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3989; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3990; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3991; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3992; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3993; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3994; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3995; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3996; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3997; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3998;
3999; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
4000; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4001; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4002; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4003; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4004; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4005; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4006; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4007; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4008; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4009; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4010; GFX90A-TGSPLIT-NEXT:    s_endpgm
4011;
4012; GFX940-NOTTGSPLIT-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
4013; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4014; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4015; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4016; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4017; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4018; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4019; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
4020; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4021; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4022; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4023; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
4024; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4025;
4026; GFX940-TGSPLIT-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
4027; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4028; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4029; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4030; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4031; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4032; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4033; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
4034; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4035; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4036; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4037; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
4038; GFX940-TGSPLIT-NEXT:    s_endpgm
4039;
4040; GFX11-WGP-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
4041; GFX11-WGP:       ; %bb.0: ; %entry
4042; GFX11-WGP-NEXT:    s_clause 0x1
4043; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4044; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4045; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4046; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
4047; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
4048; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
4049; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
4050; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4051; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4052; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4053; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4054; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4055; GFX11-WGP-NEXT:    buffer_gl0_inv
4056; GFX11-WGP-NEXT:    buffer_gl1_inv
4057; GFX11-WGP-NEXT:    s_endpgm
4058;
4059; GFX11-CU-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
4060; GFX11-CU:       ; %bb.0: ; %entry
4061; GFX11-CU-NEXT:    s_clause 0x1
4062; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4063; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4064; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4065; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
4066; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
4067; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
4068; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
4069; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4070; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4071; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4072; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4073; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4074; GFX11-CU-NEXT:    buffer_gl0_inv
4075; GFX11-CU-NEXT:    buffer_gl1_inv
4076; GFX11-CU-NEXT:    s_endpgm
4077    i32* %out, i32 %in, i32 %old) {
4078entry:
4079  %gep = getelementptr i32, i32* %out, i32 4
4080  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") monotonic seq_cst
4081  ret void
4082}
4083
4084define amdgpu_kernel void @flat_agent_acquire_seq_cst_cmpxchg(
4085; GFX7-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
4086; GFX7:       ; %bb.0: ; %entry
4087; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4088; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4089; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4090; GFX7-NEXT:    s_add_u32 s0, s0, 16
4091; GFX7-NEXT:    s_addc_u32 s1, s1, 0
4092; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4093; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4094; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4095; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4096; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4097; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4098; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4099; GFX7-NEXT:    buffer_wbinvl1_vol
4100; GFX7-NEXT:    s_endpgm
4101;
4102; GFX10-WGP-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
4103; GFX10-WGP:       ; %bb.0: ; %entry
4104; GFX10-WGP-NEXT:    s_clause 0x1
4105; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4106; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4107; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4108; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
4109; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
4110; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4111; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4112; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4113; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4114; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4115; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4116; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4117; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4118; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4119; GFX10-WGP-NEXT:    buffer_gl0_inv
4120; GFX10-WGP-NEXT:    buffer_gl1_inv
4121; GFX10-WGP-NEXT:    s_endpgm
4122;
4123; GFX10-CU-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
4124; GFX10-CU:       ; %bb.0: ; %entry
4125; GFX10-CU-NEXT:    s_clause 0x1
4126; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4127; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4128; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4129; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
4130; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
4131; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4132; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4133; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4134; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4135; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4136; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4137; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4138; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4139; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4140; GFX10-CU-NEXT:    buffer_gl0_inv
4141; GFX10-CU-NEXT:    buffer_gl1_inv
4142; GFX10-CU-NEXT:    s_endpgm
4143;
4144; SKIP-CACHE-INV-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
4145; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4146; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4147; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4148; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4149; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
4150; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
4151; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4152; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4153; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4154; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4155; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4156; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4157; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4158; SKIP-CACHE-INV-NEXT:    s_endpgm
4159;
4160; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
4161; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4162; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4163; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4164; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4165; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4166; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4167; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4168; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4169; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4170; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4171; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4172;
4173; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
4174; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4175; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4176; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4177; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4178; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4179; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4180; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4181; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4182; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4183; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4184; GFX90A-TGSPLIT-NEXT:    s_endpgm
4185;
4186; GFX940-NOTTGSPLIT-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
4187; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4188; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4189; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4190; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4191; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4192; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4193; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
4194; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4195; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4196; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4197; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
4198; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4199;
4200; GFX940-TGSPLIT-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
4201; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4202; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4203; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4204; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4205; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4206; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4207; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
4208; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4209; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4210; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4211; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
4212; GFX940-TGSPLIT-NEXT:    s_endpgm
4213;
4214; GFX11-WGP-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
4215; GFX11-WGP:       ; %bb.0: ; %entry
4216; GFX11-WGP-NEXT:    s_clause 0x1
4217; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4218; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4219; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4220; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
4221; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
4222; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
4223; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
4224; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4225; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4226; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4227; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4228; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4229; GFX11-WGP-NEXT:    buffer_gl0_inv
4230; GFX11-WGP-NEXT:    buffer_gl1_inv
4231; GFX11-WGP-NEXT:    s_endpgm
4232;
4233; GFX11-CU-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
4234; GFX11-CU:       ; %bb.0: ; %entry
4235; GFX11-CU-NEXT:    s_clause 0x1
4236; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4237; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4238; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4239; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
4240; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
4241; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
4242; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
4243; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4244; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4245; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4246; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4247; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4248; GFX11-CU-NEXT:    buffer_gl0_inv
4249; GFX11-CU-NEXT:    buffer_gl1_inv
4250; GFX11-CU-NEXT:    s_endpgm
4251    i32* %out, i32 %in, i32 %old) {
4252entry:
4253  %gep = getelementptr i32, i32* %out, i32 4
4254  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire seq_cst
4255  ret void
4256}
4257
4258define amdgpu_kernel void @flat_agent_release_seq_cst_cmpxchg(
4259; GFX7-LABEL: flat_agent_release_seq_cst_cmpxchg:
4260; GFX7:       ; %bb.0: ; %entry
4261; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4262; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4263; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4264; GFX7-NEXT:    s_add_u32 s0, s0, 16
4265; GFX7-NEXT:    s_addc_u32 s1, s1, 0
4266; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4267; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4268; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4269; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4270; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4271; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4272; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4273; GFX7-NEXT:    buffer_wbinvl1_vol
4274; GFX7-NEXT:    s_endpgm
4275;
4276; GFX10-WGP-LABEL: flat_agent_release_seq_cst_cmpxchg:
4277; GFX10-WGP:       ; %bb.0: ; %entry
4278; GFX10-WGP-NEXT:    s_clause 0x1
4279; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4280; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4281; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4282; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
4283; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
4284; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4285; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4286; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4287; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4288; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4289; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4290; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4291; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4292; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4293; GFX10-WGP-NEXT:    buffer_gl0_inv
4294; GFX10-WGP-NEXT:    buffer_gl1_inv
4295; GFX10-WGP-NEXT:    s_endpgm
4296;
4297; GFX10-CU-LABEL: flat_agent_release_seq_cst_cmpxchg:
4298; GFX10-CU:       ; %bb.0: ; %entry
4299; GFX10-CU-NEXT:    s_clause 0x1
4300; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4301; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4302; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4303; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
4304; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
4305; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4306; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4307; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4308; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4309; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4310; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4311; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4312; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4313; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4314; GFX10-CU-NEXT:    buffer_gl0_inv
4315; GFX10-CU-NEXT:    buffer_gl1_inv
4316; GFX10-CU-NEXT:    s_endpgm
4317;
4318; SKIP-CACHE-INV-LABEL: flat_agent_release_seq_cst_cmpxchg:
4319; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4320; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4321; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4322; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4323; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
4324; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
4325; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4326; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4327; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4328; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4329; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4330; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4331; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4332; SKIP-CACHE-INV-NEXT:    s_endpgm
4333;
4334; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_seq_cst_cmpxchg:
4335; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4336; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4337; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4338; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4339; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4340; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4341; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4342; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4343; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4344; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4345; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4346;
4347; GFX90A-TGSPLIT-LABEL: flat_agent_release_seq_cst_cmpxchg:
4348; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4349; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4350; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4351; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4352; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4353; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4354; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4355; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4356; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4357; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4358; GFX90A-TGSPLIT-NEXT:    s_endpgm
4359;
4360; GFX940-NOTTGSPLIT-LABEL: flat_agent_release_seq_cst_cmpxchg:
4361; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4362; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4363; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4364; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4365; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4366; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4367; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
4368; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4369; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4370; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4371; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
4372; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4373;
4374; GFX940-TGSPLIT-LABEL: flat_agent_release_seq_cst_cmpxchg:
4375; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4376; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4377; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4378; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4379; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4380; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4381; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
4382; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4383; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4384; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4385; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
4386; GFX940-TGSPLIT-NEXT:    s_endpgm
4387;
4388; GFX11-WGP-LABEL: flat_agent_release_seq_cst_cmpxchg:
4389; GFX11-WGP:       ; %bb.0: ; %entry
4390; GFX11-WGP-NEXT:    s_clause 0x1
4391; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4392; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4393; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4394; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
4395; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
4396; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
4397; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
4398; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4399; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4400; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4401; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4402; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4403; GFX11-WGP-NEXT:    buffer_gl0_inv
4404; GFX11-WGP-NEXT:    buffer_gl1_inv
4405; GFX11-WGP-NEXT:    s_endpgm
4406;
4407; GFX11-CU-LABEL: flat_agent_release_seq_cst_cmpxchg:
4408; GFX11-CU:       ; %bb.0: ; %entry
4409; GFX11-CU-NEXT:    s_clause 0x1
4410; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4411; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4412; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4413; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
4414; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
4415; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
4416; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
4417; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4418; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4419; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4420; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4421; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4422; GFX11-CU-NEXT:    buffer_gl0_inv
4423; GFX11-CU-NEXT:    buffer_gl1_inv
4424; GFX11-CU-NEXT:    s_endpgm
4425    i32* %out, i32 %in, i32 %old) {
4426entry:
4427  %gep = getelementptr i32, i32* %out, i32 4
4428  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release seq_cst
4429  ret void
4430}
4431
4432define amdgpu_kernel void @flat_agent_acq_rel_seq_cst_cmpxchg(
4433; GFX7-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
4434; GFX7:       ; %bb.0: ; %entry
4435; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4436; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4437; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4438; GFX7-NEXT:    s_add_u32 s0, s0, 16
4439; GFX7-NEXT:    s_addc_u32 s1, s1, 0
4440; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4441; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4442; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4443; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4444; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4445; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4446; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4447; GFX7-NEXT:    buffer_wbinvl1_vol
4448; GFX7-NEXT:    s_endpgm
4449;
4450; GFX10-WGP-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
4451; GFX10-WGP:       ; %bb.0: ; %entry
4452; GFX10-WGP-NEXT:    s_clause 0x1
4453; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4454; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4455; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4456; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
4457; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
4458; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4459; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4460; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4461; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4462; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4463; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4464; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4465; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4466; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4467; GFX10-WGP-NEXT:    buffer_gl0_inv
4468; GFX10-WGP-NEXT:    buffer_gl1_inv
4469; GFX10-WGP-NEXT:    s_endpgm
4470;
4471; GFX10-CU-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
4472; GFX10-CU:       ; %bb.0: ; %entry
4473; GFX10-CU-NEXT:    s_clause 0x1
4474; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4475; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4476; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4477; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
4478; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
4479; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4480; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4481; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4482; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4483; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4484; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4485; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4486; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4487; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4488; GFX10-CU-NEXT:    buffer_gl0_inv
4489; GFX10-CU-NEXT:    buffer_gl1_inv
4490; GFX10-CU-NEXT:    s_endpgm
4491;
4492; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
4493; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4494; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4495; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4496; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4497; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
4498; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
4499; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4500; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4501; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4502; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4503; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4504; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4505; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4506; SKIP-CACHE-INV-NEXT:    s_endpgm
4507;
4508; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
4509; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4510; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4511; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4512; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4513; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4514; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4515; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4516; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4517; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4518; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4519; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4520;
4521; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
4522; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4523; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4524; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4525; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4526; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4527; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4528; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4529; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4530; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4531; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4532; GFX90A-TGSPLIT-NEXT:    s_endpgm
4533;
4534; GFX940-NOTTGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
4535; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4536; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4537; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4538; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4539; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4540; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4541; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
4542; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4543; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4544; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4545; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
4546; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4547;
4548; GFX940-TGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
4549; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4550; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4551; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4552; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4553; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4554; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4555; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
4556; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4557; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4558; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4559; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
4560; GFX940-TGSPLIT-NEXT:    s_endpgm
4561;
4562; GFX11-WGP-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
4563; GFX11-WGP:       ; %bb.0: ; %entry
4564; GFX11-WGP-NEXT:    s_clause 0x1
4565; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4566; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4567; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4568; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
4569; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
4570; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
4571; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
4572; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4573; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4574; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4575; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4576; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4577; GFX11-WGP-NEXT:    buffer_gl0_inv
4578; GFX11-WGP-NEXT:    buffer_gl1_inv
4579; GFX11-WGP-NEXT:    s_endpgm
4580;
4581; GFX11-CU-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
4582; GFX11-CU:       ; %bb.0: ; %entry
4583; GFX11-CU-NEXT:    s_clause 0x1
4584; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4585; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4586; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4587; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
4588; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
4589; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
4590; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
4591; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4592; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4593; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4594; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4595; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4596; GFX11-CU-NEXT:    buffer_gl0_inv
4597; GFX11-CU-NEXT:    buffer_gl1_inv
4598; GFX11-CU-NEXT:    s_endpgm
4599    i32* %out, i32 %in, i32 %old) {
4600entry:
4601  %gep = getelementptr i32, i32* %out, i32 4
4602  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel seq_cst
4603  ret void
4604}
4605
4606define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_cmpxchg(
4607; GFX7-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
4608; GFX7:       ; %bb.0: ; %entry
4609; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4610; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4611; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4612; GFX7-NEXT:    s_add_u32 s0, s0, 16
4613; GFX7-NEXT:    s_addc_u32 s1, s1, 0
4614; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4615; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4616; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4617; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4618; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4619; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4620; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4621; GFX7-NEXT:    buffer_wbinvl1_vol
4622; GFX7-NEXT:    s_endpgm
4623;
4624; GFX10-WGP-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
4625; GFX10-WGP:       ; %bb.0: ; %entry
4626; GFX10-WGP-NEXT:    s_clause 0x1
4627; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4628; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4629; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4630; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
4631; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
4632; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4633; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4634; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4635; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4636; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4637; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4638; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4639; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4640; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4641; GFX10-WGP-NEXT:    buffer_gl0_inv
4642; GFX10-WGP-NEXT:    buffer_gl1_inv
4643; GFX10-WGP-NEXT:    s_endpgm
4644;
4645; GFX10-CU-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
4646; GFX10-CU:       ; %bb.0: ; %entry
4647; GFX10-CU-NEXT:    s_clause 0x1
4648; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4649; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4650; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4651; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
4652; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
4653; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4654; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4655; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4656; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4657; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4658; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4659; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4660; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4661; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4662; GFX10-CU-NEXT:    buffer_gl0_inv
4663; GFX10-CU-NEXT:    buffer_gl1_inv
4664; GFX10-CU-NEXT:    s_endpgm
4665;
4666; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
4667; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4668; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4669; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4670; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4671; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
4672; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
4673; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4674; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4675; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4676; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4677; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4678; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4679; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4680; SKIP-CACHE-INV-NEXT:    s_endpgm
4681;
4682; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
4683; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4684; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4685; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4686; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4687; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4688; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4689; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4690; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4691; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4692; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4693; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4694;
4695; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
4696; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4697; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4698; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4699; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4700; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4701; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4702; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4703; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4704; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4705; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4706; GFX90A-TGSPLIT-NEXT:    s_endpgm
4707;
4708; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
4709; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4710; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4711; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4712; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4713; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4714; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4715; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
4716; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4717; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4718; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4719; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
4720; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4721;
4722; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
4723; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4724; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4725; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4726; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4727; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4728; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4729; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
4730; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4731; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4732; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4733; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
4734; GFX940-TGSPLIT-NEXT:    s_endpgm
4735;
4736; GFX11-WGP-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
4737; GFX11-WGP:       ; %bb.0: ; %entry
4738; GFX11-WGP-NEXT:    s_clause 0x1
4739; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4740; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4741; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4742; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
4743; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
4744; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
4745; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
4746; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4747; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4748; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4749; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4750; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4751; GFX11-WGP-NEXT:    buffer_gl0_inv
4752; GFX11-WGP-NEXT:    buffer_gl1_inv
4753; GFX11-WGP-NEXT:    s_endpgm
4754;
4755; GFX11-CU-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
4756; GFX11-CU:       ; %bb.0: ; %entry
4757; GFX11-CU-NEXT:    s_clause 0x1
4758; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4759; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4760; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4761; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
4762; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
4763; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
4764; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
4765; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4766; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4767; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4768; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4769; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4770; GFX11-CU-NEXT:    buffer_gl0_inv
4771; GFX11-CU-NEXT:    buffer_gl1_inv
4772; GFX11-CU-NEXT:    s_endpgm
4773    i32* %out, i32 %in, i32 %old) {
4774entry:
4775  %gep = getelementptr i32, i32* %out, i32 4
4776  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
4777  ret void
4778}
4779
4780define amdgpu_kernel void @flat_agent_monotonic_monotonic_ret_cmpxchg(
4781; GFX7-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
4782; GFX7:       ; %bb.0: ; %entry
4783; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4784; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4785; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4786; GFX7-NEXT:    s_add_u32 s4, s0, 16
4787; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4788; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4789; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4790; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4791; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4792; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4793; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4794; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4795; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4796; GFX7-NEXT:    flat_store_dword v[0:1], v2
4797; GFX7-NEXT:    s_endpgm
4798;
4799; GFX10-WGP-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
4800; GFX10-WGP:       ; %bb.0: ; %entry
4801; GFX10-WGP-NEXT:    s_clause 0x1
4802; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4803; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4804; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4805; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4806; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4807; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4808; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4809; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4810; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4811; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4812; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4813; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4814; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4815; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4816; GFX10-WGP-NEXT:    s_endpgm
4817;
4818; GFX10-CU-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
4819; GFX10-CU:       ; %bb.0: ; %entry
4820; GFX10-CU-NEXT:    s_clause 0x1
4821; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4822; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4823; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4824; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4825; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4826; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4827; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4828; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4829; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4830; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4831; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4832; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4833; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4834; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4835; GFX10-CU-NEXT:    s_endpgm
4836;
4837; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
4838; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4839; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4840; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4841; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4842; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4843; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4844; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4845; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4846; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4847; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4848; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4849; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4850; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4851; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4852; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4853; SKIP-CACHE-INV-NEXT:    s_endpgm
4854;
4855; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
4856; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4857; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4858; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4859; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4860; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4861; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4862; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4863; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4864; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4865; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4866;
4867; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
4868; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4869; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4870; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4871; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4872; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4873; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4874; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4875; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4876; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4877; GFX90A-TGSPLIT-NEXT:    s_endpgm
4878;
4879; GFX940-NOTTGSPLIT-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
4880; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4881; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4882; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4883; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4884; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4885; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4886; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4887; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4888; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4889; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4890;
4891; GFX940-TGSPLIT-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
4892; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4893; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4894; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4895; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4896; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4897; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4898; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4899; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4900; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4901; GFX940-TGSPLIT-NEXT:    s_endpgm
4902;
4903; GFX11-WGP-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
4904; GFX11-WGP:       ; %bb.0: ; %entry
4905; GFX11-WGP-NEXT:    s_clause 0x1
4906; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4907; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4908; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4909; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
4910; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
4911; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
4912; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
4913; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4914; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4915; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
4916; GFX11-WGP-NEXT:    s_endpgm
4917;
4918; GFX11-CU-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
4919; GFX11-CU:       ; %bb.0: ; %entry
4920; GFX11-CU-NEXT:    s_clause 0x1
4921; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
4922; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
4923; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
4924; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
4925; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
4926; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
4927; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
4928; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4929; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4930; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
4931; GFX11-CU-NEXT:    s_endpgm
4932    i32* %out, i32 %in, i32 %old) {
4933entry:
4934  %gep = getelementptr i32, i32* %out, i32 4
4935  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") monotonic monotonic
4936  %val0 = extractvalue { i32, i1 } %val, 0
4937  store i32 %val0, i32* %out, align 4
4938  ret void
4939}
4940
4941define amdgpu_kernel void @flat_agent_acquire_monotonic_ret_cmpxchg(
4942; GFX7-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
4943; GFX7:       ; %bb.0: ; %entry
4944; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4945; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4946; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4947; GFX7-NEXT:    s_add_u32 s4, s0, 16
4948; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4949; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4950; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4951; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4952; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4953; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4954; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4955; GFX7-NEXT:    buffer_wbinvl1_vol
4956; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4957; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4958; GFX7-NEXT:    flat_store_dword v[0:1], v2
4959; GFX7-NEXT:    s_endpgm
4960;
4961; GFX10-WGP-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
4962; GFX10-WGP:       ; %bb.0: ; %entry
4963; GFX10-WGP-NEXT:    s_clause 0x1
4964; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4965; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4966; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4967; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4968; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4969; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4970; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4971; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4972; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4973; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4974; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4975; GFX10-WGP-NEXT:    buffer_gl0_inv
4976; GFX10-WGP-NEXT:    buffer_gl1_inv
4977; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4978; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4979; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4980; GFX10-WGP-NEXT:    s_endpgm
4981;
4982; GFX10-CU-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
4983; GFX10-CU:       ; %bb.0: ; %entry
4984; GFX10-CU-NEXT:    s_clause 0x1
4985; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4986; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4987; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4988; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4989; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4990; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4991; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4992; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4993; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4994; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4995; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4996; GFX10-CU-NEXT:    buffer_gl0_inv
4997; GFX10-CU-NEXT:    buffer_gl1_inv
4998; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4999; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5000; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5001; GFX10-CU-NEXT:    s_endpgm
5002;
5003; SKIP-CACHE-INV-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
5004; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5005; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5006; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5007; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5008; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5009; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5010; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5011; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5012; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5013; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5014; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5015; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5016; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5017; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5018; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5019; SKIP-CACHE-INV-NEXT:    s_endpgm
5020;
5021; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
5022; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5023; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5024; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5025; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5026; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5027; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5028; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5029; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5030; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5031; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5032; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5033;
5034; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
5035; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5036; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5037; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5038; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5039; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5040; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5041; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5042; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5043; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5044; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5045; GFX90A-TGSPLIT-NEXT:    s_endpgm
5046;
5047; GFX940-NOTTGSPLIT-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
5048; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5049; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5050; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5051; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5052; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5053; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5054; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5055; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5056; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
5057; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5058; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5059;
5060; GFX940-TGSPLIT-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
5061; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5062; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5063; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5064; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5065; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5066; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5067; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5068; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5069; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
5070; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5071; GFX940-TGSPLIT-NEXT:    s_endpgm
5072;
5073; GFX11-WGP-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
5074; GFX11-WGP:       ; %bb.0: ; %entry
5075; GFX11-WGP-NEXT:    s_clause 0x1
5076; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5077; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5078; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5079; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
5080; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
5081; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
5082; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
5083; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5084; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5085; GFX11-WGP-NEXT:    buffer_gl0_inv
5086; GFX11-WGP-NEXT:    buffer_gl1_inv
5087; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5088; GFX11-WGP-NEXT:    s_endpgm
5089;
5090; GFX11-CU-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
5091; GFX11-CU:       ; %bb.0: ; %entry
5092; GFX11-CU-NEXT:    s_clause 0x1
5093; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5094; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5095; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5096; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
5097; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
5098; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
5099; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
5100; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5101; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5102; GFX11-CU-NEXT:    buffer_gl0_inv
5103; GFX11-CU-NEXT:    buffer_gl1_inv
5104; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5105; GFX11-CU-NEXT:    s_endpgm
5106    i32* %out, i32 %in, i32 %old) {
5107entry:
5108  %gep = getelementptr i32, i32* %out, i32 4
5109  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire monotonic
5110  %val0 = extractvalue { i32, i1 } %val, 0
5111  store i32 %val0, i32* %out, align 4
5112  ret void
5113}
5114
5115define amdgpu_kernel void @flat_agent_release_monotonic_ret_cmpxchg(
5116; GFX7-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
5117; GFX7:       ; %bb.0: ; %entry
5118; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5119; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5120; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5121; GFX7-NEXT:    s_add_u32 s4, s0, 16
5122; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5123; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5124; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5125; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5126; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5127; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5128; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5129; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5130; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5131; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5132; GFX7-NEXT:    flat_store_dword v[0:1], v2
5133; GFX7-NEXT:    s_endpgm
5134;
5135; GFX10-WGP-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
5136; GFX10-WGP:       ; %bb.0: ; %entry
5137; GFX10-WGP-NEXT:    s_clause 0x1
5138; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5139; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5140; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5141; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5142; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5143; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5144; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5145; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5146; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5147; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5148; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5149; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5150; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5151; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5152; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5153; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5154; GFX10-WGP-NEXT:    s_endpgm
5155;
5156; GFX10-CU-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
5157; GFX10-CU:       ; %bb.0: ; %entry
5158; GFX10-CU-NEXT:    s_clause 0x1
5159; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5160; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5161; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5162; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5163; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5164; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5165; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5166; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5167; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5168; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5169; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5170; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5171; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5172; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5173; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5174; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5175; GFX10-CU-NEXT:    s_endpgm
5176;
5177; SKIP-CACHE-INV-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
5178; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5179; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5180; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5181; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5182; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5183; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5184; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5185; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5186; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5187; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5188; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5189; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5190; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5191; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5192; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5193; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5194; SKIP-CACHE-INV-NEXT:    s_endpgm
5195;
5196; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
5197; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5198; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5199; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5200; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5201; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5202; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5203; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5204; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5205; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5206; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5207; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5208;
5209; GFX90A-TGSPLIT-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
5210; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5211; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5212; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5213; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5214; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5215; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5216; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5217; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5218; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5219; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5220; GFX90A-TGSPLIT-NEXT:    s_endpgm
5221;
5222; GFX940-NOTTGSPLIT-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
5223; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5224; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5225; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5226; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5227; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5228; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5229; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
5230; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5231; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5232; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5233; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5234; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5235;
5236; GFX940-TGSPLIT-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
5237; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5238; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5239; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5240; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5241; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5242; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5243; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
5244; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5245; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5246; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5247; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5248; GFX940-TGSPLIT-NEXT:    s_endpgm
5249;
5250; GFX11-WGP-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
5251; GFX11-WGP:       ; %bb.0: ; %entry
5252; GFX11-WGP-NEXT:    s_clause 0x1
5253; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5254; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5255; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5256; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
5257; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
5258; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
5259; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
5260; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5261; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5262; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5263; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5264; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5265; GFX11-WGP-NEXT:    s_endpgm
5266;
5267; GFX11-CU-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
5268; GFX11-CU:       ; %bb.0: ; %entry
5269; GFX11-CU-NEXT:    s_clause 0x1
5270; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5271; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5272; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5273; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
5274; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
5275; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
5276; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
5277; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5278; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5279; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5280; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5281; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5282; GFX11-CU-NEXT:    s_endpgm
5283    i32* %out, i32 %in, i32 %old) {
5284entry:
5285  %gep = getelementptr i32, i32* %out, i32 4
5286  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release monotonic
5287  %val0 = extractvalue { i32, i1 } %val, 0
5288  store i32 %val0, i32* %out, align 4
5289  ret void
5290}
5291
5292define amdgpu_kernel void @flat_agent_acq_rel_monotonic_ret_cmpxchg(
5293; GFX7-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
5294; GFX7:       ; %bb.0: ; %entry
5295; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5296; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5297; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5298; GFX7-NEXT:    s_add_u32 s4, s0, 16
5299; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5300; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5301; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5302; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5303; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5304; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5305; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5306; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5307; GFX7-NEXT:    buffer_wbinvl1_vol
5308; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5309; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5310; GFX7-NEXT:    flat_store_dword v[0:1], v2
5311; GFX7-NEXT:    s_endpgm
5312;
5313; GFX10-WGP-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
5314; GFX10-WGP:       ; %bb.0: ; %entry
5315; GFX10-WGP-NEXT:    s_clause 0x1
5316; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5317; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5318; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5319; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5320; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5321; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5322; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5323; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5324; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5325; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5326; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5327; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5328; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5329; GFX10-WGP-NEXT:    buffer_gl0_inv
5330; GFX10-WGP-NEXT:    buffer_gl1_inv
5331; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5332; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5333; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5334; GFX10-WGP-NEXT:    s_endpgm
5335;
5336; GFX10-CU-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
5337; GFX10-CU:       ; %bb.0: ; %entry
5338; GFX10-CU-NEXT:    s_clause 0x1
5339; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5340; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5341; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5342; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5343; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5344; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5345; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5346; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5347; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5348; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5349; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5350; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5351; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5352; GFX10-CU-NEXT:    buffer_gl0_inv
5353; GFX10-CU-NEXT:    buffer_gl1_inv
5354; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5355; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5356; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5357; GFX10-CU-NEXT:    s_endpgm
5358;
5359; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
5360; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5361; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5362; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5363; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5364; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5365; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5366; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5367; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5368; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5369; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5370; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5371; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5372; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5373; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5374; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5375; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5376; SKIP-CACHE-INV-NEXT:    s_endpgm
5377;
5378; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
5379; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5380; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5381; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5382; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5383; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5384; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5385; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5386; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5387; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5388; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5389; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5390; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5391;
5392; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
5393; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5394; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5395; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5396; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5397; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5398; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5399; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5400; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5401; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5402; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5403; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5404; GFX90A-TGSPLIT-NEXT:    s_endpgm
5405;
5406; GFX940-NOTTGSPLIT-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
5407; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5408; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5409; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5410; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5411; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5412; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5413; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
5414; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5415; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5416; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5417; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
5418; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5419; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5420;
5421; GFX940-TGSPLIT-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
5422; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5423; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5424; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5425; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5426; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5427; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5428; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
5429; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5430; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5431; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5432; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
5433; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5434; GFX940-TGSPLIT-NEXT:    s_endpgm
5435;
5436; GFX11-WGP-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
5437; GFX11-WGP:       ; %bb.0: ; %entry
5438; GFX11-WGP-NEXT:    s_clause 0x1
5439; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5440; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5441; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5442; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
5443; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
5444; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
5445; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
5446; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5447; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5448; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5449; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5450; GFX11-WGP-NEXT:    buffer_gl0_inv
5451; GFX11-WGP-NEXT:    buffer_gl1_inv
5452; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5453; GFX11-WGP-NEXT:    s_endpgm
5454;
5455; GFX11-CU-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
5456; GFX11-CU:       ; %bb.0: ; %entry
5457; GFX11-CU-NEXT:    s_clause 0x1
5458; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5459; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5460; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5461; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
5462; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
5463; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
5464; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
5465; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5466; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5467; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5468; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5469; GFX11-CU-NEXT:    buffer_gl0_inv
5470; GFX11-CU-NEXT:    buffer_gl1_inv
5471; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5472; GFX11-CU-NEXT:    s_endpgm
5473    i32* %out, i32 %in, i32 %old) {
5474entry:
5475  %gep = getelementptr i32, i32* %out, i32 4
5476  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel monotonic
5477  %val0 = extractvalue { i32, i1 } %val, 0
5478  store i32 %val0, i32* %out, align 4
5479  ret void
5480}
5481
5482define amdgpu_kernel void @flat_agent_seq_cst_monotonic_ret_cmpxchg(
5483; GFX7-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
5484; GFX7:       ; %bb.0: ; %entry
5485; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5486; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5487; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5488; GFX7-NEXT:    s_add_u32 s4, s0, 16
5489; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5490; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5491; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5492; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5493; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5494; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5495; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5496; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5497; GFX7-NEXT:    buffer_wbinvl1_vol
5498; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5499; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5500; GFX7-NEXT:    flat_store_dword v[0:1], v2
5501; GFX7-NEXT:    s_endpgm
5502;
5503; GFX10-WGP-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
5504; GFX10-WGP:       ; %bb.0: ; %entry
5505; GFX10-WGP-NEXT:    s_clause 0x1
5506; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5507; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5508; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5509; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5510; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5511; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5512; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5513; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5514; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5515; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5516; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5517; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5518; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5519; GFX10-WGP-NEXT:    buffer_gl0_inv
5520; GFX10-WGP-NEXT:    buffer_gl1_inv
5521; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5522; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5523; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5524; GFX10-WGP-NEXT:    s_endpgm
5525;
5526; GFX10-CU-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
5527; GFX10-CU:       ; %bb.0: ; %entry
5528; GFX10-CU-NEXT:    s_clause 0x1
5529; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5530; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5531; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5532; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5533; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5534; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5535; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5536; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5537; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5538; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5539; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5540; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5541; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5542; GFX10-CU-NEXT:    buffer_gl0_inv
5543; GFX10-CU-NEXT:    buffer_gl1_inv
5544; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5545; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5546; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5547; GFX10-CU-NEXT:    s_endpgm
5548;
5549; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
5550; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5551; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5552; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5553; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5554; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5555; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5556; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5557; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5558; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5559; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5560; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5561; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5562; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5563; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5564; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5565; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5566; SKIP-CACHE-INV-NEXT:    s_endpgm
5567;
5568; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
5569; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5570; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5571; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5572; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5573; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5574; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5575; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5576; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5577; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5578; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5579; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5580; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5581;
5582; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
5583; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5584; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5585; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5586; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5587; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5588; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5589; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5590; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5591; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5592; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5593; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5594; GFX90A-TGSPLIT-NEXT:    s_endpgm
5595;
5596; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
5597; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5598; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5599; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5600; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5601; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5602; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5603; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
5604; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5605; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5606; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5607; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
5608; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5609; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5610;
5611; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
5612; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5613; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5614; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5615; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5616; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5617; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5618; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
5619; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5620; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5621; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5622; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
5623; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5624; GFX940-TGSPLIT-NEXT:    s_endpgm
5625;
5626; GFX11-WGP-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
5627; GFX11-WGP:       ; %bb.0: ; %entry
5628; GFX11-WGP-NEXT:    s_clause 0x1
5629; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5630; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5631; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5632; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
5633; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
5634; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
5635; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
5636; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5637; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5638; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5639; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5640; GFX11-WGP-NEXT:    buffer_gl0_inv
5641; GFX11-WGP-NEXT:    buffer_gl1_inv
5642; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5643; GFX11-WGP-NEXT:    s_endpgm
5644;
5645; GFX11-CU-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
5646; GFX11-CU:       ; %bb.0: ; %entry
5647; GFX11-CU-NEXT:    s_clause 0x1
5648; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5649; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5650; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5651; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
5652; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
5653; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
5654; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
5655; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5656; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5657; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5658; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5659; GFX11-CU-NEXT:    buffer_gl0_inv
5660; GFX11-CU-NEXT:    buffer_gl1_inv
5661; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5662; GFX11-CU-NEXT:    s_endpgm
5663    i32* %out, i32 %in, i32 %old) {
5664entry:
5665  %gep = getelementptr i32, i32* %out, i32 4
5666  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst monotonic
5667  %val0 = extractvalue { i32, i1 } %val, 0
5668  store i32 %val0, i32* %out, align 4
5669  ret void
5670}
5671
5672define amdgpu_kernel void @flat_agent_monotonic_acquire_ret_cmpxchg(
5673; GFX7-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
5674; GFX7:       ; %bb.0: ; %entry
5675; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5676; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5677; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5678; GFX7-NEXT:    s_add_u32 s4, s0, 16
5679; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5680; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5681; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5682; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5683; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5684; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5685; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5686; GFX7-NEXT:    buffer_wbinvl1_vol
5687; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5688; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5689; GFX7-NEXT:    flat_store_dword v[0:1], v2
5690; GFX7-NEXT:    s_endpgm
5691;
5692; GFX10-WGP-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
5693; GFX10-WGP:       ; %bb.0: ; %entry
5694; GFX10-WGP-NEXT:    s_clause 0x1
5695; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5696; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5697; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5698; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5699; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5700; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5701; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5702; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5703; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5704; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5705; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5706; GFX10-WGP-NEXT:    buffer_gl0_inv
5707; GFX10-WGP-NEXT:    buffer_gl1_inv
5708; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5709; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5710; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5711; GFX10-WGP-NEXT:    s_endpgm
5712;
5713; GFX10-CU-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
5714; GFX10-CU:       ; %bb.0: ; %entry
5715; GFX10-CU-NEXT:    s_clause 0x1
5716; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5717; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5718; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5719; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5720; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5721; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5722; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5723; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5724; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5725; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5726; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5727; GFX10-CU-NEXT:    buffer_gl0_inv
5728; GFX10-CU-NEXT:    buffer_gl1_inv
5729; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5730; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5731; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5732; GFX10-CU-NEXT:    s_endpgm
5733;
5734; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
5735; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5736; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5737; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5738; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5739; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5740; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5741; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5742; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5743; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5744; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5745; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5746; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5747; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5748; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5749; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5750; SKIP-CACHE-INV-NEXT:    s_endpgm
5751;
5752; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
5753; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5754; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5755; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5756; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5757; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5758; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5759; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5760; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5761; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5762; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5763; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5764;
5765; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
5766; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5767; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5768; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5769; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5770; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5771; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5772; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5773; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5774; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5775; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5776; GFX90A-TGSPLIT-NEXT:    s_endpgm
5777;
5778; GFX940-NOTTGSPLIT-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
5779; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5780; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5781; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5782; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5783; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5784; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5785; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5786; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5787; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
5788; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5789; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5790;
5791; GFX940-TGSPLIT-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
5792; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5793; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5794; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5795; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5796; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5797; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5798; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5799; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5800; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
5801; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5802; GFX940-TGSPLIT-NEXT:    s_endpgm
5803;
5804; GFX11-WGP-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
5805; GFX11-WGP:       ; %bb.0: ; %entry
5806; GFX11-WGP-NEXT:    s_clause 0x1
5807; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5808; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5809; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5810; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
5811; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
5812; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
5813; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
5814; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5815; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5816; GFX11-WGP-NEXT:    buffer_gl0_inv
5817; GFX11-WGP-NEXT:    buffer_gl1_inv
5818; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5819; GFX11-WGP-NEXT:    s_endpgm
5820;
5821; GFX11-CU-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
5822; GFX11-CU:       ; %bb.0: ; %entry
5823; GFX11-CU-NEXT:    s_clause 0x1
5824; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5825; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5826; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
5827; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
5828; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
5829; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
5830; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
5831; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5832; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5833; GFX11-CU-NEXT:    buffer_gl0_inv
5834; GFX11-CU-NEXT:    buffer_gl1_inv
5835; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
5836; GFX11-CU-NEXT:    s_endpgm
5837    i32* %out, i32 %in, i32 %old) {
5838entry:
5839  %gep = getelementptr i32, i32* %out, i32 4
5840  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") monotonic acquire
5841  %val0 = extractvalue { i32, i1 } %val, 0
5842  store i32 %val0, i32* %out, align 4
5843  ret void
5844}
5845
5846define amdgpu_kernel void @flat_agent_acquire_acquire_ret_cmpxchg(
5847; GFX7-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
5848; GFX7:       ; %bb.0: ; %entry
5849; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5850; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5851; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5852; GFX7-NEXT:    s_add_u32 s4, s0, 16
5853; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5854; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5855; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5856; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5857; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5858; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5859; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5860; GFX7-NEXT:    buffer_wbinvl1_vol
5861; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5862; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5863; GFX7-NEXT:    flat_store_dword v[0:1], v2
5864; GFX7-NEXT:    s_endpgm
5865;
5866; GFX10-WGP-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
5867; GFX10-WGP:       ; %bb.0: ; %entry
5868; GFX10-WGP-NEXT:    s_clause 0x1
5869; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5870; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5871; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5872; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5873; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5874; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5875; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5876; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5877; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5878; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5879; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5880; GFX10-WGP-NEXT:    buffer_gl0_inv
5881; GFX10-WGP-NEXT:    buffer_gl1_inv
5882; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5883; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5884; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5885; GFX10-WGP-NEXT:    s_endpgm
5886;
5887; GFX10-CU-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
5888; GFX10-CU:       ; %bb.0: ; %entry
5889; GFX10-CU-NEXT:    s_clause 0x1
5890; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5891; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5892; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5893; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5894; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5895; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5896; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5897; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5898; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5899; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5900; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5901; GFX10-CU-NEXT:    buffer_gl0_inv
5902; GFX10-CU-NEXT:    buffer_gl1_inv
5903; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5904; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5905; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5906; GFX10-CU-NEXT:    s_endpgm
5907;
5908; SKIP-CACHE-INV-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
5909; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5910; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5911; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5912; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5913; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5914; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5915; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5916; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5917; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5918; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5919; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5920; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5921; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5922; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5923; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5924; SKIP-CACHE-INV-NEXT:    s_endpgm
5925;
5926; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
5927; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5928; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5929; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5930; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5931; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5932; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5933; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5934; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5935; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5936; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5937; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5938;
5939; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
5940; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5941; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5942; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5943; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5944; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5945; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5946; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5947; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5948; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5949; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5950; GFX90A-TGSPLIT-NEXT:    s_endpgm
5951;
5952; GFX940-NOTTGSPLIT-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
5953; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5954; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5955; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5956; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5957; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5958; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5959; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5960; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5961; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
5962; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5963; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5964;
5965; GFX940-TGSPLIT-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
5966; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5967; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5968; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5969; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5970; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5971; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5972; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5973; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5974; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
5975; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5976; GFX940-TGSPLIT-NEXT:    s_endpgm
5977;
5978; GFX11-WGP-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
5979; GFX11-WGP:       ; %bb.0: ; %entry
5980; GFX11-WGP-NEXT:    s_clause 0x1
5981; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5982; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
5983; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5984; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
5985; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
5986; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
5987; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
5988; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5989; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5990; GFX11-WGP-NEXT:    buffer_gl0_inv
5991; GFX11-WGP-NEXT:    buffer_gl1_inv
5992; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
5993; GFX11-WGP-NEXT:    s_endpgm
5994;
5995; GFX11-CU-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
5996; GFX11-CU:       ; %bb.0: ; %entry
5997; GFX11-CU-NEXT:    s_clause 0x1
5998; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
5999; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6000; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6001; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
6002; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
6003; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
6004; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
6005; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6006; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6007; GFX11-CU-NEXT:    buffer_gl0_inv
6008; GFX11-CU-NEXT:    buffer_gl1_inv
6009; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6010; GFX11-CU-NEXT:    s_endpgm
6011    i32* %out, i32 %in, i32 %old) {
6012entry:
6013  %gep = getelementptr i32, i32* %out, i32 4
6014  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire acquire
6015  %val0 = extractvalue { i32, i1 } %val, 0
6016  store i32 %val0, i32* %out, align 4
6017  ret void
6018}
6019
6020define amdgpu_kernel void @flat_agent_release_acquire_ret_cmpxchg(
6021; GFX7-LABEL: flat_agent_release_acquire_ret_cmpxchg:
6022; GFX7:       ; %bb.0: ; %entry
6023; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6024; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6025; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6026; GFX7-NEXT:    s_add_u32 s4, s0, 16
6027; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6028; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6029; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6030; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6031; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6032; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6033; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6034; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6035; GFX7-NEXT:    buffer_wbinvl1_vol
6036; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6037; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6038; GFX7-NEXT:    flat_store_dword v[0:1], v2
6039; GFX7-NEXT:    s_endpgm
6040;
6041; GFX10-WGP-LABEL: flat_agent_release_acquire_ret_cmpxchg:
6042; GFX10-WGP:       ; %bb.0: ; %entry
6043; GFX10-WGP-NEXT:    s_clause 0x1
6044; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6045; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6046; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6047; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6048; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6049; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6050; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6051; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6052; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6053; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6054; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6055; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6056; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6057; GFX10-WGP-NEXT:    buffer_gl0_inv
6058; GFX10-WGP-NEXT:    buffer_gl1_inv
6059; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6060; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6061; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6062; GFX10-WGP-NEXT:    s_endpgm
6063;
6064; GFX10-CU-LABEL: flat_agent_release_acquire_ret_cmpxchg:
6065; GFX10-CU:       ; %bb.0: ; %entry
6066; GFX10-CU-NEXT:    s_clause 0x1
6067; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6068; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6069; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6070; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6071; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6072; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6073; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6074; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6075; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6076; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6077; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6078; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6079; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6080; GFX10-CU-NEXT:    buffer_gl0_inv
6081; GFX10-CU-NEXT:    buffer_gl1_inv
6082; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6083; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6084; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6085; GFX10-CU-NEXT:    s_endpgm
6086;
6087; SKIP-CACHE-INV-LABEL: flat_agent_release_acquire_ret_cmpxchg:
6088; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6089; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6090; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6091; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6092; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6093; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6094; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6095; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6096; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6097; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6098; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6099; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6100; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6101; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6102; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6103; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6104; SKIP-CACHE-INV-NEXT:    s_endpgm
6105;
6106; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_acquire_ret_cmpxchg:
6107; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6108; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6109; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6110; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6111; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6112; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6113; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6114; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6115; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6116; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6117; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6118; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6119;
6120; GFX90A-TGSPLIT-LABEL: flat_agent_release_acquire_ret_cmpxchg:
6121; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6122; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6123; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6124; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6125; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6126; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6127; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6128; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6129; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6130; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6131; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6132; GFX90A-TGSPLIT-NEXT:    s_endpgm
6133;
6134; GFX940-NOTTGSPLIT-LABEL: flat_agent_release_acquire_ret_cmpxchg:
6135; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6136; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6137; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6138; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6139; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6140; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6141; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
6142; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6143; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6144; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6145; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
6146; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6147; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6148;
6149; GFX940-TGSPLIT-LABEL: flat_agent_release_acquire_ret_cmpxchg:
6150; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6151; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6152; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6153; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6154; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6155; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6156; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
6157; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6158; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6159; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6160; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
6161; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6162; GFX940-TGSPLIT-NEXT:    s_endpgm
6163;
6164; GFX11-WGP-LABEL: flat_agent_release_acquire_ret_cmpxchg:
6165; GFX11-WGP:       ; %bb.0: ; %entry
6166; GFX11-WGP-NEXT:    s_clause 0x1
6167; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6168; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6169; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6170; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
6171; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
6172; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
6173; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
6174; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6175; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6176; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6177; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6178; GFX11-WGP-NEXT:    buffer_gl0_inv
6179; GFX11-WGP-NEXT:    buffer_gl1_inv
6180; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6181; GFX11-WGP-NEXT:    s_endpgm
6182;
6183; GFX11-CU-LABEL: flat_agent_release_acquire_ret_cmpxchg:
6184; GFX11-CU:       ; %bb.0: ; %entry
6185; GFX11-CU-NEXT:    s_clause 0x1
6186; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6187; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6188; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6189; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
6190; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
6191; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
6192; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
6193; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6194; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6195; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6196; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6197; GFX11-CU-NEXT:    buffer_gl0_inv
6198; GFX11-CU-NEXT:    buffer_gl1_inv
6199; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6200; GFX11-CU-NEXT:    s_endpgm
6201    i32* %out, i32 %in, i32 %old) {
6202entry:
6203  %gep = getelementptr i32, i32* %out, i32 4
6204  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release acquire
6205  %val0 = extractvalue { i32, i1 } %val, 0
6206  store i32 %val0, i32* %out, align 4
6207  ret void
6208}
6209
6210define amdgpu_kernel void @flat_agent_acq_rel_acquire_ret_cmpxchg(
6211; GFX7-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
6212; GFX7:       ; %bb.0: ; %entry
6213; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6214; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6215; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6216; GFX7-NEXT:    s_add_u32 s4, s0, 16
6217; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6218; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6219; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6220; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6221; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6222; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6223; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6224; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6225; GFX7-NEXT:    buffer_wbinvl1_vol
6226; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6227; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6228; GFX7-NEXT:    flat_store_dword v[0:1], v2
6229; GFX7-NEXT:    s_endpgm
6230;
6231; GFX10-WGP-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
6232; GFX10-WGP:       ; %bb.0: ; %entry
6233; GFX10-WGP-NEXT:    s_clause 0x1
6234; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6235; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6236; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6237; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6238; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6239; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6240; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6241; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6242; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6243; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6244; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6245; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6246; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6247; GFX10-WGP-NEXT:    buffer_gl0_inv
6248; GFX10-WGP-NEXT:    buffer_gl1_inv
6249; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6250; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6251; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6252; GFX10-WGP-NEXT:    s_endpgm
6253;
6254; GFX10-CU-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
6255; GFX10-CU:       ; %bb.0: ; %entry
6256; GFX10-CU-NEXT:    s_clause 0x1
6257; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6258; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6259; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6260; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6261; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6262; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6263; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6264; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6265; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6266; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6267; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6268; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6269; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6270; GFX10-CU-NEXT:    buffer_gl0_inv
6271; GFX10-CU-NEXT:    buffer_gl1_inv
6272; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6273; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6274; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6275; GFX10-CU-NEXT:    s_endpgm
6276;
6277; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
6278; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6279; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6280; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6281; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6282; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6283; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6284; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6285; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6286; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6287; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6288; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6289; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6290; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6291; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6292; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6293; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6294; SKIP-CACHE-INV-NEXT:    s_endpgm
6295;
6296; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
6297; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6298; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6299; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6300; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6301; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6302; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6303; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6304; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6305; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6306; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6307; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6308; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6309;
6310; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
6311; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6312; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6313; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6314; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6315; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6316; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6317; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6318; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6319; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6320; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6321; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6322; GFX90A-TGSPLIT-NEXT:    s_endpgm
6323;
6324; GFX940-NOTTGSPLIT-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
6325; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6326; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6327; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6328; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6329; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6330; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6331; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
6332; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6333; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6334; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6335; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
6336; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6337; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6338;
6339; GFX940-TGSPLIT-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
6340; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6341; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6342; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6343; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6344; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6345; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6346; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
6347; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6348; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6349; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6350; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
6351; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6352; GFX940-TGSPLIT-NEXT:    s_endpgm
6353;
6354; GFX11-WGP-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
6355; GFX11-WGP:       ; %bb.0: ; %entry
6356; GFX11-WGP-NEXT:    s_clause 0x1
6357; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6358; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6359; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6360; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
6361; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
6362; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
6363; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
6364; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6365; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6366; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6367; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6368; GFX11-WGP-NEXT:    buffer_gl0_inv
6369; GFX11-WGP-NEXT:    buffer_gl1_inv
6370; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6371; GFX11-WGP-NEXT:    s_endpgm
6372;
6373; GFX11-CU-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
6374; GFX11-CU:       ; %bb.0: ; %entry
6375; GFX11-CU-NEXT:    s_clause 0x1
6376; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6377; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6378; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6379; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
6380; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
6381; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
6382; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
6383; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6384; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6385; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6386; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6387; GFX11-CU-NEXT:    buffer_gl0_inv
6388; GFX11-CU-NEXT:    buffer_gl1_inv
6389; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6390; GFX11-CU-NEXT:    s_endpgm
6391    i32* %out, i32 %in, i32 %old) {
6392entry:
6393  %gep = getelementptr i32, i32* %out, i32 4
6394  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel acquire
6395  %val0 = extractvalue { i32, i1 } %val, 0
6396  store i32 %val0, i32* %out, align 4
6397  ret void
6398}
6399
6400define amdgpu_kernel void @flat_agent_seq_cst_acquire_ret_cmpxchg(
6401; GFX7-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
6402; GFX7:       ; %bb.0: ; %entry
6403; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6404; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6405; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6406; GFX7-NEXT:    s_add_u32 s4, s0, 16
6407; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6408; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6409; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6410; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6411; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6412; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6413; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6414; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6415; GFX7-NEXT:    buffer_wbinvl1_vol
6416; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6417; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6418; GFX7-NEXT:    flat_store_dword v[0:1], v2
6419; GFX7-NEXT:    s_endpgm
6420;
6421; GFX10-WGP-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
6422; GFX10-WGP:       ; %bb.0: ; %entry
6423; GFX10-WGP-NEXT:    s_clause 0x1
6424; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6425; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6426; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6427; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6428; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6429; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6430; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6431; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6432; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6433; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6434; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6435; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6436; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6437; GFX10-WGP-NEXT:    buffer_gl0_inv
6438; GFX10-WGP-NEXT:    buffer_gl1_inv
6439; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6440; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6441; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6442; GFX10-WGP-NEXT:    s_endpgm
6443;
6444; GFX10-CU-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
6445; GFX10-CU:       ; %bb.0: ; %entry
6446; GFX10-CU-NEXT:    s_clause 0x1
6447; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6448; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6449; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6450; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6451; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6452; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6453; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6454; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6455; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6456; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6457; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6458; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6459; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6460; GFX10-CU-NEXT:    buffer_gl0_inv
6461; GFX10-CU-NEXT:    buffer_gl1_inv
6462; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6463; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6464; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6465; GFX10-CU-NEXT:    s_endpgm
6466;
6467; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
6468; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6469; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6470; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6471; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6472; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6473; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6474; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6475; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6476; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6477; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6478; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6479; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6480; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6481; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6482; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6483; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6484; SKIP-CACHE-INV-NEXT:    s_endpgm
6485;
6486; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
6487; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6488; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6489; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6490; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6491; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6492; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6493; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6494; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6495; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6496; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6497; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6498; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6499;
6500; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
6501; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6502; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6503; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6504; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6505; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6506; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6507; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6508; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6509; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6510; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6511; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6512; GFX90A-TGSPLIT-NEXT:    s_endpgm
6513;
6514; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
6515; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6516; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6517; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6518; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6519; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6520; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6521; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
6522; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6523; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6524; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6525; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
6526; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6527; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6528;
6529; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
6530; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6531; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6532; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6533; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6534; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6535; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6536; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
6537; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6538; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6539; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6540; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
6541; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6542; GFX940-TGSPLIT-NEXT:    s_endpgm
6543;
6544; GFX11-WGP-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
6545; GFX11-WGP:       ; %bb.0: ; %entry
6546; GFX11-WGP-NEXT:    s_clause 0x1
6547; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6548; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6549; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6550; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
6551; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
6552; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
6553; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
6554; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6555; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6556; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6557; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6558; GFX11-WGP-NEXT:    buffer_gl0_inv
6559; GFX11-WGP-NEXT:    buffer_gl1_inv
6560; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6561; GFX11-WGP-NEXT:    s_endpgm
6562;
6563; GFX11-CU-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
6564; GFX11-CU:       ; %bb.0: ; %entry
6565; GFX11-CU-NEXT:    s_clause 0x1
6566; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6567; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6568; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6569; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
6570; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
6571; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
6572; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
6573; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6574; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6575; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6576; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6577; GFX11-CU-NEXT:    buffer_gl0_inv
6578; GFX11-CU-NEXT:    buffer_gl1_inv
6579; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6580; GFX11-CU-NEXT:    s_endpgm
6581    i32* %out, i32 %in, i32 %old) {
6582entry:
6583  %gep = getelementptr i32, i32* %out, i32 4
6584  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst acquire
6585  %val0 = extractvalue { i32, i1 } %val, 0
6586  store i32 %val0, i32* %out, align 4
6587  ret void
6588}
6589
6590define amdgpu_kernel void @flat_agent_monotonic_seq_cst_ret_cmpxchg(
6591; GFX7-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
6592; GFX7:       ; %bb.0: ; %entry
6593; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6594; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6595; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6596; GFX7-NEXT:    s_add_u32 s4, s0, 16
6597; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6598; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6599; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6600; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6601; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6602; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6603; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6604; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6605; GFX7-NEXT:    buffer_wbinvl1_vol
6606; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6607; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6608; GFX7-NEXT:    flat_store_dword v[0:1], v2
6609; GFX7-NEXT:    s_endpgm
6610;
6611; GFX10-WGP-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
6612; GFX10-WGP:       ; %bb.0: ; %entry
6613; GFX10-WGP-NEXT:    s_clause 0x1
6614; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6615; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6616; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6617; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6618; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6619; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6620; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6621; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6622; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6623; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6624; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6625; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6626; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6627; GFX10-WGP-NEXT:    buffer_gl0_inv
6628; GFX10-WGP-NEXT:    buffer_gl1_inv
6629; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6630; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6631; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6632; GFX10-WGP-NEXT:    s_endpgm
6633;
6634; GFX10-CU-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
6635; GFX10-CU:       ; %bb.0: ; %entry
6636; GFX10-CU-NEXT:    s_clause 0x1
6637; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6638; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6639; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6640; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6641; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6642; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6643; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6644; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6645; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6646; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6647; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6648; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6649; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6650; GFX10-CU-NEXT:    buffer_gl0_inv
6651; GFX10-CU-NEXT:    buffer_gl1_inv
6652; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6653; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6654; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6655; GFX10-CU-NEXT:    s_endpgm
6656;
6657; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
6658; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6659; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6660; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6661; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6662; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6663; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6664; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6665; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6666; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6667; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6668; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6669; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6670; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6671; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6672; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6673; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6674; SKIP-CACHE-INV-NEXT:    s_endpgm
6675;
6676; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
6677; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6678; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6679; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6680; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6681; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6682; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6683; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6684; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6685; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6686; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6687; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6688; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6689;
6690; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
6691; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6692; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6693; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6694; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6695; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6696; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6697; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6698; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6699; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6700; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6701; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6702; GFX90A-TGSPLIT-NEXT:    s_endpgm
6703;
6704; GFX940-NOTTGSPLIT-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
6705; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6706; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6707; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6708; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6709; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6710; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6711; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
6712; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6713; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6714; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6715; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
6716; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6717; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6718;
6719; GFX940-TGSPLIT-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
6720; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6721; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6722; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6723; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6724; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6725; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6726; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
6727; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6728; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6729; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6730; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
6731; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6732; GFX940-TGSPLIT-NEXT:    s_endpgm
6733;
6734; GFX11-WGP-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
6735; GFX11-WGP:       ; %bb.0: ; %entry
6736; GFX11-WGP-NEXT:    s_clause 0x1
6737; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6738; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6739; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6740; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
6741; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
6742; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
6743; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
6744; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6745; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6746; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6747; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6748; GFX11-WGP-NEXT:    buffer_gl0_inv
6749; GFX11-WGP-NEXT:    buffer_gl1_inv
6750; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6751; GFX11-WGP-NEXT:    s_endpgm
6752;
6753; GFX11-CU-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
6754; GFX11-CU:       ; %bb.0: ; %entry
6755; GFX11-CU-NEXT:    s_clause 0x1
6756; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6757; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6758; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6759; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
6760; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
6761; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
6762; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
6763; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6764; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6765; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6766; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6767; GFX11-CU-NEXT:    buffer_gl0_inv
6768; GFX11-CU-NEXT:    buffer_gl1_inv
6769; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6770; GFX11-CU-NEXT:    s_endpgm
6771    i32* %out, i32 %in, i32 %old) {
6772entry:
6773  %gep = getelementptr i32, i32* %out, i32 4
6774  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") monotonic seq_cst
6775  %val0 = extractvalue { i32, i1 } %val, 0
6776  store i32 %val0, i32* %out, align 4
6777  ret void
6778}
6779
6780define amdgpu_kernel void @flat_agent_acquire_seq_cst_ret_cmpxchg(
6781; GFX7-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
6782; GFX7:       ; %bb.0: ; %entry
6783; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6784; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6785; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6786; GFX7-NEXT:    s_add_u32 s4, s0, 16
6787; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6788; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6789; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6790; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6791; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6792; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6793; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6794; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6795; GFX7-NEXT:    buffer_wbinvl1_vol
6796; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6797; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6798; GFX7-NEXT:    flat_store_dword v[0:1], v2
6799; GFX7-NEXT:    s_endpgm
6800;
6801; GFX10-WGP-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
6802; GFX10-WGP:       ; %bb.0: ; %entry
6803; GFX10-WGP-NEXT:    s_clause 0x1
6804; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6805; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6806; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6807; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6808; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6809; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6810; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6811; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6812; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6813; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6814; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6815; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6816; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6817; GFX10-WGP-NEXT:    buffer_gl0_inv
6818; GFX10-WGP-NEXT:    buffer_gl1_inv
6819; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6820; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6821; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6822; GFX10-WGP-NEXT:    s_endpgm
6823;
6824; GFX10-CU-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
6825; GFX10-CU:       ; %bb.0: ; %entry
6826; GFX10-CU-NEXT:    s_clause 0x1
6827; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6828; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6829; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6830; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6831; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6832; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6833; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6834; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6835; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6836; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6837; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6838; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6839; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6840; GFX10-CU-NEXT:    buffer_gl0_inv
6841; GFX10-CU-NEXT:    buffer_gl1_inv
6842; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6843; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6844; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6845; GFX10-CU-NEXT:    s_endpgm
6846;
6847; SKIP-CACHE-INV-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
6848; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6849; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6850; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6851; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6852; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6853; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6854; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6855; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6856; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6857; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6858; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6859; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6860; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6861; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6862; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6863; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6864; SKIP-CACHE-INV-NEXT:    s_endpgm
6865;
6866; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
6867; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6868; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6869; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6870; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6871; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6872; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6873; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6874; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6875; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6876; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6877; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6878; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6879;
6880; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
6881; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6882; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6883; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6884; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6885; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6886; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6887; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6888; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6889; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6890; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6891; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6892; GFX90A-TGSPLIT-NEXT:    s_endpgm
6893;
6894; GFX940-NOTTGSPLIT-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
6895; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6896; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6897; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6898; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6899; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6900; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6901; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
6902; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6903; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6904; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6905; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
6906; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6907; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6908;
6909; GFX940-TGSPLIT-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
6910; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6911; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6912; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6913; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6914; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6915; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6916; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
6917; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6918; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6919; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6920; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
6921; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6922; GFX940-TGSPLIT-NEXT:    s_endpgm
6923;
6924; GFX11-WGP-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
6925; GFX11-WGP:       ; %bb.0: ; %entry
6926; GFX11-WGP-NEXT:    s_clause 0x1
6927; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6928; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6929; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6930; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
6931; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
6932; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
6933; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
6934; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6935; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6936; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6937; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6938; GFX11-WGP-NEXT:    buffer_gl0_inv
6939; GFX11-WGP-NEXT:    buffer_gl1_inv
6940; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
6941; GFX11-WGP-NEXT:    s_endpgm
6942;
6943; GFX11-CU-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
6944; GFX11-CU:       ; %bb.0: ; %entry
6945; GFX11-CU-NEXT:    s_clause 0x1
6946; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
6947; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
6948; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
6949; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
6950; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
6951; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
6952; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
6953; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6954; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6955; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6956; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6957; GFX11-CU-NEXT:    buffer_gl0_inv
6958; GFX11-CU-NEXT:    buffer_gl1_inv
6959; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
6960; GFX11-CU-NEXT:    s_endpgm
6961    i32* %out, i32 %in, i32 %old) {
6962entry:
6963  %gep = getelementptr i32, i32* %out, i32 4
6964  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire seq_cst
6965  %val0 = extractvalue { i32, i1 } %val, 0
6966  store i32 %val0, i32* %out, align 4
6967  ret void
6968}
6969
6970define amdgpu_kernel void @flat_agent_release_seq_cst_ret_cmpxchg(
6971; GFX7-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
6972; GFX7:       ; %bb.0: ; %entry
6973; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6974; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6975; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6976; GFX7-NEXT:    s_add_u32 s4, s0, 16
6977; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6978; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6979; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6980; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6981; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6982; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6983; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6984; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6985; GFX7-NEXT:    buffer_wbinvl1_vol
6986; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6987; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6988; GFX7-NEXT:    flat_store_dword v[0:1], v2
6989; GFX7-NEXT:    s_endpgm
6990;
6991; GFX10-WGP-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
6992; GFX10-WGP:       ; %bb.0: ; %entry
6993; GFX10-WGP-NEXT:    s_clause 0x1
6994; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6995; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6996; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6997; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6998; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6999; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
7000; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7001; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
7002; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7003; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7004; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7005; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7006; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7007; GFX10-WGP-NEXT:    buffer_gl0_inv
7008; GFX10-WGP-NEXT:    buffer_gl1_inv
7009; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7010; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7011; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7012; GFX10-WGP-NEXT:    s_endpgm
7013;
7014; GFX10-CU-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
7015; GFX10-CU:       ; %bb.0: ; %entry
7016; GFX10-CU-NEXT:    s_clause 0x1
7017; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7018; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7019; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7020; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
7021; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
7022; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
7023; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7024; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
7025; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7026; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7027; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7028; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7029; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7030; GFX10-CU-NEXT:    buffer_gl0_inv
7031; GFX10-CU-NEXT:    buffer_gl1_inv
7032; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7033; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7034; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7035; GFX10-CU-NEXT:    s_endpgm
7036;
7037; SKIP-CACHE-INV-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
7038; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7039; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7040; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7041; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7042; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
7043; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
7044; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7045; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7046; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7047; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7048; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7049; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7050; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7051; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7052; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7053; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7054; SKIP-CACHE-INV-NEXT:    s_endpgm
7055;
7056; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
7057; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7058; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7059; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7060; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7061; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7062; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7063; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7064; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7065; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7066; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7067; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7068; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7069;
7070; GFX90A-TGSPLIT-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
7071; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7072; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7073; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7074; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7075; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7076; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7077; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7078; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7079; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7080; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7081; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7082; GFX90A-TGSPLIT-NEXT:    s_endpgm
7083;
7084; GFX940-NOTTGSPLIT-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
7085; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7086; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7087; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7088; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7089; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7090; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7091; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
7092; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7093; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
7094; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7095; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
7096; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7097; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7098;
7099; GFX940-TGSPLIT-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
7100; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7101; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7102; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7103; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7104; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7105; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7106; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
7107; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7108; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
7109; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7110; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
7111; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7112; GFX940-TGSPLIT-NEXT:    s_endpgm
7113;
7114; GFX11-WGP-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
7115; GFX11-WGP:       ; %bb.0: ; %entry
7116; GFX11-WGP-NEXT:    s_clause 0x1
7117; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7118; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7119; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7120; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7121; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
7122; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7123; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7124; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7125; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7126; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7127; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7128; GFX11-WGP-NEXT:    buffer_gl0_inv
7129; GFX11-WGP-NEXT:    buffer_gl1_inv
7130; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7131; GFX11-WGP-NEXT:    s_endpgm
7132;
7133; GFX11-CU-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
7134; GFX11-CU:       ; %bb.0: ; %entry
7135; GFX11-CU-NEXT:    s_clause 0x1
7136; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7137; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7138; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7139; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7140; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
7141; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7142; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7143; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7144; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7145; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7146; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7147; GFX11-CU-NEXT:    buffer_gl0_inv
7148; GFX11-CU-NEXT:    buffer_gl1_inv
7149; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7150; GFX11-CU-NEXT:    s_endpgm
7151    i32* %out, i32 %in, i32 %old) {
7152entry:
7153  %gep = getelementptr i32, i32* %out, i32 4
7154  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release seq_cst
7155  %val0 = extractvalue { i32, i1 } %val, 0
7156  store i32 %val0, i32* %out, align 4
7157  ret void
7158}
7159
7160define amdgpu_kernel void @flat_agent_acq_rel_seq_cst_ret_cmpxchg(
7161; GFX7-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
7162; GFX7:       ; %bb.0: ; %entry
7163; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7164; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7165; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7166; GFX7-NEXT:    s_add_u32 s4, s0, 16
7167; GFX7-NEXT:    s_addc_u32 s5, s1, 0
7168; GFX7-NEXT:    v_mov_b32_e32 v0, s4
7169; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7170; GFX7-NEXT:    v_mov_b32_e32 v1, s5
7171; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7172; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7173; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7174; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7175; GFX7-NEXT:    buffer_wbinvl1_vol
7176; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7177; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7178; GFX7-NEXT:    flat_store_dword v[0:1], v2
7179; GFX7-NEXT:    s_endpgm
7180;
7181; GFX10-WGP-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
7182; GFX10-WGP:       ; %bb.0: ; %entry
7183; GFX10-WGP-NEXT:    s_clause 0x1
7184; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7185; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7186; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7187; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
7188; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
7189; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
7190; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7191; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
7192; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7193; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7194; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7195; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7196; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7197; GFX10-WGP-NEXT:    buffer_gl0_inv
7198; GFX10-WGP-NEXT:    buffer_gl1_inv
7199; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7200; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7201; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7202; GFX10-WGP-NEXT:    s_endpgm
7203;
7204; GFX10-CU-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
7205; GFX10-CU:       ; %bb.0: ; %entry
7206; GFX10-CU-NEXT:    s_clause 0x1
7207; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7208; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7209; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7210; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
7211; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
7212; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
7213; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7214; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
7215; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7216; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7217; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7218; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7219; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7220; GFX10-CU-NEXT:    buffer_gl0_inv
7221; GFX10-CU-NEXT:    buffer_gl1_inv
7222; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7223; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7224; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7225; GFX10-CU-NEXT:    s_endpgm
7226;
7227; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
7228; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7229; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7230; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7231; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7232; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
7233; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
7234; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7235; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7236; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7237; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7238; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7239; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7240; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7241; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7242; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7243; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7244; SKIP-CACHE-INV-NEXT:    s_endpgm
7245;
7246; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
7247; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7248; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7249; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7250; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7251; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7252; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7253; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7254; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7255; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7256; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7257; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7258; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7259;
7260; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
7261; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7262; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7263; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7264; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7265; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7266; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7267; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7268; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7269; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7270; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7271; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7272; GFX90A-TGSPLIT-NEXT:    s_endpgm
7273;
7274; GFX940-NOTTGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
7275; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7276; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7277; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7278; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7279; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7280; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7281; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
7282; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7283; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
7284; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7285; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
7286; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7287; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7288;
7289; GFX940-TGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
7290; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7291; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7292; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7293; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7294; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7295; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7296; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
7297; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7298; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
7299; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7300; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
7301; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7302; GFX940-TGSPLIT-NEXT:    s_endpgm
7303;
7304; GFX11-WGP-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
7305; GFX11-WGP:       ; %bb.0: ; %entry
7306; GFX11-WGP-NEXT:    s_clause 0x1
7307; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7308; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7309; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7310; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7311; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
7312; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7313; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7314; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7315; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7316; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7317; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7318; GFX11-WGP-NEXT:    buffer_gl0_inv
7319; GFX11-WGP-NEXT:    buffer_gl1_inv
7320; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7321; GFX11-WGP-NEXT:    s_endpgm
7322;
7323; GFX11-CU-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
7324; GFX11-CU:       ; %bb.0: ; %entry
7325; GFX11-CU-NEXT:    s_clause 0x1
7326; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7327; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7328; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7329; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7330; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
7331; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7332; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7333; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7334; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7335; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7336; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7337; GFX11-CU-NEXT:    buffer_gl0_inv
7338; GFX11-CU-NEXT:    buffer_gl1_inv
7339; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7340; GFX11-CU-NEXT:    s_endpgm
7341    i32* %out, i32 %in, i32 %old) {
7342entry:
7343  %gep = getelementptr i32, i32* %out, i32 4
7344  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel seq_cst
7345  %val0 = extractvalue { i32, i1 } %val, 0
7346  store i32 %val0, i32* %out, align 4
7347  ret void
7348}
7349
7350define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_ret_cmpxchg(
7351; GFX7-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
7352; GFX7:       ; %bb.0: ; %entry
7353; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7354; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7355; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7356; GFX7-NEXT:    s_add_u32 s4, s0, 16
7357; GFX7-NEXT:    s_addc_u32 s5, s1, 0
7358; GFX7-NEXT:    v_mov_b32_e32 v0, s4
7359; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7360; GFX7-NEXT:    v_mov_b32_e32 v1, s5
7361; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7362; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7363; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7364; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7365; GFX7-NEXT:    buffer_wbinvl1_vol
7366; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7367; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7368; GFX7-NEXT:    flat_store_dword v[0:1], v2
7369; GFX7-NEXT:    s_endpgm
7370;
7371; GFX10-WGP-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
7372; GFX10-WGP:       ; %bb.0: ; %entry
7373; GFX10-WGP-NEXT:    s_clause 0x1
7374; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7375; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7376; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7377; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
7378; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
7379; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
7380; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7381; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
7382; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7383; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7384; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7385; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7386; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7387; GFX10-WGP-NEXT:    buffer_gl0_inv
7388; GFX10-WGP-NEXT:    buffer_gl1_inv
7389; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7390; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7391; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7392; GFX10-WGP-NEXT:    s_endpgm
7393;
7394; GFX10-CU-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
7395; GFX10-CU:       ; %bb.0: ; %entry
7396; GFX10-CU-NEXT:    s_clause 0x1
7397; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7398; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7399; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7400; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
7401; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
7402; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
7403; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7404; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
7405; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7406; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7407; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7408; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7409; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7410; GFX10-CU-NEXT:    buffer_gl0_inv
7411; GFX10-CU-NEXT:    buffer_gl1_inv
7412; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7413; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7414; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7415; GFX10-CU-NEXT:    s_endpgm
7416;
7417; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
7418; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7419; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7420; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7421; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7422; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
7423; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
7424; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7425; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7426; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7427; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7428; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7429; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7430; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7431; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7432; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7433; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7434; SKIP-CACHE-INV-NEXT:    s_endpgm
7435;
7436; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
7437; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7438; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7439; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7440; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7441; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7442; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7443; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7444; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7445; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7446; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7447; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7448; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7449;
7450; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
7451; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7452; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7453; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7454; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7455; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7456; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7457; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7458; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7459; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7460; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7461; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7462; GFX90A-TGSPLIT-NEXT:    s_endpgm
7463;
7464; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
7465; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7466; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7467; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7468; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7469; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7470; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7471; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
7472; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7473; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
7474; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7475; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
7476; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7477; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7478;
7479; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
7480; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7481; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7482; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7483; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7484; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7485; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7486; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
7487; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7488; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
7489; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7490; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
7491; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7492; GFX940-TGSPLIT-NEXT:    s_endpgm
7493;
7494; GFX11-WGP-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
7495; GFX11-WGP:       ; %bb.0: ; %entry
7496; GFX11-WGP-NEXT:    s_clause 0x1
7497; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7498; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7499; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7500; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7501; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
7502; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7503; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
7504; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7505; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7506; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7507; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7508; GFX11-WGP-NEXT:    buffer_gl0_inv
7509; GFX11-WGP-NEXT:    buffer_gl1_inv
7510; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7511; GFX11-WGP-NEXT:    s_endpgm
7512;
7513; GFX11-CU-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
7514; GFX11-CU:       ; %bb.0: ; %entry
7515; GFX11-CU-NEXT:    s_clause 0x1
7516; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
7517; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
7518; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7519; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7520; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
7521; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7522; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
7523; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7524; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7525; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7526; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7527; GFX11-CU-NEXT:    buffer_gl0_inv
7528; GFX11-CU-NEXT:    buffer_gl1_inv
7529; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7530; GFX11-CU-NEXT:    s_endpgm
7531    i32* %out, i32 %in, i32 %old) {
7532entry:
7533  %gep = getelementptr i32, i32* %out, i32 4
7534  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
7535  %val0 = extractvalue { i32, i1 } %val, 0
7536  store i32 %val0, i32* %out, align 4
7537  ret void
7538}
7539
7540define amdgpu_kernel void @flat_agent_one_as_unordered_load(
7541; GFX7-LABEL: flat_agent_one_as_unordered_load:
7542; GFX7:       ; %bb.0: ; %entry
7543; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7544; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7545; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7546; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7547; GFX7-NEXT:    flat_load_dword v2, v[0:1]
7548; GFX7-NEXT:    v_mov_b32_e32 v0, s2
7549; GFX7-NEXT:    v_mov_b32_e32 v1, s3
7550; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7551; GFX7-NEXT:    flat_store_dword v[0:1], v2
7552; GFX7-NEXT:    s_endpgm
7553;
7554; GFX10-WGP-LABEL: flat_agent_one_as_unordered_load:
7555; GFX10-WGP:       ; %bb.0: ; %entry
7556; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7557; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7558; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7559; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7560; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
7561; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
7562; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
7563; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7564; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7565; GFX10-WGP-NEXT:    s_endpgm
7566;
7567; GFX10-CU-LABEL: flat_agent_one_as_unordered_load:
7568; GFX10-CU:       ; %bb.0: ; %entry
7569; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7570; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7571; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7572; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7573; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
7574; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
7575; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
7576; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7577; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7578; GFX10-CU-NEXT:    s_endpgm
7579;
7580; SKIP-CACHE-INV-LABEL: flat_agent_one_as_unordered_load:
7581; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7582; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7583; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7584; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7585; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
7586; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
7587; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7588; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7589; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7590; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7591; SKIP-CACHE-INV-NEXT:    s_endpgm
7592;
7593; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_unordered_load:
7594; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7595; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7596; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7597; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7598; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7599; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
7600; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7601; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7602; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7603; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7604; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7605;
7606; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_unordered_load:
7607; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7608; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7609; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7610; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7611; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7612; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
7613; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7614; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7615; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7616; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7617; GFX90A-TGSPLIT-NEXT:    s_endpgm
7618;
7619; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_unordered_load:
7620; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7621; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7622; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7623; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7624; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7625; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
7626; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7627; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7628; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7629; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7630; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7631;
7632; GFX940-TGSPLIT-LABEL: flat_agent_one_as_unordered_load:
7633; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7634; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7635; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7636; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7637; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7638; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
7639; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7640; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7641; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7642; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7643; GFX940-TGSPLIT-NEXT:    s_endpgm
7644;
7645; GFX11-WGP-LABEL: flat_agent_one_as_unordered_load:
7646; GFX11-WGP:       ; %bb.0: ; %entry
7647; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7648; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7649; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
7650; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
7651; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1]
7652; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7653; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7654; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7655; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7656; GFX11-WGP-NEXT:    s_endpgm
7657;
7658; GFX11-CU-LABEL: flat_agent_one_as_unordered_load:
7659; GFX11-CU:       ; %bb.0: ; %entry
7660; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7661; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7662; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
7663; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
7664; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
7665; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7666; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7667; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7668; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7669; GFX11-CU-NEXT:    s_endpgm
7670    i32* %in, i32* %out) {
7671entry:
7672  %val = load atomic i32, i32* %in syncscope("agent-one-as") unordered, align 4
7673  store i32 %val, i32* %out
7674  ret void
7675}
7676
7677define amdgpu_kernel void @flat_agent_one_as_monotonic_load(
7678; GFX7-LABEL: flat_agent_one_as_monotonic_load:
7679; GFX7:       ; %bb.0: ; %entry
7680; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7681; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7682; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7683; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7684; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
7685; GFX7-NEXT:    v_mov_b32_e32 v0, s2
7686; GFX7-NEXT:    v_mov_b32_e32 v1, s3
7687; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7688; GFX7-NEXT:    flat_store_dword v[0:1], v2
7689; GFX7-NEXT:    s_endpgm
7690;
7691; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_load:
7692; GFX10-WGP:       ; %bb.0: ; %entry
7693; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7694; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7695; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7696; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7697; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
7698; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
7699; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
7700; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7701; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7702; GFX10-WGP-NEXT:    s_endpgm
7703;
7704; GFX10-CU-LABEL: flat_agent_one_as_monotonic_load:
7705; GFX10-CU:       ; %bb.0: ; %entry
7706; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7707; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7708; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7709; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7710; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
7711; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
7712; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
7713; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7714; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7715; GFX10-CU-NEXT:    s_endpgm
7716;
7717; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_load:
7718; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7719; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7720; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7721; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7722; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
7723; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
7724; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7725; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7726; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7727; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7728; SKIP-CACHE-INV-NEXT:    s_endpgm
7729;
7730; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_load:
7731; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7732; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7733; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7734; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7735; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7736; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
7737; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7738; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7739; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7740; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7741; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7742;
7743; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_load:
7744; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7745; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7746; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7747; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7748; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7749; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
7750; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7751; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7752; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7753; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7754; GFX90A-TGSPLIT-NEXT:    s_endpgm
7755;
7756; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_load:
7757; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7758; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7759; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7760; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7761; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7762; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc1
7763; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7764; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7765; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7766; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7767; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7768;
7769; GFX940-TGSPLIT-LABEL: flat_agent_one_as_monotonic_load:
7770; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7771; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7772; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7773; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7774; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7775; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc1
7776; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7777; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7778; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7779; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7780; GFX940-TGSPLIT-NEXT:    s_endpgm
7781;
7782; GFX11-WGP-LABEL: flat_agent_one_as_monotonic_load:
7783; GFX11-WGP:       ; %bb.0: ; %entry
7784; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7785; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7786; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
7787; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
7788; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc
7789; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7790; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7791; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7792; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7793; GFX11-WGP-NEXT:    s_endpgm
7794;
7795; GFX11-CU-LABEL: flat_agent_one_as_monotonic_load:
7796; GFX11-CU:       ; %bb.0: ; %entry
7797; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7798; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7799; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
7800; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
7801; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1] glc
7802; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7803; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7804; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7805; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7806; GFX11-CU-NEXT:    s_endpgm
7807    i32* %in, i32* %out) {
7808entry:
7809  %val = load atomic i32, i32* %in syncscope("agent-one-as") monotonic, align 4
7810  store i32 %val, i32* %out
7811  ret void
7812}
7813
7814define amdgpu_kernel void @flat_agent_one_as_acquire_load(
7815; GFX7-LABEL: flat_agent_one_as_acquire_load:
7816; GFX7:       ; %bb.0: ; %entry
7817; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7818; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7819; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7820; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7821; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
7822; GFX7-NEXT:    s_waitcnt vmcnt(0)
7823; GFX7-NEXT:    buffer_wbinvl1_vol
7824; GFX7-NEXT:    v_mov_b32_e32 v0, s2
7825; GFX7-NEXT:    v_mov_b32_e32 v1, s3
7826; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7827; GFX7-NEXT:    flat_store_dword v[0:1], v2
7828; GFX7-NEXT:    s_endpgm
7829;
7830; GFX10-WGP-LABEL: flat_agent_one_as_acquire_load:
7831; GFX10-WGP:       ; %bb.0: ; %entry
7832; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7833; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7834; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7835; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7836; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
7837; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7838; GFX10-WGP-NEXT:    buffer_gl0_inv
7839; GFX10-WGP-NEXT:    buffer_gl1_inv
7840; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
7841; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
7842; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7843; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7844; GFX10-WGP-NEXT:    s_endpgm
7845;
7846; GFX10-CU-LABEL: flat_agent_one_as_acquire_load:
7847; GFX10-CU:       ; %bb.0: ; %entry
7848; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7849; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7850; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7851; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7852; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
7853; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7854; GFX10-CU-NEXT:    buffer_gl0_inv
7855; GFX10-CU-NEXT:    buffer_gl1_inv
7856; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
7857; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
7858; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7859; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7860; GFX10-CU-NEXT:    s_endpgm
7861;
7862; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_load:
7863; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7864; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7865; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7866; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7867; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
7868; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
7869; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7870; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7871; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7872; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7873; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7874; SKIP-CACHE-INV-NEXT:    s_endpgm
7875;
7876; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_load:
7877; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7878; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7879; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7880; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7881; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7882; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
7883; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7884; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7885; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7886; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7887; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7888; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7889; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7890;
7891; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_load:
7892; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7893; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7894; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7895; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7896; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7897; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
7898; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7899; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7900; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7901; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7902; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7903; GFX90A-TGSPLIT-NEXT:    s_endpgm
7904;
7905; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_load:
7906; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7907; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7908; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7909; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7910; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7911; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc1
7912; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7913; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
7914; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7915; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7916; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7917; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7918; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7919;
7920; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acquire_load:
7921; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7922; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
7923; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7924; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
7925; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
7926; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc1
7927; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7928; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
7929; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
7930; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
7931; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7932; GFX940-TGSPLIT-NEXT:    s_endpgm
7933;
7934; GFX11-WGP-LABEL: flat_agent_one_as_acquire_load:
7935; GFX11-WGP:       ; %bb.0: ; %entry
7936; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7937; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7938; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
7939; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
7940; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc
7941; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
7942; GFX11-WGP-NEXT:    buffer_gl0_inv
7943; GFX11-WGP-NEXT:    buffer_gl1_inv
7944; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
7945; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
7946; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7947; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
7948; GFX11-WGP-NEXT:    s_endpgm
7949;
7950; GFX11-CU-LABEL: flat_agent_one_as_acquire_load:
7951; GFX11-CU:       ; %bb.0: ; %entry
7952; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
7953; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7954; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
7955; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
7956; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1] glc
7957; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
7958; GFX11-CU-NEXT:    buffer_gl0_inv
7959; GFX11-CU-NEXT:    buffer_gl1_inv
7960; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
7961; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
7962; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
7963; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
7964; GFX11-CU-NEXT:    s_endpgm
7965    i32* %in, i32* %out) {
7966entry:
7967  %val = load atomic i32, i32* %in syncscope("agent-one-as") acquire, align 4
7968  store i32 %val, i32* %out
7969  ret void
7970}
7971
7972define amdgpu_kernel void @flat_agent_one_as_seq_cst_load(
7973; GFX7-LABEL: flat_agent_one_as_seq_cst_load:
7974; GFX7:       ; %bb.0: ; %entry
7975; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7976; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7977; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7978; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7979; GFX7-NEXT:    s_waitcnt vmcnt(0)
7980; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
7981; GFX7-NEXT:    s_waitcnt vmcnt(0)
7982; GFX7-NEXT:    buffer_wbinvl1_vol
7983; GFX7-NEXT:    v_mov_b32_e32 v0, s2
7984; GFX7-NEXT:    v_mov_b32_e32 v1, s3
7985; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7986; GFX7-NEXT:    flat_store_dword v[0:1], v2
7987; GFX7-NEXT:    s_endpgm
7988;
7989; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_load:
7990; GFX10-WGP:       ; %bb.0: ; %entry
7991; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
7992; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7993; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7994; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7995; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7996; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7997; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
7998; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7999; GFX10-WGP-NEXT:    buffer_gl0_inv
8000; GFX10-WGP-NEXT:    buffer_gl1_inv
8001; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
8002; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
8003; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8004; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8005; GFX10-WGP-NEXT:    s_endpgm
8006;
8007; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_load:
8008; GFX10-CU:       ; %bb.0: ; %entry
8009; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8010; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8011; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8012; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8013; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8014; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8015; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
8016; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8017; GFX10-CU-NEXT:    buffer_gl0_inv
8018; GFX10-CU-NEXT:    buffer_gl1_inv
8019; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
8020; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
8021; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8022; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8023; GFX10-CU-NEXT:    s_endpgm
8024;
8025; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_load:
8026; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8027; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8028; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8029; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
8030; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
8031; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8032; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
8033; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8034; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8035; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8036; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8037; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8038; SKIP-CACHE-INV-NEXT:    s_endpgm
8039;
8040; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_load:
8041; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8042; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8043; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8044; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8045; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
8046; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8047; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
8048; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8049; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
8050; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
8051; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
8052; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8053; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8054; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8055;
8056; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_load:
8057; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8058; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
8059; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8060; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8061; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
8062; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8063; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
8064; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8065; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8066; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
8067; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
8068; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8069; GFX90A-TGSPLIT-NEXT:    s_endpgm
8070;
8071; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_load:
8072; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8073; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8074; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8075; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8076; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
8077; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8078; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc1
8079; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8080; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
8081; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
8082; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
8083; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8084; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8085; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8086;
8087; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_load:
8088; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8089; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
8090; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8091; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
8092; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
8093; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8094; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc1
8095; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8096; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
8097; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
8098; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
8099; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8100; GFX940-TGSPLIT-NEXT:    s_endpgm
8101;
8102; GFX11-WGP-LABEL: flat_agent_one_as_seq_cst_load:
8103; GFX11-WGP:       ; %bb.0: ; %entry
8104; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8105; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8106; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
8107; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
8108; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8109; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8110; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc
8111; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8112; GFX11-WGP-NEXT:    buffer_gl0_inv
8113; GFX11-WGP-NEXT:    buffer_gl1_inv
8114; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8115; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8116; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8117; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
8118; GFX11-WGP-NEXT:    s_endpgm
8119;
8120; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_load:
8121; GFX11-CU:       ; %bb.0: ; %entry
8122; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
8123; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8124; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
8125; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
8126; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
8127; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8128; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1] glc
8129; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
8130; GFX11-CU-NEXT:    buffer_gl0_inv
8131; GFX11-CU-NEXT:    buffer_gl1_inv
8132; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8133; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8134; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8135; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8136; GFX11-CU-NEXT:    s_endpgm
8137    i32* %in, i32* %out) {
8138entry:
8139  %val = load atomic i32, i32* %in syncscope("agent-one-as") seq_cst, align 4
8140  store i32 %val, i32* %out
8141  ret void
8142}
8143
8144define amdgpu_kernel void @flat_agent_one_as_unordered_store(
8145; GFX7-LABEL: flat_agent_one_as_unordered_store:
8146; GFX7:       ; %bb.0: ; %entry
8147; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
8148; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
8149; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8150; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8151; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8152; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8153; GFX7-NEXT:    flat_store_dword v[0:1], v2
8154; GFX7-NEXT:    s_endpgm
8155;
8156; GFX10-WGP-LABEL: flat_agent_one_as_unordered_store:
8157; GFX10-WGP:       ; %bb.0: ; %entry
8158; GFX10-WGP-NEXT:    s_clause 0x1
8159; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8160; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
8161; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8162; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8163; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8164; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8165; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8166; GFX10-WGP-NEXT:    s_endpgm
8167;
8168; GFX10-CU-LABEL: flat_agent_one_as_unordered_store:
8169; GFX10-CU:       ; %bb.0: ; %entry
8170; GFX10-CU-NEXT:    s_clause 0x1
8171; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8172; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
8173; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8174; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8175; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8176; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8177; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8178; GFX10-CU-NEXT:    s_endpgm
8179;
8180; SKIP-CACHE-INV-LABEL: flat_agent_one_as_unordered_store:
8181; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8182; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
8183; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
8184; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8185; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8186; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8187; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8188; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8189; SKIP-CACHE-INV-NEXT:    s_endpgm
8190;
8191; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_unordered_store:
8192; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8193; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8194; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
8195; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8196; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8197; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8198; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8199; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8200;
8201; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_unordered_store:
8202; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8203; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8204; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
8205; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8206; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8207; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8208; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8209; GFX90A-TGSPLIT-NEXT:    s_endpgm
8210;
8211; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_unordered_store:
8212; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8213; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8214; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
8215; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8216; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8217; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8218; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8219; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8220;
8221; GFX940-TGSPLIT-LABEL: flat_agent_one_as_unordered_store:
8222; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8223; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8224; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
8225; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8226; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8227; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8228; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8229; GFX940-TGSPLIT-NEXT:    s_endpgm
8230;
8231; GFX11-WGP-LABEL: flat_agent_one_as_unordered_store:
8232; GFX11-WGP:       ; %bb.0: ; %entry
8233; GFX11-WGP-NEXT:    s_clause 0x1
8234; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8235; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
8236; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8237; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8238; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8239; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8240; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
8241; GFX11-WGP-NEXT:    s_endpgm
8242;
8243; GFX11-CU-LABEL: flat_agent_one_as_unordered_store:
8244; GFX11-CU:       ; %bb.0: ; %entry
8245; GFX11-CU-NEXT:    s_clause 0x1
8246; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8247; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
8248; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8249; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8250; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8251; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8252; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8253; GFX11-CU-NEXT:    s_endpgm
8254    i32 %in, i32* %out) {
8255entry:
8256  store atomic i32 %in, i32* %out syncscope("agent-one-as") unordered, align 4
8257  ret void
8258}
8259
8260define amdgpu_kernel void @flat_agent_one_as_monotonic_store(
8261; GFX7-LABEL: flat_agent_one_as_monotonic_store:
8262; GFX7:       ; %bb.0: ; %entry
8263; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
8264; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
8265; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8266; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8267; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8268; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8269; GFX7-NEXT:    flat_store_dword v[0:1], v2
8270; GFX7-NEXT:    s_endpgm
8271;
8272; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_store:
8273; GFX10-WGP:       ; %bb.0: ; %entry
8274; GFX10-WGP-NEXT:    s_clause 0x1
8275; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8276; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
8277; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8278; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8279; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8280; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8281; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8282; GFX10-WGP-NEXT:    s_endpgm
8283;
8284; GFX10-CU-LABEL: flat_agent_one_as_monotonic_store:
8285; GFX10-CU:       ; %bb.0: ; %entry
8286; GFX10-CU-NEXT:    s_clause 0x1
8287; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8288; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
8289; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8290; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8291; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8292; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8293; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8294; GFX10-CU-NEXT:    s_endpgm
8295;
8296; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_store:
8297; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8298; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
8299; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
8300; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8301; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8302; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8303; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8304; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8305; SKIP-CACHE-INV-NEXT:    s_endpgm
8306;
8307; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_store:
8308; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8309; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8310; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
8311; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8312; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8313; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8314; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8315; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8316;
8317; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_store:
8318; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8319; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8320; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
8321; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8322; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8323; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8324; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8325; GFX90A-TGSPLIT-NEXT:    s_endpgm
8326;
8327; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_store:
8328; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8329; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8330; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
8331; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8332; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8333; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8334; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc1
8335; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8336;
8337; GFX940-TGSPLIT-LABEL: flat_agent_one_as_monotonic_store:
8338; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8339; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8340; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
8341; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8342; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8343; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8344; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc1
8345; GFX940-TGSPLIT-NEXT:    s_endpgm
8346;
8347; GFX11-WGP-LABEL: flat_agent_one_as_monotonic_store:
8348; GFX11-WGP:       ; %bb.0: ; %entry
8349; GFX11-WGP-NEXT:    s_clause 0x1
8350; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8351; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
8352; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8353; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8354; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8355; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8356; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
8357; GFX11-WGP-NEXT:    s_endpgm
8358;
8359; GFX11-CU-LABEL: flat_agent_one_as_monotonic_store:
8360; GFX11-CU:       ; %bb.0: ; %entry
8361; GFX11-CU-NEXT:    s_clause 0x1
8362; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8363; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
8364; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8365; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8366; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8367; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8368; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8369; GFX11-CU-NEXT:    s_endpgm
8370    i32 %in, i32* %out) {
8371entry:
8372  store atomic i32 %in, i32* %out syncscope("agent-one-as") monotonic, align 4
8373  ret void
8374}
8375
8376define amdgpu_kernel void @flat_agent_one_as_release_store(
8377; GFX7-LABEL: flat_agent_one_as_release_store:
8378; GFX7:       ; %bb.0: ; %entry
8379; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
8380; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
8381; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8382; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8383; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8384; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8385; GFX7-NEXT:    s_waitcnt vmcnt(0)
8386; GFX7-NEXT:    flat_store_dword v[0:1], v2
8387; GFX7-NEXT:    s_endpgm
8388;
8389; GFX10-WGP-LABEL: flat_agent_one_as_release_store:
8390; GFX10-WGP:       ; %bb.0: ; %entry
8391; GFX10-WGP-NEXT:    s_clause 0x1
8392; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8393; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
8394; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8395; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8396; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8397; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8398; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8399; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8400; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8401; GFX10-WGP-NEXT:    s_endpgm
8402;
8403; GFX10-CU-LABEL: flat_agent_one_as_release_store:
8404; GFX10-CU:       ; %bb.0: ; %entry
8405; GFX10-CU-NEXT:    s_clause 0x1
8406; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8407; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
8408; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8409; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8410; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8411; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8412; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8413; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8414; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8415; GFX10-CU-NEXT:    s_endpgm
8416;
8417; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_store:
8418; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8419; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
8420; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
8421; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8422; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8423; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8424; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8425; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8426; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8427; SKIP-CACHE-INV-NEXT:    s_endpgm
8428;
8429; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_store:
8430; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8431; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8432; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
8433; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8434; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8435; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8436; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8437; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8438; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8439;
8440; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_store:
8441; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8442; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8443; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
8444; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8445; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8446; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8447; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8448; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8449; GFX90A-TGSPLIT-NEXT:    s_endpgm
8450;
8451; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_release_store:
8452; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8453; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8454; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
8455; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8456; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8457; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8458; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
8459; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8460; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc1
8461; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8462;
8463; GFX940-TGSPLIT-LABEL: flat_agent_one_as_release_store:
8464; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8465; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8466; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
8467; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8468; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8469; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8470; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
8471; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8472; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc1
8473; GFX940-TGSPLIT-NEXT:    s_endpgm
8474;
8475; GFX11-WGP-LABEL: flat_agent_one_as_release_store:
8476; GFX11-WGP:       ; %bb.0: ; %entry
8477; GFX11-WGP-NEXT:    s_clause 0x1
8478; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8479; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
8480; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8481; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8482; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8483; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8484; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8485; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8486; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
8487; GFX11-WGP-NEXT:    s_endpgm
8488;
8489; GFX11-CU-LABEL: flat_agent_one_as_release_store:
8490; GFX11-CU:       ; %bb.0: ; %entry
8491; GFX11-CU-NEXT:    s_clause 0x1
8492; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8493; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
8494; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8495; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8496; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8497; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8498; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
8499; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8500; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8501; GFX11-CU-NEXT:    s_endpgm
8502    i32 %in, i32* %out) {
8503entry:
8504  store atomic i32 %in, i32* %out syncscope("agent-one-as") release, align 4
8505  ret void
8506}
8507
8508define amdgpu_kernel void @flat_agent_one_as_seq_cst_store(
8509; GFX7-LABEL: flat_agent_one_as_seq_cst_store:
8510; GFX7:       ; %bb.0: ; %entry
8511; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
8512; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
8513; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8514; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8515; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8516; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8517; GFX7-NEXT:    s_waitcnt vmcnt(0)
8518; GFX7-NEXT:    flat_store_dword v[0:1], v2
8519; GFX7-NEXT:    s_endpgm
8520;
8521; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_store:
8522; GFX10-WGP:       ; %bb.0: ; %entry
8523; GFX10-WGP-NEXT:    s_clause 0x1
8524; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8525; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
8526; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8527; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8528; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8529; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8530; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8531; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8532; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8533; GFX10-WGP-NEXT:    s_endpgm
8534;
8535; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_store:
8536; GFX10-CU:       ; %bb.0: ; %entry
8537; GFX10-CU-NEXT:    s_clause 0x1
8538; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8539; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
8540; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8541; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8542; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8543; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8544; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8545; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8546; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8547; GFX10-CU-NEXT:    s_endpgm
8548;
8549; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_store:
8550; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8551; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
8552; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
8553; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8554; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8555; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8556; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8557; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8558; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8559; SKIP-CACHE-INV-NEXT:    s_endpgm
8560;
8561; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_store:
8562; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8563; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8564; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
8565; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8566; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8567; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8568; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8569; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8570; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8571;
8572; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_store:
8573; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8574; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8575; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
8576; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8577; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8578; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8579; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8580; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8581; GFX90A-TGSPLIT-NEXT:    s_endpgm
8582;
8583; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_store:
8584; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8585; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8586; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
8587; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8588; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8589; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8590; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
8591; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8592; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc1
8593; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8594;
8595; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_store:
8596; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8597; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
8598; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
8599; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8600; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8601; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8602; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
8603; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8604; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc1
8605; GFX940-TGSPLIT-NEXT:    s_endpgm
8606;
8607; GFX11-WGP-LABEL: flat_agent_one_as_seq_cst_store:
8608; GFX11-WGP:       ; %bb.0: ; %entry
8609; GFX11-WGP-NEXT:    s_clause 0x1
8610; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8611; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
8612; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8613; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8614; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8615; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8616; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
8617; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8618; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
8619; GFX11-WGP-NEXT:    s_endpgm
8620;
8621; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_store:
8622; GFX11-CU:       ; %bb.0: ; %entry
8623; GFX11-CU-NEXT:    s_clause 0x1
8624; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
8625; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
8626; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8627; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8628; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8629; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8630; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
8631; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8632; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
8633; GFX11-CU-NEXT:    s_endpgm
8634    i32 %in, i32* %out) {
8635entry:
8636  store atomic i32 %in, i32* %out syncscope("agent-one-as") seq_cst, align 4
8637  ret void
8638}
8639
8640define amdgpu_kernel void @flat_agent_one_as_monotonic_atomicrmw(
8641; GFX7-LABEL: flat_agent_one_as_monotonic_atomicrmw:
8642; GFX7:       ; %bb.0: ; %entry
8643; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8644; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
8645; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8646; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8647; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8648; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8649; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
8650; GFX7-NEXT:    s_endpgm
8651;
8652; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_atomicrmw:
8653; GFX10-WGP:       ; %bb.0: ; %entry
8654; GFX10-WGP-NEXT:    s_clause 0x1
8655; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8656; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
8657; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8658; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8659; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8660; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8661; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
8662; GFX10-WGP-NEXT:    s_endpgm
8663;
8664; GFX10-CU-LABEL: flat_agent_one_as_monotonic_atomicrmw:
8665; GFX10-CU:       ; %bb.0: ; %entry
8666; GFX10-CU-NEXT:    s_clause 0x1
8667; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8668; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8669; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8670; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8671; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8672; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8673; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
8674; GFX10-CU-NEXT:    s_endpgm
8675;
8676; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_atomicrmw:
8677; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8678; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8679; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
8680; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8681; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8682; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8683; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8684; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
8685; SKIP-CACHE-INV-NEXT:    s_endpgm
8686;
8687; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_atomicrmw:
8688; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8689; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8690; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8691; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8692; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8693; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8694; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8695; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8696;
8697; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_atomicrmw:
8698; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8699; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8700; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8701; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8702; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8703; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8704; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8705; GFX90A-TGSPLIT-NEXT:    s_endpgm
8706;
8707; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_atomicrmw:
8708; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8709; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8710; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8711; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8712; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8713; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8714; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8715; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8716;
8717; GFX940-TGSPLIT-LABEL: flat_agent_one_as_monotonic_atomicrmw:
8718; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8719; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8720; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8721; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8722; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8723; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8724; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8725; GFX940-TGSPLIT-NEXT:    s_endpgm
8726;
8727; GFX11-WGP-LABEL: flat_agent_one_as_monotonic_atomicrmw:
8728; GFX11-WGP:       ; %bb.0: ; %entry
8729; GFX11-WGP-NEXT:    s_clause 0x1
8730; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8731; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
8732; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8733; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8734; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8735; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8736; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
8737; GFX11-WGP-NEXT:    s_endpgm
8738;
8739; GFX11-CU-LABEL: flat_agent_one_as_monotonic_atomicrmw:
8740; GFX11-CU:       ; %bb.0: ; %entry
8741; GFX11-CU-NEXT:    s_clause 0x1
8742; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8743; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
8744; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8745; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8746; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8747; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8748; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
8749; GFX11-CU-NEXT:    s_endpgm
8750    i32* %out, i32 %in) {
8751entry:
8752  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") monotonic
8753  ret void
8754}
8755
8756define amdgpu_kernel void @flat_agent_one_as_acquire_atomicrmw(
8757; GFX7-LABEL: flat_agent_one_as_acquire_atomicrmw:
8758; GFX7:       ; %bb.0: ; %entry
8759; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8760; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
8761; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8762; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8763; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8764; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8765; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
8766; GFX7-NEXT:    s_waitcnt vmcnt(0)
8767; GFX7-NEXT:    buffer_wbinvl1_vol
8768; GFX7-NEXT:    s_endpgm
8769;
8770; GFX10-WGP-LABEL: flat_agent_one_as_acquire_atomicrmw:
8771; GFX10-WGP:       ; %bb.0: ; %entry
8772; GFX10-WGP-NEXT:    s_clause 0x1
8773; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8774; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
8775; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8776; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8777; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8778; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8779; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
8780; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8781; GFX10-WGP-NEXT:    buffer_gl0_inv
8782; GFX10-WGP-NEXT:    buffer_gl1_inv
8783; GFX10-WGP-NEXT:    s_endpgm
8784;
8785; GFX10-CU-LABEL: flat_agent_one_as_acquire_atomicrmw:
8786; GFX10-CU:       ; %bb.0: ; %entry
8787; GFX10-CU-NEXT:    s_clause 0x1
8788; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8789; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8790; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8791; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8792; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8793; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8794; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
8795; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8796; GFX10-CU-NEXT:    buffer_gl0_inv
8797; GFX10-CU-NEXT:    buffer_gl1_inv
8798; GFX10-CU-NEXT:    s_endpgm
8799;
8800; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_atomicrmw:
8801; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8802; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8803; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
8804; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8805; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8806; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8807; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8808; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
8809; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8810; SKIP-CACHE-INV-NEXT:    s_endpgm
8811;
8812; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_atomicrmw:
8813; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8814; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8815; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8816; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8817; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8818; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8819; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8820; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8821; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
8822; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8823;
8824; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_atomicrmw:
8825; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8826; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8827; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8828; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8829; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8830; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8831; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8832; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8833; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8834; GFX90A-TGSPLIT-NEXT:    s_endpgm
8835;
8836; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_atomicrmw:
8837; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8838; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8839; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8840; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8841; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8842; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8843; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8844; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8845; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
8846; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8847;
8848; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acquire_atomicrmw:
8849; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8850; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8851; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8852; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8853; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8854; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8855; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8856; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8857; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
8858; GFX940-TGSPLIT-NEXT:    s_endpgm
8859;
8860; GFX11-WGP-LABEL: flat_agent_one_as_acquire_atomicrmw:
8861; GFX11-WGP:       ; %bb.0: ; %entry
8862; GFX11-WGP-NEXT:    s_clause 0x1
8863; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8864; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
8865; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8866; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
8867; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
8868; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
8869; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
8870; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8871; GFX11-WGP-NEXT:    buffer_gl0_inv
8872; GFX11-WGP-NEXT:    buffer_gl1_inv
8873; GFX11-WGP-NEXT:    s_endpgm
8874;
8875; GFX11-CU-LABEL: flat_agent_one_as_acquire_atomicrmw:
8876; GFX11-CU:       ; %bb.0: ; %entry
8877; GFX11-CU-NEXT:    s_clause 0x1
8878; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8879; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
8880; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
8881; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
8882; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
8883; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
8884; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
8885; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8886; GFX11-CU-NEXT:    buffer_gl0_inv
8887; GFX11-CU-NEXT:    buffer_gl1_inv
8888; GFX11-CU-NEXT:    s_endpgm
8889    i32* %out, i32 %in) {
8890entry:
8891  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") acquire
8892  ret void
8893}
8894
8895define amdgpu_kernel void @flat_agent_one_as_release_atomicrmw(
8896; GFX7-LABEL: flat_agent_one_as_release_atomicrmw:
8897; GFX7:       ; %bb.0: ; %entry
8898; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8899; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
8900; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8901; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8902; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8903; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8904; GFX7-NEXT:    s_waitcnt vmcnt(0)
8905; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
8906; GFX7-NEXT:    s_endpgm
8907;
8908; GFX10-WGP-LABEL: flat_agent_one_as_release_atomicrmw:
8909; GFX10-WGP:       ; %bb.0: ; %entry
8910; GFX10-WGP-NEXT:    s_clause 0x1
8911; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8912; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
8913; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8914; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8915; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8916; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8917; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8918; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8919; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
8920; GFX10-WGP-NEXT:    s_endpgm
8921;
8922; GFX10-CU-LABEL: flat_agent_one_as_release_atomicrmw:
8923; GFX10-CU:       ; %bb.0: ; %entry
8924; GFX10-CU-NEXT:    s_clause 0x1
8925; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8926; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
8927; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8928; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8929; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8930; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8931; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8932; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8933; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
8934; GFX10-CU-NEXT:    s_endpgm
8935;
8936; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_atomicrmw:
8937; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8938; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8939; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
8940; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8941; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8942; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8943; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8944; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8945; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
8946; SKIP-CACHE-INV-NEXT:    s_endpgm
8947;
8948; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_atomicrmw:
8949; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8950; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8951; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8952; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8953; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8954; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8955; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8956; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8957; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8958;
8959; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_atomicrmw:
8960; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8961; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8962; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
8963; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8964; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8965; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
8966; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8967; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8968; GFX90A-TGSPLIT-NEXT:    s_endpgm
8969;
8970; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_release_atomicrmw:
8971; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8972; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8973; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8974; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8975; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8976; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8977; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
8978; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8979; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8980; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8981;
8982; GFX940-TGSPLIT-LABEL: flat_agent_one_as_release_atomicrmw:
8983; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8984; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8985; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
8986; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8987; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8988; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
8989; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
8990; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8991; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
8992; GFX940-TGSPLIT-NEXT:    s_endpgm
8993;
8994; GFX11-WGP-LABEL: flat_agent_one_as_release_atomicrmw:
8995; GFX11-WGP:       ; %bb.0: ; %entry
8996; GFX11-WGP-NEXT:    s_clause 0x1
8997; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
8998; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
8999; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9000; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9001; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9002; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9003; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9004; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9005; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
9006; GFX11-WGP-NEXT:    s_endpgm
9007;
9008; GFX11-CU-LABEL: flat_agent_one_as_release_atomicrmw:
9009; GFX11-CU:       ; %bb.0: ; %entry
9010; GFX11-CU-NEXT:    s_clause 0x1
9011; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9012; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
9013; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9014; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9015; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9016; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9017; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9018; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9019; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
9020; GFX11-CU-NEXT:    s_endpgm
9021    i32* %out, i32 %in) {
9022entry:
9023  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") release
9024  ret void
9025}
9026
9027define amdgpu_kernel void @flat_agent_one_as_acq_rel_atomicrmw(
9028; GFX7-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
9029; GFX7:       ; %bb.0: ; %entry
9030; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9031; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
9032; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9033; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9034; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9035; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9036; GFX7-NEXT:    s_waitcnt vmcnt(0)
9037; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
9038; GFX7-NEXT:    s_waitcnt vmcnt(0)
9039; GFX7-NEXT:    buffer_wbinvl1_vol
9040; GFX7-NEXT:    s_endpgm
9041;
9042; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
9043; GFX10-WGP:       ; %bb.0: ; %entry
9044; GFX10-WGP-NEXT:    s_clause 0x1
9045; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9046; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
9047; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9048; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9049; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9050; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9051; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9052; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9053; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
9054; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9055; GFX10-WGP-NEXT:    buffer_gl0_inv
9056; GFX10-WGP-NEXT:    buffer_gl1_inv
9057; GFX10-WGP-NEXT:    s_endpgm
9058;
9059; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
9060; GFX10-CU:       ; %bb.0: ; %entry
9061; GFX10-CU-NEXT:    s_clause 0x1
9062; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9063; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
9064; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9065; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9066; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9067; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9068; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9069; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9070; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
9071; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9072; GFX10-CU-NEXT:    buffer_gl0_inv
9073; GFX10-CU-NEXT:    buffer_gl1_inv
9074; GFX10-CU-NEXT:    s_endpgm
9075;
9076; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
9077; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9078; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9079; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
9080; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9081; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9082; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9083; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9084; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9085; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
9086; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9087; SKIP-CACHE-INV-NEXT:    s_endpgm
9088;
9089; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
9090; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9091; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9092; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9093; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9094; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9095; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9096; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9097; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
9098; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9099; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9100; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9101;
9102; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
9103; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9104; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9105; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9106; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9107; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9108; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9109; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9110; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
9111; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9112; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9113; GFX90A-TGSPLIT-NEXT:    s_endpgm
9114;
9115; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
9116; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9117; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9118; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9119; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9120; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9121; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9122; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
9123; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9124; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
9125; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9126; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
9127; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9128;
9129; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
9130; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9131; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9132; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9133; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9134; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9135; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9136; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
9137; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9138; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
9139; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9140; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
9141; GFX940-TGSPLIT-NEXT:    s_endpgm
9142;
9143; GFX11-WGP-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
9144; GFX11-WGP:       ; %bb.0: ; %entry
9145; GFX11-WGP-NEXT:    s_clause 0x1
9146; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9147; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
9148; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9149; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9150; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9151; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9152; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9153; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9154; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
9155; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9156; GFX11-WGP-NEXT:    buffer_gl0_inv
9157; GFX11-WGP-NEXT:    buffer_gl1_inv
9158; GFX11-WGP-NEXT:    s_endpgm
9159;
9160; GFX11-CU-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
9161; GFX11-CU:       ; %bb.0: ; %entry
9162; GFX11-CU-NEXT:    s_clause 0x1
9163; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9164; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
9165; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9166; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9167; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9168; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9169; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9170; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9171; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
9172; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9173; GFX11-CU-NEXT:    buffer_gl0_inv
9174; GFX11-CU-NEXT:    buffer_gl1_inv
9175; GFX11-CU-NEXT:    s_endpgm
9176    i32* %out, i32 %in) {
9177entry:
9178  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") acq_rel
9179  ret void
9180}
9181
9182define amdgpu_kernel void @flat_agent_one_as_seq_cst_atomicrmw(
9183; GFX7-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
9184; GFX7:       ; %bb.0: ; %entry
9185; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9186; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
9187; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9188; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9189; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9190; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9191; GFX7-NEXT:    s_waitcnt vmcnt(0)
9192; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
9193; GFX7-NEXT:    s_waitcnt vmcnt(0)
9194; GFX7-NEXT:    buffer_wbinvl1_vol
9195; GFX7-NEXT:    s_endpgm
9196;
9197; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
9198; GFX10-WGP:       ; %bb.0: ; %entry
9199; GFX10-WGP-NEXT:    s_clause 0x1
9200; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9201; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
9202; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9203; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9204; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9205; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9206; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9207; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9208; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
9209; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9210; GFX10-WGP-NEXT:    buffer_gl0_inv
9211; GFX10-WGP-NEXT:    buffer_gl1_inv
9212; GFX10-WGP-NEXT:    s_endpgm
9213;
9214; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
9215; GFX10-CU:       ; %bb.0: ; %entry
9216; GFX10-CU-NEXT:    s_clause 0x1
9217; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9218; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
9219; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9220; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9221; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9222; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9223; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9224; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9225; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
9226; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9227; GFX10-CU-NEXT:    buffer_gl0_inv
9228; GFX10-CU-NEXT:    buffer_gl1_inv
9229; GFX10-CU-NEXT:    s_endpgm
9230;
9231; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
9232; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9233; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9234; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
9235; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9236; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9237; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9238; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9239; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9240; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
9241; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9242; SKIP-CACHE-INV-NEXT:    s_endpgm
9243;
9244; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
9245; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9246; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9247; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9248; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9249; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9250; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9251; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9252; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
9253; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9254; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9255; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9256;
9257; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
9258; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9259; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9260; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9261; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9262; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9263; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9264; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9265; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
9266; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9267; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9268; GFX90A-TGSPLIT-NEXT:    s_endpgm
9269;
9270; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
9271; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9272; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9273; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9274; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9275; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9276; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9277; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
9278; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9279; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
9280; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9281; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
9282; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9283;
9284; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
9285; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9286; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9287; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9288; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9289; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9290; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9291; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
9292; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9293; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
9294; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9295; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
9296; GFX940-TGSPLIT-NEXT:    s_endpgm
9297;
9298; GFX11-WGP-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
9299; GFX11-WGP:       ; %bb.0: ; %entry
9300; GFX11-WGP-NEXT:    s_clause 0x1
9301; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9302; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
9303; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9304; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9305; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9306; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9307; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9308; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9309; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
9310; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9311; GFX11-WGP-NEXT:    buffer_gl0_inv
9312; GFX11-WGP-NEXT:    buffer_gl1_inv
9313; GFX11-WGP-NEXT:    s_endpgm
9314;
9315; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
9316; GFX11-CU:       ; %bb.0: ; %entry
9317; GFX11-CU-NEXT:    s_clause 0x1
9318; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9319; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
9320; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9321; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9322; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9323; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9324; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9325; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9326; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
9327; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9328; GFX11-CU-NEXT:    buffer_gl0_inv
9329; GFX11-CU-NEXT:    buffer_gl1_inv
9330; GFX11-CU-NEXT:    s_endpgm
9331    i32* %out, i32 %in) {
9332entry:
9333  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") seq_cst
9334  ret void
9335}
9336
9337define amdgpu_kernel void @flat_agent_one_as_acquire_ret_atomicrmw(
9338; GFX7-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
9339; GFX7:       ; %bb.0: ; %entry
9340; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9341; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
9342; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9343; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9344; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9345; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9346; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9347; GFX7-NEXT:    s_waitcnt vmcnt(0)
9348; GFX7-NEXT:    buffer_wbinvl1_vol
9349; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9350; GFX7-NEXT:    flat_store_dword v[0:1], v2
9351; GFX7-NEXT:    s_endpgm
9352;
9353; GFX10-WGP-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
9354; GFX10-WGP:       ; %bb.0: ; %entry
9355; GFX10-WGP-NEXT:    s_clause 0x1
9356; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9357; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
9358; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9359; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9360; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9361; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9362; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9363; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9364; GFX10-WGP-NEXT:    buffer_gl0_inv
9365; GFX10-WGP-NEXT:    buffer_gl1_inv
9366; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9367; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
9368; GFX10-WGP-NEXT:    s_endpgm
9369;
9370; GFX10-CU-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
9371; GFX10-CU:       ; %bb.0: ; %entry
9372; GFX10-CU-NEXT:    s_clause 0x1
9373; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9374; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
9375; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9376; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9377; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9378; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9379; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9380; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9381; GFX10-CU-NEXT:    buffer_gl0_inv
9382; GFX10-CU-NEXT:    buffer_gl1_inv
9383; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9384; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
9385; GFX10-CU-NEXT:    s_endpgm
9386;
9387; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
9388; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9389; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9390; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
9391; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9392; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9393; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9394; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9395; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9396; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9397; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
9398; SKIP-CACHE-INV-NEXT:    s_endpgm
9399;
9400; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
9401; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9402; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9403; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9404; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9405; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9406; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9407; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9408; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9409; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9410; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9411; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9412; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9413;
9414; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
9415; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9416; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9417; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9418; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9419; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9420; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9421; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9422; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9423; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9424; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9425; GFX90A-TGSPLIT-NEXT:    s_endpgm
9426;
9427; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
9428; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9429; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9430; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9431; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9432; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9433; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9434; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
9435; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9436; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
9437; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9438; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9439; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9440;
9441; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
9442; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9443; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9444; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9445; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9446; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9447; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9448; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
9449; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9450; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
9451; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9452; GFX940-TGSPLIT-NEXT:    s_endpgm
9453;
9454; GFX11-WGP-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
9455; GFX11-WGP:       ; %bb.0: ; %entry
9456; GFX11-WGP-NEXT:    s_clause 0x1
9457; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9458; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
9459; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9460; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9461; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9462; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9463; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
9464; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9465; GFX11-WGP-NEXT:    buffer_gl0_inv
9466; GFX11-WGP-NEXT:    buffer_gl1_inv
9467; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9468; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
9469; GFX11-WGP-NEXT:    s_endpgm
9470;
9471; GFX11-CU-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
9472; GFX11-CU:       ; %bb.0: ; %entry
9473; GFX11-CU-NEXT:    s_clause 0x1
9474; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9475; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
9476; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9477; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9478; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9479; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9480; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
9481; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9482; GFX11-CU-NEXT:    buffer_gl0_inv
9483; GFX11-CU-NEXT:    buffer_gl1_inv
9484; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9485; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
9486; GFX11-CU-NEXT:    s_endpgm
9487    i32* %out, i32 %in) {
9488entry:
9489  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") acquire
9490  store i32 %val, i32* %out, align 4
9491  ret void
9492}
9493
9494define amdgpu_kernel void @flat_agent_one_as_acq_rel_ret_atomicrmw(
9495; GFX7-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
9496; GFX7:       ; %bb.0: ; %entry
9497; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9498; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
9499; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9500; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9501; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9502; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9503; GFX7-NEXT:    s_waitcnt vmcnt(0)
9504; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9505; GFX7-NEXT:    s_waitcnt vmcnt(0)
9506; GFX7-NEXT:    buffer_wbinvl1_vol
9507; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9508; GFX7-NEXT:    flat_store_dword v[0:1], v2
9509; GFX7-NEXT:    s_endpgm
9510;
9511; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
9512; GFX10-WGP:       ; %bb.0: ; %entry
9513; GFX10-WGP-NEXT:    s_clause 0x1
9514; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9515; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
9516; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9517; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9518; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9519; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9520; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9521; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9522; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9523; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9524; GFX10-WGP-NEXT:    buffer_gl0_inv
9525; GFX10-WGP-NEXT:    buffer_gl1_inv
9526; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9527; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
9528; GFX10-WGP-NEXT:    s_endpgm
9529;
9530; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
9531; GFX10-CU:       ; %bb.0: ; %entry
9532; GFX10-CU-NEXT:    s_clause 0x1
9533; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9534; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
9535; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9536; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9537; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9538; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9539; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9540; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9541; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9542; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9543; GFX10-CU-NEXT:    buffer_gl0_inv
9544; GFX10-CU-NEXT:    buffer_gl1_inv
9545; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9546; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
9547; GFX10-CU-NEXT:    s_endpgm
9548;
9549; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
9550; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9551; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9552; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
9553; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9554; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9555; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9556; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9557; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9558; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9559; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9560; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
9561; SKIP-CACHE-INV-NEXT:    s_endpgm
9562;
9563; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
9564; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9565; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9566; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9567; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9568; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9569; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9570; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9571; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9572; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9573; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9574; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9575; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9576; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9577;
9578; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
9579; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9580; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9581; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9582; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9583; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9584; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9585; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9586; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9587; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9588; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9589; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9590; GFX90A-TGSPLIT-NEXT:    s_endpgm
9591;
9592; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
9593; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9594; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9595; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9596; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9597; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9598; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9599; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
9600; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9601; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
9602; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9603; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
9604; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9605; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9606; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9607;
9608; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
9609; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9610; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9611; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9612; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9613; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9614; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9615; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
9616; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9617; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
9618; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9619; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
9620; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9621; GFX940-TGSPLIT-NEXT:    s_endpgm
9622;
9623; GFX11-WGP-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
9624; GFX11-WGP:       ; %bb.0: ; %entry
9625; GFX11-WGP-NEXT:    s_clause 0x1
9626; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9627; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
9628; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9629; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9630; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9631; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9632; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9633; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9634; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
9635; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9636; GFX11-WGP-NEXT:    buffer_gl0_inv
9637; GFX11-WGP-NEXT:    buffer_gl1_inv
9638; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9639; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
9640; GFX11-WGP-NEXT:    s_endpgm
9641;
9642; GFX11-CU-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
9643; GFX11-CU:       ; %bb.0: ; %entry
9644; GFX11-CU-NEXT:    s_clause 0x1
9645; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9646; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
9647; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9648; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9649; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9650; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9651; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9652; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9653; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
9654; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9655; GFX11-CU-NEXT:    buffer_gl0_inv
9656; GFX11-CU-NEXT:    buffer_gl1_inv
9657; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9658; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
9659; GFX11-CU-NEXT:    s_endpgm
9660    i32* %out, i32 %in) {
9661entry:
9662  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") acq_rel
9663  store i32 %val, i32* %out, align 4
9664  ret void
9665}
9666
9667define amdgpu_kernel void @flat_agent_one_as_seq_cst_ret_atomicrmw(
9668; GFX7-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
9669; GFX7:       ; %bb.0: ; %entry
9670; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9671; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
9672; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9673; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9674; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9675; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9676; GFX7-NEXT:    s_waitcnt vmcnt(0)
9677; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9678; GFX7-NEXT:    s_waitcnt vmcnt(0)
9679; GFX7-NEXT:    buffer_wbinvl1_vol
9680; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9681; GFX7-NEXT:    flat_store_dword v[0:1], v2
9682; GFX7-NEXT:    s_endpgm
9683;
9684; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
9685; GFX10-WGP:       ; %bb.0: ; %entry
9686; GFX10-WGP-NEXT:    s_clause 0x1
9687; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9688; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
9689; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9690; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9691; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9692; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9693; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9694; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9695; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9696; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9697; GFX10-WGP-NEXT:    buffer_gl0_inv
9698; GFX10-WGP-NEXT:    buffer_gl1_inv
9699; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9700; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
9701; GFX10-WGP-NEXT:    s_endpgm
9702;
9703; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
9704; GFX10-CU:       ; %bb.0: ; %entry
9705; GFX10-CU-NEXT:    s_clause 0x1
9706; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9707; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
9708; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9709; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9710; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9711; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9712; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9713; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9714; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9715; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9716; GFX10-CU-NEXT:    buffer_gl0_inv
9717; GFX10-CU-NEXT:    buffer_gl1_inv
9718; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9719; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
9720; GFX10-CU-NEXT:    s_endpgm
9721;
9722; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
9723; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9724; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9725; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
9726; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9727; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9728; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9729; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9730; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9731; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9732; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9733; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
9734; SKIP-CACHE-INV-NEXT:    s_endpgm
9735;
9736; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
9737; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9738; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9739; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9740; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9741; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9742; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9743; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9744; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9745; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9746; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9747; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9748; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9749; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9750;
9751; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
9752; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9753; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9754; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
9755; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9756; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9757; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
9758; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9759; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
9760; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9761; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9762; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9763; GFX90A-TGSPLIT-NEXT:    s_endpgm
9764;
9765; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
9766; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9767; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9768; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9769; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9770; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9771; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9772; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
9773; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9774; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
9775; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9776; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
9777; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9778; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9779; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9780;
9781; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
9782; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9783; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9784; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
9785; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9786; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9787; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
9788; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
9789; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9790; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
9791; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9792; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
9793; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9794; GFX940-TGSPLIT-NEXT:    s_endpgm
9795;
9796; GFX11-WGP-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
9797; GFX11-WGP:       ; %bb.0: ; %entry
9798; GFX11-WGP-NEXT:    s_clause 0x1
9799; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9800; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x8
9801; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9802; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9803; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9804; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9805; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9806; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9807; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
9808; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
9809; GFX11-WGP-NEXT:    buffer_gl0_inv
9810; GFX11-WGP-NEXT:    buffer_gl1_inv
9811; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9812; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
9813; GFX11-WGP-NEXT:    s_endpgm
9814;
9815; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
9816; GFX11-CU:       ; %bb.0: ; %entry
9817; GFX11-CU-NEXT:    s_clause 0x1
9818; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9819; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x8
9820; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9821; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9822; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9823; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9824; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9825; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9826; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
9827; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
9828; GFX11-CU-NEXT:    buffer_gl0_inv
9829; GFX11-CU-NEXT:    buffer_gl1_inv
9830; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9831; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
9832; GFX11-CU-NEXT:    s_endpgm
9833    i32* %out, i32 %in) {
9834entry:
9835  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") seq_cst
9836  store i32 %val, i32* %out, align 4
9837  ret void
9838}
9839
9840define amdgpu_kernel void @flat_agent_one_as_monotonic_monotonic_cmpxchg(
9841; GFX7-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
9842; GFX7:       ; %bb.0: ; %entry
9843; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9844; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9845; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9846; GFX7-NEXT:    s_add_u32 s0, s0, 16
9847; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9848; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9849; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9850; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9851; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9852; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9853; GFX7-NEXT:    s_endpgm
9854;
9855; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
9856; GFX10-WGP:       ; %bb.0: ; %entry
9857; GFX10-WGP-NEXT:    s_clause 0x1
9858; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9859; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9860; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9861; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9862; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9863; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9864; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9865; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9866; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9867; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9868; GFX10-WGP-NEXT:    s_endpgm
9869;
9870; GFX10-CU-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
9871; GFX10-CU:       ; %bb.0: ; %entry
9872; GFX10-CU-NEXT:    s_clause 0x1
9873; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9874; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9875; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9876; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9877; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9878; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9879; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9880; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9881; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9882; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9883; GFX10-CU-NEXT:    s_endpgm
9884;
9885; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
9886; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9887; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9888; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9889; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9890; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9891; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9892; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9893; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9894; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9895; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9896; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9897; SKIP-CACHE-INV-NEXT:    s_endpgm
9898;
9899; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
9900; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9901; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9902; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9903; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9904; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9905; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9906; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9907; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9908;
9909; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
9910; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9911; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9912; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9913; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9914; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9915; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9916; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9917; GFX90A-TGSPLIT-NEXT:    s_endpgm
9918;
9919; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
9920; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9921; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9922; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9923; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9924; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9925; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9926; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9927; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9928;
9929; GFX940-TGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
9930; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9931; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9932; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9933; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9934; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9935; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9936; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9937; GFX940-TGSPLIT-NEXT:    s_endpgm
9938;
9939; GFX11-WGP-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
9940; GFX11-WGP:       ; %bb.0: ; %entry
9941; GFX11-WGP-NEXT:    s_clause 0x1
9942; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9943; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9944; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9945; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
9946; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
9947; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
9948; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
9949; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9950; GFX11-WGP-NEXT:    s_endpgm
9951;
9952; GFX11-CU-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
9953; GFX11-CU:       ; %bb.0: ; %entry
9954; GFX11-CU-NEXT:    s_clause 0x1
9955; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
9956; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
9957; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
9958; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
9959; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
9960; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
9961; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
9962; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9963; GFX11-CU-NEXT:    s_endpgm
9964    i32* %out, i32 %in, i32 %old) {
9965entry:
9966  %gep = getelementptr i32, i32* %out, i32 4
9967  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") monotonic monotonic
9968  ret void
9969}
9970
9971define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_cmpxchg(
9972; GFX7-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
9973; GFX7:       ; %bb.0: ; %entry
9974; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9975; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9976; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9977; GFX7-NEXT:    s_add_u32 s0, s0, 16
9978; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9979; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9980; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9981; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9982; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9983; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9984; GFX7-NEXT:    s_waitcnt vmcnt(0)
9985; GFX7-NEXT:    buffer_wbinvl1_vol
9986; GFX7-NEXT:    s_endpgm
9987;
9988; GFX10-WGP-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
9989; GFX10-WGP:       ; %bb.0: ; %entry
9990; GFX10-WGP-NEXT:    s_clause 0x1
9991; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9992; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9993; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9994; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9995; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9996; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9997; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9998; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9999; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10000; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10001; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10002; GFX10-WGP-NEXT:    buffer_gl0_inv
10003; GFX10-WGP-NEXT:    buffer_gl1_inv
10004; GFX10-WGP-NEXT:    s_endpgm
10005;
10006; GFX10-CU-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
10007; GFX10-CU:       ; %bb.0: ; %entry
10008; GFX10-CU-NEXT:    s_clause 0x1
10009; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10010; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10011; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10012; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10013; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10014; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10015; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10016; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10017; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10018; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10019; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10020; GFX10-CU-NEXT:    buffer_gl0_inv
10021; GFX10-CU-NEXT:    buffer_gl1_inv
10022; GFX10-CU-NEXT:    s_endpgm
10023;
10024; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
10025; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10026; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10027; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10028; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10029; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10030; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10031; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10032; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10033; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10034; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10035; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10036; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10037; SKIP-CACHE-INV-NEXT:    s_endpgm
10038;
10039; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
10040; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10041; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10042; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10043; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10044; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10045; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10046; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10047; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10048; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
10049; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10050;
10051; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
10052; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10053; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10054; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10055; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10056; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10057; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10058; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10059; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10060; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10061; GFX90A-TGSPLIT-NEXT:    s_endpgm
10062;
10063; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
10064; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10065; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10066; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10067; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10068; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10069; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10070; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10071; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10072; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
10073; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10074;
10075; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
10076; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10077; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10078; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10079; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10080; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10081; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10082; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10083; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10084; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
10085; GFX940-TGSPLIT-NEXT:    s_endpgm
10086;
10087; GFX11-WGP-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
10088; GFX11-WGP:       ; %bb.0: ; %entry
10089; GFX11-WGP-NEXT:    s_clause 0x1
10090; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10091; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10092; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10093; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10094; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
10095; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10096; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
10097; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10098; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10099; GFX11-WGP-NEXT:    buffer_gl0_inv
10100; GFX11-WGP-NEXT:    buffer_gl1_inv
10101; GFX11-WGP-NEXT:    s_endpgm
10102;
10103; GFX11-CU-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
10104; GFX11-CU:       ; %bb.0: ; %entry
10105; GFX11-CU-NEXT:    s_clause 0x1
10106; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10107; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10108; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10109; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10110; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
10111; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10112; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
10113; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10114; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10115; GFX11-CU-NEXT:    buffer_gl0_inv
10116; GFX11-CU-NEXT:    buffer_gl1_inv
10117; GFX11-CU-NEXT:    s_endpgm
10118    i32* %out, i32 %in, i32 %old) {
10119entry:
10120  %gep = getelementptr i32, i32* %out, i32 4
10121  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire monotonic
10122  ret void
10123}
10124
10125define amdgpu_kernel void @flat_agent_one_as_release_monotonic_cmpxchg(
10126; GFX7-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
10127; GFX7:       ; %bb.0: ; %entry
10128; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10129; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10130; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10131; GFX7-NEXT:    s_add_u32 s0, s0, 16
10132; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10133; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10134; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10135; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10136; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10137; GFX7-NEXT:    s_waitcnt vmcnt(0)
10138; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10139; GFX7-NEXT:    s_endpgm
10140;
10141; GFX10-WGP-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
10142; GFX10-WGP:       ; %bb.0: ; %entry
10143; GFX10-WGP-NEXT:    s_clause 0x1
10144; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10145; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10146; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10147; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10148; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10149; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10150; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10151; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10152; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10153; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10154; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10155; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10156; GFX10-WGP-NEXT:    s_endpgm
10157;
10158; GFX10-CU-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
10159; GFX10-CU:       ; %bb.0: ; %entry
10160; GFX10-CU-NEXT:    s_clause 0x1
10161; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10162; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10163; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10164; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10165; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10166; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10167; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10168; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10169; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10170; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10171; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10172; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10173; GFX10-CU-NEXT:    s_endpgm
10174;
10175; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
10176; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10177; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10178; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10179; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10180; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10181; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10182; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10183; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10184; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10185; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10186; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10187; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10188; SKIP-CACHE-INV-NEXT:    s_endpgm
10189;
10190; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
10191; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10192; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10193; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10194; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10195; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10196; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10197; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10198; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10199; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10200;
10201; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
10202; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10203; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10204; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10205; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10206; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10207; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10208; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10209; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10210; GFX90A-TGSPLIT-NEXT:    s_endpgm
10211;
10212; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
10213; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10214; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10215; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10216; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10217; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10218; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10219; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
10220; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10221; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10222; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10223;
10224; GFX940-TGSPLIT-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
10225; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10226; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10227; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10228; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10229; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10230; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10231; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
10232; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10233; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10234; GFX940-TGSPLIT-NEXT:    s_endpgm
10235;
10236; GFX11-WGP-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
10237; GFX11-WGP:       ; %bb.0: ; %entry
10238; GFX11-WGP-NEXT:    s_clause 0x1
10239; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10240; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10241; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10242; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10243; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
10244; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10245; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
10246; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
10247; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10248; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10249; GFX11-WGP-NEXT:    s_endpgm
10250;
10251; GFX11-CU-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
10252; GFX11-CU:       ; %bb.0: ; %entry
10253; GFX11-CU-NEXT:    s_clause 0x1
10254; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10255; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10256; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10257; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10258; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
10259; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10260; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
10261; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
10262; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10263; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10264; GFX11-CU-NEXT:    s_endpgm
10265    i32* %out, i32 %in, i32 %old) {
10266entry:
10267  %gep = getelementptr i32, i32* %out, i32 4
10268  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release monotonic
10269  ret void
10270}
10271
10272define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_cmpxchg(
10273; GFX7-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
10274; GFX7:       ; %bb.0: ; %entry
10275; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10276; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10277; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10278; GFX7-NEXT:    s_add_u32 s0, s0, 16
10279; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10280; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10281; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10282; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10283; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10284; GFX7-NEXT:    s_waitcnt vmcnt(0)
10285; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10286; GFX7-NEXT:    s_waitcnt vmcnt(0)
10287; GFX7-NEXT:    buffer_wbinvl1_vol
10288; GFX7-NEXT:    s_endpgm
10289;
10290; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
10291; GFX10-WGP:       ; %bb.0: ; %entry
10292; GFX10-WGP-NEXT:    s_clause 0x1
10293; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10294; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10295; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10296; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10297; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10298; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10299; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10300; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10301; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10302; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10303; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10304; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10305; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10306; GFX10-WGP-NEXT:    buffer_gl0_inv
10307; GFX10-WGP-NEXT:    buffer_gl1_inv
10308; GFX10-WGP-NEXT:    s_endpgm
10309;
10310; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
10311; GFX10-CU:       ; %bb.0: ; %entry
10312; GFX10-CU-NEXT:    s_clause 0x1
10313; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10314; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10315; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10316; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10317; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10318; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10319; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10320; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10321; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10322; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10323; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10324; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10325; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10326; GFX10-CU-NEXT:    buffer_gl0_inv
10327; GFX10-CU-NEXT:    buffer_gl1_inv
10328; GFX10-CU-NEXT:    s_endpgm
10329;
10330; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
10331; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10332; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10333; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10334; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10335; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10336; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10337; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10338; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10339; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10340; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10341; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10342; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10343; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10344; SKIP-CACHE-INV-NEXT:    s_endpgm
10345;
10346; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
10347; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10348; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10349; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10350; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10351; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10352; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10353; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10354; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10355; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10356; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
10357; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10358;
10359; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
10360; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10361; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10362; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10363; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10364; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10365; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10366; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10367; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10368; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10369; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10370; GFX90A-TGSPLIT-NEXT:    s_endpgm
10371;
10372; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
10373; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10374; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10375; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10376; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10377; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10378; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10379; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
10380; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10381; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10382; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10383; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
10384; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10385;
10386; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
10387; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10388; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10389; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10390; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10391; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10392; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10393; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
10394; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10395; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10396; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10397; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
10398; GFX940-TGSPLIT-NEXT:    s_endpgm
10399;
10400; GFX11-WGP-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
10401; GFX11-WGP:       ; %bb.0: ; %entry
10402; GFX11-WGP-NEXT:    s_clause 0x1
10403; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10404; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10405; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10406; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10407; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
10408; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10409; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
10410; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
10411; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10412; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10413; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10414; GFX11-WGP-NEXT:    buffer_gl0_inv
10415; GFX11-WGP-NEXT:    buffer_gl1_inv
10416; GFX11-WGP-NEXT:    s_endpgm
10417;
10418; GFX11-CU-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
10419; GFX11-CU:       ; %bb.0: ; %entry
10420; GFX11-CU-NEXT:    s_clause 0x1
10421; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10422; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10423; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10424; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10425; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
10426; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10427; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
10428; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
10429; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10430; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10431; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10432; GFX11-CU-NEXT:    buffer_gl0_inv
10433; GFX11-CU-NEXT:    buffer_gl1_inv
10434; GFX11-CU-NEXT:    s_endpgm
10435    i32* %out, i32 %in, i32 %old) {
10436entry:
10437  %gep = getelementptr i32, i32* %out, i32 4
10438  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel monotonic
10439  ret void
10440}
10441
10442define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_cmpxchg(
10443; GFX7-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
10444; GFX7:       ; %bb.0: ; %entry
10445; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10446; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10447; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10448; GFX7-NEXT:    s_add_u32 s0, s0, 16
10449; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10450; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10451; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10452; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10453; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10454; GFX7-NEXT:    s_waitcnt vmcnt(0)
10455; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10456; GFX7-NEXT:    s_waitcnt vmcnt(0)
10457; GFX7-NEXT:    buffer_wbinvl1_vol
10458; GFX7-NEXT:    s_endpgm
10459;
10460; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
10461; GFX10-WGP:       ; %bb.0: ; %entry
10462; GFX10-WGP-NEXT:    s_clause 0x1
10463; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10464; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10465; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10466; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10467; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10468; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10469; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10470; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10471; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10472; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10473; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10474; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10475; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10476; GFX10-WGP-NEXT:    buffer_gl0_inv
10477; GFX10-WGP-NEXT:    buffer_gl1_inv
10478; GFX10-WGP-NEXT:    s_endpgm
10479;
10480; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
10481; GFX10-CU:       ; %bb.0: ; %entry
10482; GFX10-CU-NEXT:    s_clause 0x1
10483; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10484; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10485; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10486; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10487; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10488; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10489; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10490; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10491; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10492; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10493; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10494; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10495; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10496; GFX10-CU-NEXT:    buffer_gl0_inv
10497; GFX10-CU-NEXT:    buffer_gl1_inv
10498; GFX10-CU-NEXT:    s_endpgm
10499;
10500; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
10501; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10502; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10503; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10504; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10505; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10506; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10507; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10508; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10509; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10510; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10511; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10512; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10513; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10514; SKIP-CACHE-INV-NEXT:    s_endpgm
10515;
10516; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
10517; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10518; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10519; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10520; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10521; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10522; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10523; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10524; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10525; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10526; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
10527; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10528;
10529; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
10530; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10531; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10532; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10533; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10534; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10535; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10536; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10537; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10538; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10539; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10540; GFX90A-TGSPLIT-NEXT:    s_endpgm
10541;
10542; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
10543; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10544; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10545; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10546; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10547; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10548; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10549; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
10550; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10551; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10552; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10553; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
10554; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10555;
10556; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
10557; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10558; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10559; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10560; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10561; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10562; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10563; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
10564; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10565; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10566; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10567; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
10568; GFX940-TGSPLIT-NEXT:    s_endpgm
10569;
10570; GFX11-WGP-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
10571; GFX11-WGP:       ; %bb.0: ; %entry
10572; GFX11-WGP-NEXT:    s_clause 0x1
10573; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10574; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10575; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10576; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10577; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
10578; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10579; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
10580; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
10581; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10582; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10583; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10584; GFX11-WGP-NEXT:    buffer_gl0_inv
10585; GFX11-WGP-NEXT:    buffer_gl1_inv
10586; GFX11-WGP-NEXT:    s_endpgm
10587;
10588; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
10589; GFX11-CU:       ; %bb.0: ; %entry
10590; GFX11-CU-NEXT:    s_clause 0x1
10591; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10592; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10593; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10594; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10595; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
10596; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10597; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
10598; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
10599; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10600; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10601; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10602; GFX11-CU-NEXT:    buffer_gl0_inv
10603; GFX11-CU-NEXT:    buffer_gl1_inv
10604; GFX11-CU-NEXT:    s_endpgm
10605    i32* %out, i32 %in, i32 %old) {
10606entry:
10607  %gep = getelementptr i32, i32* %out, i32 4
10608  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst monotonic
10609  ret void
10610}
10611
10612define amdgpu_kernel void @flat_agent_one_as_monotonic_acquire_cmpxchg(
10613; GFX7-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
10614; GFX7:       ; %bb.0: ; %entry
10615; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10616; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10617; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10618; GFX7-NEXT:    s_add_u32 s0, s0, 16
10619; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10620; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10621; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10622; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10623; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10624; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10625; GFX7-NEXT:    s_waitcnt vmcnt(0)
10626; GFX7-NEXT:    buffer_wbinvl1_vol
10627; GFX7-NEXT:    s_endpgm
10628;
10629; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
10630; GFX10-WGP:       ; %bb.0: ; %entry
10631; GFX10-WGP-NEXT:    s_clause 0x1
10632; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10633; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10634; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10635; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10636; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10637; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10638; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10639; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10640; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10641; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10642; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10643; GFX10-WGP-NEXT:    buffer_gl0_inv
10644; GFX10-WGP-NEXT:    buffer_gl1_inv
10645; GFX10-WGP-NEXT:    s_endpgm
10646;
10647; GFX10-CU-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
10648; GFX10-CU:       ; %bb.0: ; %entry
10649; GFX10-CU-NEXT:    s_clause 0x1
10650; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10651; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10652; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10653; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10654; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10655; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10656; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10657; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10658; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10659; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10660; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10661; GFX10-CU-NEXT:    buffer_gl0_inv
10662; GFX10-CU-NEXT:    buffer_gl1_inv
10663; GFX10-CU-NEXT:    s_endpgm
10664;
10665; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
10666; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10667; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10668; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10669; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10670; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10671; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10672; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10673; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10674; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10675; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10676; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10677; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10678; SKIP-CACHE-INV-NEXT:    s_endpgm
10679;
10680; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
10681; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10682; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10683; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10684; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10685; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10686; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10687; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10688; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10689; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
10690; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10691;
10692; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
10693; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10694; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10695; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10696; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10697; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10698; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10699; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10700; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10701; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10702; GFX90A-TGSPLIT-NEXT:    s_endpgm
10703;
10704; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
10705; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10706; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10707; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10708; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10709; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10710; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10711; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10712; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10713; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
10714; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10715;
10716; GFX940-TGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
10717; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10718; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10719; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10720; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10721; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10722; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10723; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10724; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10725; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
10726; GFX940-TGSPLIT-NEXT:    s_endpgm
10727;
10728; GFX11-WGP-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
10729; GFX11-WGP:       ; %bb.0: ; %entry
10730; GFX11-WGP-NEXT:    s_clause 0x1
10731; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10732; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10733; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10734; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10735; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
10736; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10737; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
10738; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10739; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10740; GFX11-WGP-NEXT:    buffer_gl0_inv
10741; GFX11-WGP-NEXT:    buffer_gl1_inv
10742; GFX11-WGP-NEXT:    s_endpgm
10743;
10744; GFX11-CU-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
10745; GFX11-CU:       ; %bb.0: ; %entry
10746; GFX11-CU-NEXT:    s_clause 0x1
10747; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10748; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10749; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10750; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10751; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
10752; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10753; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
10754; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10755; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10756; GFX11-CU-NEXT:    buffer_gl0_inv
10757; GFX11-CU-NEXT:    buffer_gl1_inv
10758; GFX11-CU-NEXT:    s_endpgm
10759    i32* %out, i32 %in, i32 %old) {
10760entry:
10761  %gep = getelementptr i32, i32* %out, i32 4
10762  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") monotonic acquire
10763  ret void
10764}
10765
10766define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_cmpxchg(
10767; GFX7-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
10768; GFX7:       ; %bb.0: ; %entry
10769; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10770; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10771; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10772; GFX7-NEXT:    s_add_u32 s0, s0, 16
10773; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10774; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10775; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10776; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10777; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10778; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10779; GFX7-NEXT:    s_waitcnt vmcnt(0)
10780; GFX7-NEXT:    buffer_wbinvl1_vol
10781; GFX7-NEXT:    s_endpgm
10782;
10783; GFX10-WGP-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
10784; GFX10-WGP:       ; %bb.0: ; %entry
10785; GFX10-WGP-NEXT:    s_clause 0x1
10786; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10787; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10788; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10789; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10790; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10791; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10792; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10793; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10794; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10795; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10796; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10797; GFX10-WGP-NEXT:    buffer_gl0_inv
10798; GFX10-WGP-NEXT:    buffer_gl1_inv
10799; GFX10-WGP-NEXT:    s_endpgm
10800;
10801; GFX10-CU-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
10802; GFX10-CU:       ; %bb.0: ; %entry
10803; GFX10-CU-NEXT:    s_clause 0x1
10804; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10805; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10806; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10807; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10808; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10809; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10810; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10811; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10812; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10813; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10814; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10815; GFX10-CU-NEXT:    buffer_gl0_inv
10816; GFX10-CU-NEXT:    buffer_gl1_inv
10817; GFX10-CU-NEXT:    s_endpgm
10818;
10819; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
10820; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10821; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10822; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10823; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10824; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10825; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10826; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10827; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10828; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10829; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10830; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10831; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10832; SKIP-CACHE-INV-NEXT:    s_endpgm
10833;
10834; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
10835; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10836; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10837; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10838; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10839; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10840; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10841; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10842; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10843; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
10844; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10845;
10846; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
10847; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10848; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10849; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10850; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10851; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10852; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10853; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10854; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10855; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10856; GFX90A-TGSPLIT-NEXT:    s_endpgm
10857;
10858; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
10859; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10860; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10861; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10862; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10863; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10864; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10865; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10866; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10867; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
10868; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10869;
10870; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
10871; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10872; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10873; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10874; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10875; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10876; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10877; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
10878; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10879; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
10880; GFX940-TGSPLIT-NEXT:    s_endpgm
10881;
10882; GFX11-WGP-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
10883; GFX11-WGP:       ; %bb.0: ; %entry
10884; GFX11-WGP-NEXT:    s_clause 0x1
10885; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10886; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10887; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10888; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
10889; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
10890; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
10891; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
10892; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10893; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10894; GFX11-WGP-NEXT:    buffer_gl0_inv
10895; GFX11-WGP-NEXT:    buffer_gl1_inv
10896; GFX11-WGP-NEXT:    s_endpgm
10897;
10898; GFX11-CU-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
10899; GFX11-CU:       ; %bb.0: ; %entry
10900; GFX11-CU-NEXT:    s_clause 0x1
10901; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
10902; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
10903; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
10904; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
10905; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
10906; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
10907; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
10908; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
10909; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10910; GFX11-CU-NEXT:    buffer_gl0_inv
10911; GFX11-CU-NEXT:    buffer_gl1_inv
10912; GFX11-CU-NEXT:    s_endpgm
10913    i32* %out, i32 %in, i32 %old) {
10914entry:
10915  %gep = getelementptr i32, i32* %out, i32 4
10916  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire acquire
10917  ret void
10918}
10919
10920define amdgpu_kernel void @flat_agent_one_as_release_acquire_cmpxchg(
10921; GFX7-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
10922; GFX7:       ; %bb.0: ; %entry
10923; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10924; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10925; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10926; GFX7-NEXT:    s_add_u32 s0, s0, 16
10927; GFX7-NEXT:    s_addc_u32 s1, s1, 0
10928; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10929; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10930; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10931; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10932; GFX7-NEXT:    s_waitcnt vmcnt(0)
10933; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10934; GFX7-NEXT:    s_waitcnt vmcnt(0)
10935; GFX7-NEXT:    buffer_wbinvl1_vol
10936; GFX7-NEXT:    s_endpgm
10937;
10938; GFX10-WGP-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
10939; GFX10-WGP:       ; %bb.0: ; %entry
10940; GFX10-WGP-NEXT:    s_clause 0x1
10941; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10942; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10943; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10944; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
10945; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
10946; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10947; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10948; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10949; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10950; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10951; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10952; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10953; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10954; GFX10-WGP-NEXT:    buffer_gl0_inv
10955; GFX10-WGP-NEXT:    buffer_gl1_inv
10956; GFX10-WGP-NEXT:    s_endpgm
10957;
10958; GFX10-CU-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
10959; GFX10-CU:       ; %bb.0: ; %entry
10960; GFX10-CU-NEXT:    s_clause 0x1
10961; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10962; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10963; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10964; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
10965; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
10966; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10967; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10968; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10969; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10970; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10971; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10972; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10973; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10974; GFX10-CU-NEXT:    buffer_gl0_inv
10975; GFX10-CU-NEXT:    buffer_gl1_inv
10976; GFX10-CU-NEXT:    s_endpgm
10977;
10978; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
10979; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10980; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10981; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10982; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10983; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
10984; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
10985; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10986; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10987; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10988; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10989; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10990; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
10991; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10992; SKIP-CACHE-INV-NEXT:    s_endpgm
10993;
10994; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
10995; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10996; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10997; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10998; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10999; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11000; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11001; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11002; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11003; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11004; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
11005; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11006;
11007; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
11008; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11009; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11010; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11011; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11012; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11013; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11014; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11015; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11016; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11017; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11018; GFX90A-TGSPLIT-NEXT:    s_endpgm
11019;
11020; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
11021; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11022; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11023; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11024; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11025; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11026; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11027; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
11028; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11029; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11030; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11031; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
11032; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11033;
11034; GFX940-TGSPLIT-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
11035; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11036; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11037; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11038; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11039; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11040; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11041; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
11042; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11043; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11044; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11045; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
11046; GFX940-TGSPLIT-NEXT:    s_endpgm
11047;
11048; GFX11-WGP-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
11049; GFX11-WGP:       ; %bb.0: ; %entry
11050; GFX11-WGP-NEXT:    s_clause 0x1
11051; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11052; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11053; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11054; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
11055; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
11056; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
11057; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
11058; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11059; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11060; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11061; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11062; GFX11-WGP-NEXT:    buffer_gl0_inv
11063; GFX11-WGP-NEXT:    buffer_gl1_inv
11064; GFX11-WGP-NEXT:    s_endpgm
11065;
11066; GFX11-CU-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
11067; GFX11-CU:       ; %bb.0: ; %entry
11068; GFX11-CU-NEXT:    s_clause 0x1
11069; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11070; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11071; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11072; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
11073; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
11074; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
11075; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
11076; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11077; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11078; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11079; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11080; GFX11-CU-NEXT:    buffer_gl0_inv
11081; GFX11-CU-NEXT:    buffer_gl1_inv
11082; GFX11-CU-NEXT:    s_endpgm
11083    i32* %out, i32 %in, i32 %old) {
11084entry:
11085  %gep = getelementptr i32, i32* %out, i32 4
11086  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release acquire
11087  ret void
11088}
11089
11090define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_cmpxchg(
11091; GFX7-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
11092; GFX7:       ; %bb.0: ; %entry
11093; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11094; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11095; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11096; GFX7-NEXT:    s_add_u32 s0, s0, 16
11097; GFX7-NEXT:    s_addc_u32 s1, s1, 0
11098; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11099; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11100; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11101; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11102; GFX7-NEXT:    s_waitcnt vmcnt(0)
11103; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11104; GFX7-NEXT:    s_waitcnt vmcnt(0)
11105; GFX7-NEXT:    buffer_wbinvl1_vol
11106; GFX7-NEXT:    s_endpgm
11107;
11108; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
11109; GFX10-WGP:       ; %bb.0: ; %entry
11110; GFX10-WGP-NEXT:    s_clause 0x1
11111; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11112; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11113; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11114; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
11115; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
11116; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11117; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11118; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11119; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11120; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11121; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11122; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11123; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11124; GFX10-WGP-NEXT:    buffer_gl0_inv
11125; GFX10-WGP-NEXT:    buffer_gl1_inv
11126; GFX10-WGP-NEXT:    s_endpgm
11127;
11128; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
11129; GFX10-CU:       ; %bb.0: ; %entry
11130; GFX10-CU-NEXT:    s_clause 0x1
11131; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11132; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11133; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11134; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
11135; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
11136; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11137; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11138; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11139; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11140; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11141; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11142; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11143; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11144; GFX10-CU-NEXT:    buffer_gl0_inv
11145; GFX10-CU-NEXT:    buffer_gl1_inv
11146; GFX10-CU-NEXT:    s_endpgm
11147;
11148; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
11149; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11150; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11151; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11152; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11153; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
11154; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
11155; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11156; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11157; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11158; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11159; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11160; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11161; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11162; SKIP-CACHE-INV-NEXT:    s_endpgm
11163;
11164; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
11165; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11166; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11167; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11168; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11169; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11170; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11171; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11172; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11173; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11174; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
11175; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11176;
11177; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
11178; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11179; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11180; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11181; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11182; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11183; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11184; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11185; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11186; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11187; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11188; GFX90A-TGSPLIT-NEXT:    s_endpgm
11189;
11190; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
11191; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11192; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11193; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11194; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11195; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11196; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11197; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
11198; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11199; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11200; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11201; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
11202; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11203;
11204; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
11205; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11206; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11207; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11208; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11209; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11210; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11211; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
11212; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11213; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11214; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11215; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
11216; GFX940-TGSPLIT-NEXT:    s_endpgm
11217;
11218; GFX11-WGP-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
11219; GFX11-WGP:       ; %bb.0: ; %entry
11220; GFX11-WGP-NEXT:    s_clause 0x1
11221; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11222; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11223; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11224; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
11225; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
11226; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
11227; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
11228; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11229; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11230; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11231; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11232; GFX11-WGP-NEXT:    buffer_gl0_inv
11233; GFX11-WGP-NEXT:    buffer_gl1_inv
11234; GFX11-WGP-NEXT:    s_endpgm
11235;
11236; GFX11-CU-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
11237; GFX11-CU:       ; %bb.0: ; %entry
11238; GFX11-CU-NEXT:    s_clause 0x1
11239; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11240; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11241; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11242; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
11243; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
11244; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
11245; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
11246; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11247; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11248; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11249; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11250; GFX11-CU-NEXT:    buffer_gl0_inv
11251; GFX11-CU-NEXT:    buffer_gl1_inv
11252; GFX11-CU-NEXT:    s_endpgm
11253    i32* %out, i32 %in, i32 %old) {
11254entry:
11255  %gep = getelementptr i32, i32* %out, i32 4
11256  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel acquire
11257  ret void
11258}
11259
11260define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_cmpxchg(
11261; GFX7-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
11262; GFX7:       ; %bb.0: ; %entry
11263; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11264; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11265; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11266; GFX7-NEXT:    s_add_u32 s0, s0, 16
11267; GFX7-NEXT:    s_addc_u32 s1, s1, 0
11268; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11269; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11270; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11271; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11272; GFX7-NEXT:    s_waitcnt vmcnt(0)
11273; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11274; GFX7-NEXT:    s_waitcnt vmcnt(0)
11275; GFX7-NEXT:    buffer_wbinvl1_vol
11276; GFX7-NEXT:    s_endpgm
11277;
11278; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
11279; GFX10-WGP:       ; %bb.0: ; %entry
11280; GFX10-WGP-NEXT:    s_clause 0x1
11281; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11282; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11283; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11284; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
11285; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
11286; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11287; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11288; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11289; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11290; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11291; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11292; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11293; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11294; GFX10-WGP-NEXT:    buffer_gl0_inv
11295; GFX10-WGP-NEXT:    buffer_gl1_inv
11296; GFX10-WGP-NEXT:    s_endpgm
11297;
11298; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
11299; GFX10-CU:       ; %bb.0: ; %entry
11300; GFX10-CU-NEXT:    s_clause 0x1
11301; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11302; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11303; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11304; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
11305; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
11306; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11307; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11308; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11309; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11310; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11311; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11312; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11313; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11314; GFX10-CU-NEXT:    buffer_gl0_inv
11315; GFX10-CU-NEXT:    buffer_gl1_inv
11316; GFX10-CU-NEXT:    s_endpgm
11317;
11318; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
11319; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11320; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11321; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11322; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11323; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
11324; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
11325; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11326; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11327; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11328; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11329; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11330; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11331; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11332; SKIP-CACHE-INV-NEXT:    s_endpgm
11333;
11334; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
11335; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11336; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11337; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11338; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11339; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11340; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11341; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11342; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11343; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11344; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
11345; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11346;
11347; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
11348; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11349; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11350; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11351; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11352; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11353; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11354; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11355; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11356; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11357; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11358; GFX90A-TGSPLIT-NEXT:    s_endpgm
11359;
11360; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
11361; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11362; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11363; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11364; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11365; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11366; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11367; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
11368; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11369; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11370; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11371; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
11372; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11373;
11374; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
11375; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11376; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11377; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11378; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11379; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11380; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11381; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
11382; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11383; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11384; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11385; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
11386; GFX940-TGSPLIT-NEXT:    s_endpgm
11387;
11388; GFX11-WGP-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
11389; GFX11-WGP:       ; %bb.0: ; %entry
11390; GFX11-WGP-NEXT:    s_clause 0x1
11391; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11392; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11393; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11394; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
11395; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
11396; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
11397; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
11398; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11399; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11400; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11401; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11402; GFX11-WGP-NEXT:    buffer_gl0_inv
11403; GFX11-WGP-NEXT:    buffer_gl1_inv
11404; GFX11-WGP-NEXT:    s_endpgm
11405;
11406; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
11407; GFX11-CU:       ; %bb.0: ; %entry
11408; GFX11-CU-NEXT:    s_clause 0x1
11409; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11410; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11411; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11412; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
11413; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
11414; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
11415; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
11416; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11417; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11418; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11419; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11420; GFX11-CU-NEXT:    buffer_gl0_inv
11421; GFX11-CU-NEXT:    buffer_gl1_inv
11422; GFX11-CU-NEXT:    s_endpgm
11423    i32* %out, i32 %in, i32 %old) {
11424entry:
11425  %gep = getelementptr i32, i32* %out, i32 4
11426  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst acquire
11427  ret void
11428}
11429
11430define amdgpu_kernel void @flat_agent_one_as_monotonic_seq_cst_cmpxchg(
11431; GFX7-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
11432; GFX7:       ; %bb.0: ; %entry
11433; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11434; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11435; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11436; GFX7-NEXT:    s_add_u32 s0, s0, 16
11437; GFX7-NEXT:    s_addc_u32 s1, s1, 0
11438; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11439; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11440; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11441; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11442; GFX7-NEXT:    s_waitcnt vmcnt(0)
11443; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11444; GFX7-NEXT:    s_waitcnt vmcnt(0)
11445; GFX7-NEXT:    buffer_wbinvl1_vol
11446; GFX7-NEXT:    s_endpgm
11447;
11448; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
11449; GFX10-WGP:       ; %bb.0: ; %entry
11450; GFX10-WGP-NEXT:    s_clause 0x1
11451; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11452; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11453; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11454; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
11455; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
11456; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11457; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11458; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11459; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11460; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11461; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11462; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11463; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11464; GFX10-WGP-NEXT:    buffer_gl0_inv
11465; GFX10-WGP-NEXT:    buffer_gl1_inv
11466; GFX10-WGP-NEXT:    s_endpgm
11467;
11468; GFX10-CU-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
11469; GFX10-CU:       ; %bb.0: ; %entry
11470; GFX10-CU-NEXT:    s_clause 0x1
11471; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11472; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11473; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11474; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
11475; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
11476; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11477; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11478; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11479; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11480; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11481; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11482; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11483; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11484; GFX10-CU-NEXT:    buffer_gl0_inv
11485; GFX10-CU-NEXT:    buffer_gl1_inv
11486; GFX10-CU-NEXT:    s_endpgm
11487;
11488; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
11489; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11490; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11491; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11492; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11493; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
11494; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
11495; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11496; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11497; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11498; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11499; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11500; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11501; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11502; SKIP-CACHE-INV-NEXT:    s_endpgm
11503;
11504; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
11505; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11506; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11507; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11508; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11509; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11510; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11511; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11512; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11513; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11514; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
11515; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11516;
11517; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
11518; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11519; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11520; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11521; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11522; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11523; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11524; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11525; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11526; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11527; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11528; GFX90A-TGSPLIT-NEXT:    s_endpgm
11529;
11530; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
11531; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11532; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11533; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11534; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11535; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11536; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11537; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
11538; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11539; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11540; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11541; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
11542; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11543;
11544; GFX940-TGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
11545; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11546; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11547; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11548; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11549; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11550; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11551; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
11552; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11553; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11554; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11555; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
11556; GFX940-TGSPLIT-NEXT:    s_endpgm
11557;
11558; GFX11-WGP-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
11559; GFX11-WGP:       ; %bb.0: ; %entry
11560; GFX11-WGP-NEXT:    s_clause 0x1
11561; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11562; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11563; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11564; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
11565; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
11566; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
11567; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
11568; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11569; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11570; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11571; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11572; GFX11-WGP-NEXT:    buffer_gl0_inv
11573; GFX11-WGP-NEXT:    buffer_gl1_inv
11574; GFX11-WGP-NEXT:    s_endpgm
11575;
11576; GFX11-CU-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
11577; GFX11-CU:       ; %bb.0: ; %entry
11578; GFX11-CU-NEXT:    s_clause 0x1
11579; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11580; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11581; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11582; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
11583; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
11584; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
11585; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
11586; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11587; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11588; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11589; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11590; GFX11-CU-NEXT:    buffer_gl0_inv
11591; GFX11-CU-NEXT:    buffer_gl1_inv
11592; GFX11-CU-NEXT:    s_endpgm
11593    i32* %out, i32 %in, i32 %old) {
11594entry:
11595  %gep = getelementptr i32, i32* %out, i32 4
11596  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") monotonic seq_cst
11597  ret void
11598}
11599
11600define amdgpu_kernel void @flat_agent_one_as_acquire_seq_cst_cmpxchg(
11601; GFX7-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
11602; GFX7:       ; %bb.0: ; %entry
11603; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11604; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11605; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11606; GFX7-NEXT:    s_add_u32 s0, s0, 16
11607; GFX7-NEXT:    s_addc_u32 s1, s1, 0
11608; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11609; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11610; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11611; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11612; GFX7-NEXT:    s_waitcnt vmcnt(0)
11613; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11614; GFX7-NEXT:    s_waitcnt vmcnt(0)
11615; GFX7-NEXT:    buffer_wbinvl1_vol
11616; GFX7-NEXT:    s_endpgm
11617;
11618; GFX10-WGP-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
11619; GFX10-WGP:       ; %bb.0: ; %entry
11620; GFX10-WGP-NEXT:    s_clause 0x1
11621; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11622; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11623; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11624; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
11625; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
11626; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11627; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11628; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11629; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11630; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11631; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11632; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11633; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11634; GFX10-WGP-NEXT:    buffer_gl0_inv
11635; GFX10-WGP-NEXT:    buffer_gl1_inv
11636; GFX10-WGP-NEXT:    s_endpgm
11637;
11638; GFX10-CU-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
11639; GFX10-CU:       ; %bb.0: ; %entry
11640; GFX10-CU-NEXT:    s_clause 0x1
11641; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11642; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11643; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11644; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
11645; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
11646; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11647; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11648; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11649; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11650; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11651; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11652; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11653; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11654; GFX10-CU-NEXT:    buffer_gl0_inv
11655; GFX10-CU-NEXT:    buffer_gl1_inv
11656; GFX10-CU-NEXT:    s_endpgm
11657;
11658; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
11659; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11660; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11661; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11662; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11663; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
11664; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
11665; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11666; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11667; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11668; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11669; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11670; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11671; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11672; SKIP-CACHE-INV-NEXT:    s_endpgm
11673;
11674; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
11675; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11676; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11677; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11678; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11679; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11680; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11681; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11682; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11683; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11684; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
11685; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11686;
11687; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
11688; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11689; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11690; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11691; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11692; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11693; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11694; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11695; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11696; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11697; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11698; GFX90A-TGSPLIT-NEXT:    s_endpgm
11699;
11700; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
11701; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11702; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11703; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11704; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11705; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11706; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11707; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
11708; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11709; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11710; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11711; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
11712; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11713;
11714; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
11715; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11716; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11717; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11718; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11719; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11720; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11721; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
11722; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11723; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11724; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11725; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
11726; GFX940-TGSPLIT-NEXT:    s_endpgm
11727;
11728; GFX11-WGP-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
11729; GFX11-WGP:       ; %bb.0: ; %entry
11730; GFX11-WGP-NEXT:    s_clause 0x1
11731; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11732; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11733; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11734; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
11735; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
11736; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
11737; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
11738; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11739; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11740; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11741; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11742; GFX11-WGP-NEXT:    buffer_gl0_inv
11743; GFX11-WGP-NEXT:    buffer_gl1_inv
11744; GFX11-WGP-NEXT:    s_endpgm
11745;
11746; GFX11-CU-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
11747; GFX11-CU:       ; %bb.0: ; %entry
11748; GFX11-CU-NEXT:    s_clause 0x1
11749; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11750; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11751; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11752; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
11753; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
11754; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
11755; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
11756; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11757; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11758; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11759; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11760; GFX11-CU-NEXT:    buffer_gl0_inv
11761; GFX11-CU-NEXT:    buffer_gl1_inv
11762; GFX11-CU-NEXT:    s_endpgm
11763    i32* %out, i32 %in, i32 %old) {
11764entry:
11765  %gep = getelementptr i32, i32* %out, i32 4
11766  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire seq_cst
11767  ret void
11768}
11769
11770define amdgpu_kernel void @flat_agent_one_as_release_seq_cst_cmpxchg(
11771; GFX7-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
11772; GFX7:       ; %bb.0: ; %entry
11773; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11774; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11775; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11776; GFX7-NEXT:    s_add_u32 s0, s0, 16
11777; GFX7-NEXT:    s_addc_u32 s1, s1, 0
11778; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11779; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11780; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11781; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11782; GFX7-NEXT:    s_waitcnt vmcnt(0)
11783; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11784; GFX7-NEXT:    s_waitcnt vmcnt(0)
11785; GFX7-NEXT:    buffer_wbinvl1_vol
11786; GFX7-NEXT:    s_endpgm
11787;
11788; GFX10-WGP-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
11789; GFX10-WGP:       ; %bb.0: ; %entry
11790; GFX10-WGP-NEXT:    s_clause 0x1
11791; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11792; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11793; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11794; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
11795; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
11796; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11797; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11798; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11799; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11800; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11801; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11802; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11803; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11804; GFX10-WGP-NEXT:    buffer_gl0_inv
11805; GFX10-WGP-NEXT:    buffer_gl1_inv
11806; GFX10-WGP-NEXT:    s_endpgm
11807;
11808; GFX10-CU-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
11809; GFX10-CU:       ; %bb.0: ; %entry
11810; GFX10-CU-NEXT:    s_clause 0x1
11811; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11812; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11813; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11814; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
11815; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
11816; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11817; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11818; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11819; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11820; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11821; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11822; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11823; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11824; GFX10-CU-NEXT:    buffer_gl0_inv
11825; GFX10-CU-NEXT:    buffer_gl1_inv
11826; GFX10-CU-NEXT:    s_endpgm
11827;
11828; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
11829; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11830; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11831; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11832; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11833; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
11834; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
11835; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11836; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11837; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11838; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11839; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11840; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11841; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11842; SKIP-CACHE-INV-NEXT:    s_endpgm
11843;
11844; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
11845; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11846; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11847; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11848; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11849; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11850; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11851; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11852; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11853; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11854; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
11855; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11856;
11857; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
11858; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11859; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11860; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11861; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11862; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11863; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11864; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11865; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11866; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11867; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11868; GFX90A-TGSPLIT-NEXT:    s_endpgm
11869;
11870; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
11871; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11872; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11873; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11874; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11875; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11876; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11877; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
11878; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11879; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11880; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11881; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
11882; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11883;
11884; GFX940-TGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
11885; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11886; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11887; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11888; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11889; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11890; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11891; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
11892; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11893; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
11894; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11895; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
11896; GFX940-TGSPLIT-NEXT:    s_endpgm
11897;
11898; GFX11-WGP-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
11899; GFX11-WGP:       ; %bb.0: ; %entry
11900; GFX11-WGP-NEXT:    s_clause 0x1
11901; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11902; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11903; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11904; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
11905; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
11906; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
11907; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
11908; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
11909; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11910; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11911; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11912; GFX11-WGP-NEXT:    buffer_gl0_inv
11913; GFX11-WGP-NEXT:    buffer_gl1_inv
11914; GFX11-WGP-NEXT:    s_endpgm
11915;
11916; GFX11-CU-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
11917; GFX11-CU:       ; %bb.0: ; %entry
11918; GFX11-CU-NEXT:    s_clause 0x1
11919; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
11920; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
11921; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
11922; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
11923; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
11924; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
11925; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
11926; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
11927; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11928; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
11929; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11930; GFX11-CU-NEXT:    buffer_gl0_inv
11931; GFX11-CU-NEXT:    buffer_gl1_inv
11932; GFX11-CU-NEXT:    s_endpgm
11933    i32* %out, i32 %in, i32 %old) {
11934entry:
11935  %gep = getelementptr i32, i32* %out, i32 4
11936  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release seq_cst
11937  ret void
11938}
11939
11940define amdgpu_kernel void @flat_agent_one_as_acq_rel_seq_cst_cmpxchg(
11941; GFX7-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
11942; GFX7:       ; %bb.0: ; %entry
11943; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11944; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11945; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11946; GFX7-NEXT:    s_add_u32 s0, s0, 16
11947; GFX7-NEXT:    s_addc_u32 s1, s1, 0
11948; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11949; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11950; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11951; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11952; GFX7-NEXT:    s_waitcnt vmcnt(0)
11953; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11954; GFX7-NEXT:    s_waitcnt vmcnt(0)
11955; GFX7-NEXT:    buffer_wbinvl1_vol
11956; GFX7-NEXT:    s_endpgm
11957;
11958; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
11959; GFX10-WGP:       ; %bb.0: ; %entry
11960; GFX10-WGP-NEXT:    s_clause 0x1
11961; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11962; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11963; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11964; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
11965; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
11966; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11967; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11968; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11969; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11970; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11971; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11972; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11973; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11974; GFX10-WGP-NEXT:    buffer_gl0_inv
11975; GFX10-WGP-NEXT:    buffer_gl1_inv
11976; GFX10-WGP-NEXT:    s_endpgm
11977;
11978; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
11979; GFX10-CU:       ; %bb.0: ; %entry
11980; GFX10-CU-NEXT:    s_clause 0x1
11981; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11982; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11983; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11984; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
11985; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
11986; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11987; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11988; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11989; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11990; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11991; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11992; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
11993; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11994; GFX10-CU-NEXT:    buffer_gl0_inv
11995; GFX10-CU-NEXT:    buffer_gl1_inv
11996; GFX10-CU-NEXT:    s_endpgm
11997;
11998; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
11999; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12000; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12001; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12002; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12003; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
12004; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
12005; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12006; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12007; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12008; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12009; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12010; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
12011; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12012; SKIP-CACHE-INV-NEXT:    s_endpgm
12013;
12014; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
12015; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12016; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12017; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12018; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12019; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12020; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12021; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12022; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
12023; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12024; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
12025; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12026;
12027; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
12028; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12029; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12030; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12031; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12032; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12033; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12034; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12035; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
12036; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12037; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
12038; GFX90A-TGSPLIT-NEXT:    s_endpgm
12039;
12040; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
12041; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12042; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12043; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12044; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12045; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12046; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12047; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
12048; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12049; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
12050; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12051; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
12052; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12053;
12054; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
12055; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12056; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12057; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12058; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12059; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12060; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12061; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
12062; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12063; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
12064; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12065; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
12066; GFX940-TGSPLIT-NEXT:    s_endpgm
12067;
12068; GFX11-WGP-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
12069; GFX11-WGP:       ; %bb.0: ; %entry
12070; GFX11-WGP-NEXT:    s_clause 0x1
12071; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12072; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12073; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12074; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
12075; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
12076; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
12077; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
12078; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12079; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12080; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
12081; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12082; GFX11-WGP-NEXT:    buffer_gl0_inv
12083; GFX11-WGP-NEXT:    buffer_gl1_inv
12084; GFX11-WGP-NEXT:    s_endpgm
12085;
12086; GFX11-CU-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
12087; GFX11-CU:       ; %bb.0: ; %entry
12088; GFX11-CU-NEXT:    s_clause 0x1
12089; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12090; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12091; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12092; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
12093; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
12094; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
12095; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
12096; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
12097; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12098; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
12099; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12100; GFX11-CU-NEXT:    buffer_gl0_inv
12101; GFX11-CU-NEXT:    buffer_gl1_inv
12102; GFX11-CU-NEXT:    s_endpgm
12103    i32* %out, i32 %in, i32 %old) {
12104entry:
12105  %gep = getelementptr i32, i32* %out, i32 4
12106  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel seq_cst
12107  ret void
12108}
12109
12110define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_cmpxchg(
12111; GFX7-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
12112; GFX7:       ; %bb.0: ; %entry
12113; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12114; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12115; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12116; GFX7-NEXT:    s_add_u32 s0, s0, 16
12117; GFX7-NEXT:    s_addc_u32 s1, s1, 0
12118; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12119; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12120; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12121; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12122; GFX7-NEXT:    s_waitcnt vmcnt(0)
12123; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
12124; GFX7-NEXT:    s_waitcnt vmcnt(0)
12125; GFX7-NEXT:    buffer_wbinvl1_vol
12126; GFX7-NEXT:    s_endpgm
12127;
12128; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
12129; GFX10-WGP:       ; %bb.0: ; %entry
12130; GFX10-WGP-NEXT:    s_clause 0x1
12131; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12132; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12133; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12134; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
12135; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
12136; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12137; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12138; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12139; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12140; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12141; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12142; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
12143; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12144; GFX10-WGP-NEXT:    buffer_gl0_inv
12145; GFX10-WGP-NEXT:    buffer_gl1_inv
12146; GFX10-WGP-NEXT:    s_endpgm
12147;
12148; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
12149; GFX10-CU:       ; %bb.0: ; %entry
12150; GFX10-CU-NEXT:    s_clause 0x1
12151; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12152; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12153; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12154; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
12155; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
12156; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12157; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12158; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12159; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12160; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12161; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12162; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
12163; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12164; GFX10-CU-NEXT:    buffer_gl0_inv
12165; GFX10-CU-NEXT:    buffer_gl1_inv
12166; GFX10-CU-NEXT:    s_endpgm
12167;
12168; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
12169; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12170; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12171; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12172; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12173; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
12174; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
12175; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12176; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12177; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12178; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12179; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12180; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
12181; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12182; SKIP-CACHE-INV-NEXT:    s_endpgm
12183;
12184; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
12185; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12186; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12187; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12188; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12189; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12190; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12191; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12192; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
12193; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12194; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
12195; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12196;
12197; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
12198; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12199; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12200; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12201; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12202; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12203; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12204; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12205; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
12206; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12207; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
12208; GFX90A-TGSPLIT-NEXT:    s_endpgm
12209;
12210; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
12211; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12212; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12213; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12214; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12215; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12216; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12217; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
12218; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12219; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
12220; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12221; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
12222; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12223;
12224; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
12225; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12226; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12227; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12228; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12229; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12230; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12231; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
12232; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12233; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
12234; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12235; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
12236; GFX940-TGSPLIT-NEXT:    s_endpgm
12237;
12238; GFX11-WGP-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
12239; GFX11-WGP:       ; %bb.0: ; %entry
12240; GFX11-WGP-NEXT:    s_clause 0x1
12241; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12242; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12243; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12244; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
12245; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
12246; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
12247; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
12248; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12249; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12250; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
12251; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12252; GFX11-WGP-NEXT:    buffer_gl0_inv
12253; GFX11-WGP-NEXT:    buffer_gl1_inv
12254; GFX11-WGP-NEXT:    s_endpgm
12255;
12256; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
12257; GFX11-CU:       ; %bb.0: ; %entry
12258; GFX11-CU-NEXT:    s_clause 0x1
12259; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12260; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12261; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12262; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
12263; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
12264; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
12265; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
12266; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
12267; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12268; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
12269; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12270; GFX11-CU-NEXT:    buffer_gl0_inv
12271; GFX11-CU-NEXT:    buffer_gl1_inv
12272; GFX11-CU-NEXT:    s_endpgm
12273    i32* %out, i32 %in, i32 %old) {
12274entry:
12275  %gep = getelementptr i32, i32* %out, i32 4
12276  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst seq_cst
12277  ret void
12278}
12279
12280define amdgpu_kernel void @flat_agent_one_as_monotonic_monotonic_ret_cmpxchg(
12281; GFX7-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
12282; GFX7:       ; %bb.0: ; %entry
12283; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12284; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12285; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12286; GFX7-NEXT:    s_add_u32 s4, s0, 16
12287; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12288; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12289; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12290; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12291; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12292; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12293; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12294; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12295; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12296; GFX7-NEXT:    flat_store_dword v[0:1], v2
12297; GFX7-NEXT:    s_endpgm
12298;
12299; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
12300; GFX10-WGP:       ; %bb.0: ; %entry
12301; GFX10-WGP-NEXT:    s_clause 0x1
12302; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12303; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12304; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12305; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12306; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12307; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12308; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12309; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12310; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12311; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12312; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12313; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12314; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12315; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12316; GFX10-WGP-NEXT:    s_endpgm
12317;
12318; GFX10-CU-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
12319; GFX10-CU:       ; %bb.0: ; %entry
12320; GFX10-CU-NEXT:    s_clause 0x1
12321; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12322; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12323; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12324; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12325; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12326; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12327; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12328; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12329; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12330; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12331; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12332; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12333; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12334; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12335; GFX10-CU-NEXT:    s_endpgm
12336;
12337; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
12338; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12339; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12340; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12341; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12342; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12343; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12344; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12345; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12346; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12347; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12348; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12349; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12350; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12351; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12352; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12353; SKIP-CACHE-INV-NEXT:    s_endpgm
12354;
12355; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
12356; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12357; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12358; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12359; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12360; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12361; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12362; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12363; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12364; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12365; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12366;
12367; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
12368; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12369; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12370; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12371; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12372; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12373; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12374; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12375; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12376; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12377; GFX90A-TGSPLIT-NEXT:    s_endpgm
12378;
12379; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
12380; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12381; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12382; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12383; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12384; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12385; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12386; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12387; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12388; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12389; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12390;
12391; GFX940-TGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
12392; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12393; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12394; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12395; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12396; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12397; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12398; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12399; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12400; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12401; GFX940-TGSPLIT-NEXT:    s_endpgm
12402;
12403; GFX11-WGP-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
12404; GFX11-WGP:       ; %bb.0: ; %entry
12405; GFX11-WGP-NEXT:    s_clause 0x1
12406; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12407; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12408; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12409; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
12410; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
12411; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
12412; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
12413; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12414; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12415; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12416; GFX11-WGP-NEXT:    s_endpgm
12417;
12418; GFX11-CU-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
12419; GFX11-CU:       ; %bb.0: ; %entry
12420; GFX11-CU-NEXT:    s_clause 0x1
12421; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12422; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12423; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12424; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
12425; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
12426; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
12427; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
12428; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12429; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12430; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12431; GFX11-CU-NEXT:    s_endpgm
12432    i32* %out, i32 %in, i32 %old) {
12433entry:
12434  %gep = getelementptr i32, i32* %out, i32 4
12435  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") monotonic monotonic
12436  %val0 = extractvalue { i32, i1 } %val, 0
12437  store i32 %val0, i32* %out, align 4
12438  ret void
12439}
12440
12441define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_ret_cmpxchg(
12442; GFX7-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
12443; GFX7:       ; %bb.0: ; %entry
12444; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12445; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12446; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12447; GFX7-NEXT:    s_add_u32 s4, s0, 16
12448; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12449; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12450; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12451; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12452; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12453; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12454; GFX7-NEXT:    s_waitcnt vmcnt(0)
12455; GFX7-NEXT:    buffer_wbinvl1_vol
12456; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12457; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12458; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12459; GFX7-NEXT:    flat_store_dword v[0:1], v2
12460; GFX7-NEXT:    s_endpgm
12461;
12462; GFX10-WGP-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
12463; GFX10-WGP:       ; %bb.0: ; %entry
12464; GFX10-WGP-NEXT:    s_clause 0x1
12465; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12466; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12467; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12468; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12469; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12470; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12471; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12472; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12473; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12474; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12475; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12476; GFX10-WGP-NEXT:    buffer_gl0_inv
12477; GFX10-WGP-NEXT:    buffer_gl1_inv
12478; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12479; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12480; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12481; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12482; GFX10-WGP-NEXT:    s_endpgm
12483;
12484; GFX10-CU-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
12485; GFX10-CU:       ; %bb.0: ; %entry
12486; GFX10-CU-NEXT:    s_clause 0x1
12487; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12488; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12489; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12490; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12491; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12492; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12493; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12494; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12495; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12496; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12497; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12498; GFX10-CU-NEXT:    buffer_gl0_inv
12499; GFX10-CU-NEXT:    buffer_gl1_inv
12500; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12501; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12502; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12503; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12504; GFX10-CU-NEXT:    s_endpgm
12505;
12506; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
12507; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12508; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12509; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12510; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12511; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12512; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12513; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12514; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12515; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12516; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12517; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12518; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12519; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12520; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12521; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12522; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12523; SKIP-CACHE-INV-NEXT:    s_endpgm
12524;
12525; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
12526; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12527; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12528; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12529; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12530; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12531; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12532; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12533; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12534; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
12535; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12536; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12537; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12538;
12539; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
12540; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12541; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12542; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12543; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12544; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12545; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12546; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12547; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12548; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
12549; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12550; GFX90A-TGSPLIT-NEXT:    s_endpgm
12551;
12552; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
12553; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12554; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12555; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12556; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12557; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12558; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12559; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12560; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12561; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
12562; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12563; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12564; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12565;
12566; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
12567; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12568; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12569; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12570; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12571; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12572; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12573; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12574; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12575; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
12576; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12577; GFX940-TGSPLIT-NEXT:    s_endpgm
12578;
12579; GFX11-WGP-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
12580; GFX11-WGP:       ; %bb.0: ; %entry
12581; GFX11-WGP-NEXT:    s_clause 0x1
12582; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12583; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12584; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12585; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
12586; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
12587; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
12588; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
12589; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12590; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12591; GFX11-WGP-NEXT:    buffer_gl0_inv
12592; GFX11-WGP-NEXT:    buffer_gl1_inv
12593; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12594; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12595; GFX11-WGP-NEXT:    s_endpgm
12596;
12597; GFX11-CU-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
12598; GFX11-CU:       ; %bb.0: ; %entry
12599; GFX11-CU-NEXT:    s_clause 0x1
12600; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12601; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12602; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12603; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
12604; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
12605; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
12606; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
12607; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12608; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
12609; GFX11-CU-NEXT:    buffer_gl0_inv
12610; GFX11-CU-NEXT:    buffer_gl1_inv
12611; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12612; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12613; GFX11-CU-NEXT:    s_endpgm
12614    i32* %out, i32 %in, i32 %old) {
12615entry:
12616  %gep = getelementptr i32, i32* %out, i32 4
12617  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire monotonic
12618  %val0 = extractvalue { i32, i1 } %val, 0
12619  store i32 %val0, i32* %out, align 4
12620  ret void
12621}
12622
12623define amdgpu_kernel void @flat_agent_one_as_release_monotonic_ret_cmpxchg(
12624; GFX7-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
12625; GFX7:       ; %bb.0: ; %entry
12626; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12627; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12628; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12629; GFX7-NEXT:    s_add_u32 s4, s0, 16
12630; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12631; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12632; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12633; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12634; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12635; GFX7-NEXT:    s_waitcnt vmcnt(0)
12636; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12637; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12638; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12639; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12640; GFX7-NEXT:    flat_store_dword v[0:1], v2
12641; GFX7-NEXT:    s_endpgm
12642;
12643; GFX10-WGP-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
12644; GFX10-WGP:       ; %bb.0: ; %entry
12645; GFX10-WGP-NEXT:    s_clause 0x1
12646; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12647; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12648; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12649; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12650; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12651; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12652; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12653; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12654; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12655; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12656; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12657; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12658; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12659; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12660; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12661; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12662; GFX10-WGP-NEXT:    s_endpgm
12663;
12664; GFX10-CU-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
12665; GFX10-CU:       ; %bb.0: ; %entry
12666; GFX10-CU-NEXT:    s_clause 0x1
12667; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12668; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12669; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12670; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12671; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12672; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12673; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12674; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12675; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12676; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12677; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12678; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12679; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12680; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12681; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12682; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12683; GFX10-CU-NEXT:    s_endpgm
12684;
12685; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
12686; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12687; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12688; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12689; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12690; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12691; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12692; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12693; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12694; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12695; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12696; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12697; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12698; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12699; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12700; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12701; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12702; SKIP-CACHE-INV-NEXT:    s_endpgm
12703;
12704; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
12705; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12706; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12707; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12708; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12709; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12710; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12711; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12712; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12713; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12714; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12715; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12716;
12717; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
12718; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12719; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12720; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12721; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12722; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12723; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12724; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12725; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12726; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12727; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12728; GFX90A-TGSPLIT-NEXT:    s_endpgm
12729;
12730; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
12731; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12732; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12733; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12734; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12735; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12736; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12737; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
12738; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12739; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12740; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12741; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12742; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12743;
12744; GFX940-TGSPLIT-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
12745; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12746; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12747; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12748; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12749; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12750; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12751; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
12752; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12753; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12754; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12755; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12756; GFX940-TGSPLIT-NEXT:    s_endpgm
12757;
12758; GFX11-WGP-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
12759; GFX11-WGP:       ; %bb.0: ; %entry
12760; GFX11-WGP-NEXT:    s_clause 0x1
12761; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12762; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12763; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12764; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
12765; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
12766; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
12767; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
12768; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12769; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12770; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12771; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12772; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12773; GFX11-WGP-NEXT:    s_endpgm
12774;
12775; GFX11-CU-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
12776; GFX11-CU:       ; %bb.0: ; %entry
12777; GFX11-CU-NEXT:    s_clause 0x1
12778; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12779; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12780; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12781; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
12782; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
12783; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
12784; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
12785; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
12786; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12787; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12788; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
12789; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12790; GFX11-CU-NEXT:    s_endpgm
12791    i32* %out, i32 %in, i32 %old) {
12792entry:
12793  %gep = getelementptr i32, i32* %out, i32 4
12794  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release monotonic
12795  %val0 = extractvalue { i32, i1 } %val, 0
12796  store i32 %val0, i32* %out, align 4
12797  ret void
12798}
12799
12800define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg(
12801; GFX7-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
12802; GFX7:       ; %bb.0: ; %entry
12803; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12804; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12805; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12806; GFX7-NEXT:    s_add_u32 s4, s0, 16
12807; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12808; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12809; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12810; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12811; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12812; GFX7-NEXT:    s_waitcnt vmcnt(0)
12813; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12814; GFX7-NEXT:    s_waitcnt vmcnt(0)
12815; GFX7-NEXT:    buffer_wbinvl1_vol
12816; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12817; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12818; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12819; GFX7-NEXT:    flat_store_dword v[0:1], v2
12820; GFX7-NEXT:    s_endpgm
12821;
12822; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
12823; GFX10-WGP:       ; %bb.0: ; %entry
12824; GFX10-WGP-NEXT:    s_clause 0x1
12825; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12826; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12827; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12828; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12829; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12830; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12831; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12832; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12833; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12834; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12835; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12836; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12837; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12838; GFX10-WGP-NEXT:    buffer_gl0_inv
12839; GFX10-WGP-NEXT:    buffer_gl1_inv
12840; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12841; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12842; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12843; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12844; GFX10-WGP-NEXT:    s_endpgm
12845;
12846; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
12847; GFX10-CU:       ; %bb.0: ; %entry
12848; GFX10-CU-NEXT:    s_clause 0x1
12849; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12850; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12851; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12852; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12853; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12854; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12855; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12856; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12857; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12858; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12859; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12860; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12861; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12862; GFX10-CU-NEXT:    buffer_gl0_inv
12863; GFX10-CU-NEXT:    buffer_gl1_inv
12864; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12865; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12866; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12867; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12868; GFX10-CU-NEXT:    s_endpgm
12869;
12870; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
12871; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12872; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12873; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12874; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12875; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12876; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12877; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12878; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12879; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12880; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12881; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12882; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12883; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12884; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12885; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12886; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12887; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12888; SKIP-CACHE-INV-NEXT:    s_endpgm
12889;
12890; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
12891; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12892; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12893; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12894; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12895; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12896; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12897; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12898; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12899; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12900; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
12901; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12902; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12903; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12904;
12905; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
12906; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12907; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12908; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12909; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12910; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12911; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12912; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12913; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12914; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12915; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
12916; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12917; GFX90A-TGSPLIT-NEXT:    s_endpgm
12918;
12919; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
12920; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12921; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12922; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12923; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12924; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12925; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12926; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
12927; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12928; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12929; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12930; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
12931; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12932; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12933; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12934;
12935; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
12936; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12937; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12938; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12939; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12940; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12941; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12942; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
12943; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12944; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
12945; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12946; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
12947; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12948; GFX940-TGSPLIT-NEXT:    s_endpgm
12949;
12950; GFX11-WGP-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
12951; GFX11-WGP:       ; %bb.0: ; %entry
12952; GFX11-WGP-NEXT:    s_clause 0x1
12953; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12954; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12955; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12956; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
12957; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
12958; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
12959; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
12960; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12961; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12962; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12963; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
12964; GFX11-WGP-NEXT:    buffer_gl0_inv
12965; GFX11-WGP-NEXT:    buffer_gl1_inv
12966; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12967; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
12968; GFX11-WGP-NEXT:    s_endpgm
12969;
12970; GFX11-CU-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
12971; GFX11-CU:       ; %bb.0: ; %entry
12972; GFX11-CU-NEXT:    s_clause 0x1
12973; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
12974; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
12975; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12976; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
12977; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
12978; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
12979; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
12980; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
12981; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12982; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
12983; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
12984; GFX11-CU-NEXT:    buffer_gl0_inv
12985; GFX11-CU-NEXT:    buffer_gl1_inv
12986; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
12987; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
12988; GFX11-CU-NEXT:    s_endpgm
12989    i32* %out, i32 %in, i32 %old) {
12990entry:
12991  %gep = getelementptr i32, i32* %out, i32 4
12992  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel monotonic
12993  %val0 = extractvalue { i32, i1 } %val, 0
12994  store i32 %val0, i32* %out, align 4
12995  ret void
12996}
12997
12998define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg(
12999; GFX7-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
13000; GFX7:       ; %bb.0: ; %entry
13001; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13002; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
13003; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13004; GFX7-NEXT:    s_add_u32 s4, s0, 16
13005; GFX7-NEXT:    s_addc_u32 s5, s1, 0
13006; GFX7-NEXT:    v_mov_b32_e32 v0, s4
13007; GFX7-NEXT:    v_mov_b32_e32 v2, s2
13008; GFX7-NEXT:    v_mov_b32_e32 v1, s5
13009; GFX7-NEXT:    v_mov_b32_e32 v3, s3
13010; GFX7-NEXT:    s_waitcnt vmcnt(0)
13011; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13012; GFX7-NEXT:    s_waitcnt vmcnt(0)
13013; GFX7-NEXT:    buffer_wbinvl1_vol
13014; GFX7-NEXT:    v_mov_b32_e32 v0, s0
13015; GFX7-NEXT:    v_mov_b32_e32 v1, s1
13016; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13017; GFX7-NEXT:    flat_store_dword v[0:1], v2
13018; GFX7-NEXT:    s_endpgm
13019;
13020; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
13021; GFX10-WGP:       ; %bb.0: ; %entry
13022; GFX10-WGP-NEXT:    s_clause 0x1
13023; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13024; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13025; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13026; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
13027; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
13028; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
13029; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
13030; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
13031; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
13032; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13033; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
13034; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13035; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13036; GFX10-WGP-NEXT:    buffer_gl0_inv
13037; GFX10-WGP-NEXT:    buffer_gl1_inv
13038; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
13039; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
13040; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13041; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
13042; GFX10-WGP-NEXT:    s_endpgm
13043;
13044; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
13045; GFX10-CU:       ; %bb.0: ; %entry
13046; GFX10-CU-NEXT:    s_clause 0x1
13047; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13048; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13049; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13050; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
13051; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
13052; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
13053; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
13054; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
13055; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
13056; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13057; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
13058; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13059; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13060; GFX10-CU-NEXT:    buffer_gl0_inv
13061; GFX10-CU-NEXT:    buffer_gl1_inv
13062; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
13063; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
13064; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13065; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
13066; GFX10-CU-NEXT:    s_endpgm
13067;
13068; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
13069; SKIP-CACHE-INV:       ; %bb.0: ; %entry
13070; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13071; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
13072; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13073; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
13074; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
13075; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
13076; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
13077; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
13078; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
13079; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
13080; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13081; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
13082; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
13083; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
13084; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13085; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
13086; SKIP-CACHE-INV-NEXT:    s_endpgm
13087;
13088; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
13089; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
13090; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13091; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13092; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13093; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13094; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13095; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13096; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13097; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13098; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
13099; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13100; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13101; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
13102;
13103; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
13104; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
13105; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13106; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13107; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13108; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13109; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13110; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13111; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13112; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13113; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
13114; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13115; GFX90A-TGSPLIT-NEXT:    s_endpgm
13116;
13117; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
13118; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
13119; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13120; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13121; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13122; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13123; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13124; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
13125; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13126; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
13127; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13128; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
13129; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13130; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13131; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
13132;
13133; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
13134; GFX940-TGSPLIT:       ; %bb.0: ; %entry
13135; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13136; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13137; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13138; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13139; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13140; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
13141; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13142; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
13143; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13144; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
13145; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13146; GFX940-TGSPLIT-NEXT:    s_endpgm
13147;
13148; GFX11-WGP-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
13149; GFX11-WGP:       ; %bb.0: ; %entry
13150; GFX11-WGP-NEXT:    s_clause 0x1
13151; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13152; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13153; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13154; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
13155; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
13156; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
13157; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
13158; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13159; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
13160; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13161; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13162; GFX11-WGP-NEXT:    buffer_gl0_inv
13163; GFX11-WGP-NEXT:    buffer_gl1_inv
13164; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13165; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
13166; GFX11-WGP-NEXT:    s_endpgm
13167;
13168; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
13169; GFX11-CU:       ; %bb.0: ; %entry
13170; GFX11-CU-NEXT:    s_clause 0x1
13171; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13172; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13173; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13174; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
13175; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
13176; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
13177; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
13178; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13179; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
13180; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13181; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13182; GFX11-CU-NEXT:    buffer_gl0_inv
13183; GFX11-CU-NEXT:    buffer_gl1_inv
13184; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13185; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
13186; GFX11-CU-NEXT:    s_endpgm
13187    i32* %out, i32 %in, i32 %old) {
13188entry:
13189  %gep = getelementptr i32, i32* %out, i32 4
13190  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst monotonic
13191  %val0 = extractvalue { i32, i1 } %val, 0
13192  store i32 %val0, i32* %out, align 4
13193  ret void
13194}
13195
13196define amdgpu_kernel void @flat_agent_one_as_monotonic_acquire_ret_cmpxchg(
13197; GFX7-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
13198; GFX7:       ; %bb.0: ; %entry
13199; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13200; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
13201; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13202; GFX7-NEXT:    s_add_u32 s4, s0, 16
13203; GFX7-NEXT:    s_addc_u32 s5, s1, 0
13204; GFX7-NEXT:    v_mov_b32_e32 v0, s4
13205; GFX7-NEXT:    v_mov_b32_e32 v2, s2
13206; GFX7-NEXT:    v_mov_b32_e32 v1, s5
13207; GFX7-NEXT:    v_mov_b32_e32 v3, s3
13208; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13209; GFX7-NEXT:    s_waitcnt vmcnt(0)
13210; GFX7-NEXT:    buffer_wbinvl1_vol
13211; GFX7-NEXT:    v_mov_b32_e32 v0, s0
13212; GFX7-NEXT:    v_mov_b32_e32 v1, s1
13213; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13214; GFX7-NEXT:    flat_store_dword v[0:1], v2
13215; GFX7-NEXT:    s_endpgm
13216;
13217; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
13218; GFX10-WGP:       ; %bb.0: ; %entry
13219; GFX10-WGP-NEXT:    s_clause 0x1
13220; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13221; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13222; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13223; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
13224; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
13225; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
13226; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
13227; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
13228; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
13229; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13230; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13231; GFX10-WGP-NEXT:    buffer_gl0_inv
13232; GFX10-WGP-NEXT:    buffer_gl1_inv
13233; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
13234; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
13235; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13236; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
13237; GFX10-WGP-NEXT:    s_endpgm
13238;
13239; GFX10-CU-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
13240; GFX10-CU:       ; %bb.0: ; %entry
13241; GFX10-CU-NEXT:    s_clause 0x1
13242; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13243; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13244; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13245; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
13246; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
13247; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
13248; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
13249; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
13250; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
13251; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13252; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13253; GFX10-CU-NEXT:    buffer_gl0_inv
13254; GFX10-CU-NEXT:    buffer_gl1_inv
13255; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
13256; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
13257; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13258; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
13259; GFX10-CU-NEXT:    s_endpgm
13260;
13261; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
13262; SKIP-CACHE-INV:       ; %bb.0: ; %entry
13263; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13264; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
13265; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13266; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
13267; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
13268; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
13269; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
13270; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
13271; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
13272; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13273; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
13274; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
13275; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
13276; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13277; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
13278; SKIP-CACHE-INV-NEXT:    s_endpgm
13279;
13280; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
13281; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
13282; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13283; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13284; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13285; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13286; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13287; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13288; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13289; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
13290; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13291; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13292; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
13293;
13294; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
13295; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
13296; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13297; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13298; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13299; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13300; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13301; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13302; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13303; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
13304; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13305; GFX90A-TGSPLIT-NEXT:    s_endpgm
13306;
13307; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
13308; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
13309; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13310; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13311; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13312; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13313; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13314; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
13315; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13316; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
13317; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13318; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13319; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
13320;
13321; GFX940-TGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
13322; GFX940-TGSPLIT:       ; %bb.0: ; %entry
13323; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13324; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13325; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13326; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13327; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13328; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
13329; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13330; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
13331; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13332; GFX940-TGSPLIT-NEXT:    s_endpgm
13333;
13334; GFX11-WGP-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
13335; GFX11-WGP:       ; %bb.0: ; %entry
13336; GFX11-WGP-NEXT:    s_clause 0x1
13337; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13338; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13339; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13340; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
13341; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
13342; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
13343; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
13344; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13345; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13346; GFX11-WGP-NEXT:    buffer_gl0_inv
13347; GFX11-WGP-NEXT:    buffer_gl1_inv
13348; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13349; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
13350; GFX11-WGP-NEXT:    s_endpgm
13351;
13352; GFX11-CU-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
13353; GFX11-CU:       ; %bb.0: ; %entry
13354; GFX11-CU-NEXT:    s_clause 0x1
13355; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13356; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13357; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13358; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
13359; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
13360; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
13361; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
13362; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13363; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13364; GFX11-CU-NEXT:    buffer_gl0_inv
13365; GFX11-CU-NEXT:    buffer_gl1_inv
13366; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13367; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
13368; GFX11-CU-NEXT:    s_endpgm
13369    i32* %out, i32 %in, i32 %old) {
13370entry:
13371  %gep = getelementptr i32, i32* %out, i32 4
13372  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") monotonic acquire
13373  %val0 = extractvalue { i32, i1 } %val, 0
13374  store i32 %val0, i32* %out, align 4
13375  ret void
13376}
13377
13378define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_ret_cmpxchg(
13379; GFX7-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
13380; GFX7:       ; %bb.0: ; %entry
13381; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13382; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
13383; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13384; GFX7-NEXT:    s_add_u32 s4, s0, 16
13385; GFX7-NEXT:    s_addc_u32 s5, s1, 0
13386; GFX7-NEXT:    v_mov_b32_e32 v0, s4
13387; GFX7-NEXT:    v_mov_b32_e32 v2, s2
13388; GFX7-NEXT:    v_mov_b32_e32 v1, s5
13389; GFX7-NEXT:    v_mov_b32_e32 v3, s3
13390; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13391; GFX7-NEXT:    s_waitcnt vmcnt(0)
13392; GFX7-NEXT:    buffer_wbinvl1_vol
13393; GFX7-NEXT:    v_mov_b32_e32 v0, s0
13394; GFX7-NEXT:    v_mov_b32_e32 v1, s1
13395; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13396; GFX7-NEXT:    flat_store_dword v[0:1], v2
13397; GFX7-NEXT:    s_endpgm
13398;
13399; GFX10-WGP-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
13400; GFX10-WGP:       ; %bb.0: ; %entry
13401; GFX10-WGP-NEXT:    s_clause 0x1
13402; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13403; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13404; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13405; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
13406; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
13407; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
13408; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
13409; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
13410; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
13411; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13412; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13413; GFX10-WGP-NEXT:    buffer_gl0_inv
13414; GFX10-WGP-NEXT:    buffer_gl1_inv
13415; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
13416; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
13417; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13418; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
13419; GFX10-WGP-NEXT:    s_endpgm
13420;
13421; GFX10-CU-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
13422; GFX10-CU:       ; %bb.0: ; %entry
13423; GFX10-CU-NEXT:    s_clause 0x1
13424; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13425; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13426; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13427; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
13428; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
13429; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
13430; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
13431; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
13432; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
13433; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13434; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13435; GFX10-CU-NEXT:    buffer_gl0_inv
13436; GFX10-CU-NEXT:    buffer_gl1_inv
13437; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
13438; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
13439; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13440; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
13441; GFX10-CU-NEXT:    s_endpgm
13442;
13443; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
13444; SKIP-CACHE-INV:       ; %bb.0: ; %entry
13445; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13446; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
13447; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13448; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
13449; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
13450; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
13451; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
13452; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
13453; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
13454; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13455; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
13456; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
13457; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
13458; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13459; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
13460; SKIP-CACHE-INV-NEXT:    s_endpgm
13461;
13462; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
13463; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
13464; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13465; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13466; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13467; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13468; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13469; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13470; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13471; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
13472; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13473; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13474; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
13475;
13476; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
13477; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
13478; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13479; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13480; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13481; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13482; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13483; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13484; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13485; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
13486; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13487; GFX90A-TGSPLIT-NEXT:    s_endpgm
13488;
13489; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
13490; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
13491; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13492; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13493; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13494; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13495; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13496; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
13497; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13498; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
13499; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13500; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13501; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
13502;
13503; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
13504; GFX940-TGSPLIT:       ; %bb.0: ; %entry
13505; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13506; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13507; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13508; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13509; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13510; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
13511; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13512; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
13513; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13514; GFX940-TGSPLIT-NEXT:    s_endpgm
13515;
13516; GFX11-WGP-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
13517; GFX11-WGP:       ; %bb.0: ; %entry
13518; GFX11-WGP-NEXT:    s_clause 0x1
13519; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13520; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13521; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13522; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
13523; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
13524; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
13525; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
13526; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13527; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13528; GFX11-WGP-NEXT:    buffer_gl0_inv
13529; GFX11-WGP-NEXT:    buffer_gl1_inv
13530; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13531; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
13532; GFX11-WGP-NEXT:    s_endpgm
13533;
13534; GFX11-CU-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
13535; GFX11-CU:       ; %bb.0: ; %entry
13536; GFX11-CU-NEXT:    s_clause 0x1
13537; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13538; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13539; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13540; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
13541; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
13542; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
13543; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
13544; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13545; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13546; GFX11-CU-NEXT:    buffer_gl0_inv
13547; GFX11-CU-NEXT:    buffer_gl1_inv
13548; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13549; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
13550; GFX11-CU-NEXT:    s_endpgm
13551    i32* %out, i32 %in, i32 %old) {
13552entry:
13553  %gep = getelementptr i32, i32* %out, i32 4
13554  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire acquire
13555  %val0 = extractvalue { i32, i1 } %val, 0
13556  store i32 %val0, i32* %out, align 4
13557  ret void
13558}
13559
13560define amdgpu_kernel void @flat_agent_one_as_release_acquire_ret_cmpxchg(
13561; GFX7-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
13562; GFX7:       ; %bb.0: ; %entry
13563; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13564; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
13565; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13566; GFX7-NEXT:    s_add_u32 s4, s0, 16
13567; GFX7-NEXT:    s_addc_u32 s5, s1, 0
13568; GFX7-NEXT:    v_mov_b32_e32 v0, s4
13569; GFX7-NEXT:    v_mov_b32_e32 v2, s2
13570; GFX7-NEXT:    v_mov_b32_e32 v1, s5
13571; GFX7-NEXT:    v_mov_b32_e32 v3, s3
13572; GFX7-NEXT:    s_waitcnt vmcnt(0)
13573; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13574; GFX7-NEXT:    s_waitcnt vmcnt(0)
13575; GFX7-NEXT:    buffer_wbinvl1_vol
13576; GFX7-NEXT:    v_mov_b32_e32 v0, s0
13577; GFX7-NEXT:    v_mov_b32_e32 v1, s1
13578; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13579; GFX7-NEXT:    flat_store_dword v[0:1], v2
13580; GFX7-NEXT:    s_endpgm
13581;
13582; GFX10-WGP-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
13583; GFX10-WGP:       ; %bb.0: ; %entry
13584; GFX10-WGP-NEXT:    s_clause 0x1
13585; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13586; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13587; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13588; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
13589; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
13590; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
13591; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
13592; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
13593; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
13594; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13595; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
13596; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13597; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13598; GFX10-WGP-NEXT:    buffer_gl0_inv
13599; GFX10-WGP-NEXT:    buffer_gl1_inv
13600; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
13601; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
13602; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13603; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
13604; GFX10-WGP-NEXT:    s_endpgm
13605;
13606; GFX10-CU-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
13607; GFX10-CU:       ; %bb.0: ; %entry
13608; GFX10-CU-NEXT:    s_clause 0x1
13609; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13610; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13611; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13612; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
13613; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
13614; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
13615; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
13616; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
13617; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
13618; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13619; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
13620; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13621; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13622; GFX10-CU-NEXT:    buffer_gl0_inv
13623; GFX10-CU-NEXT:    buffer_gl1_inv
13624; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
13625; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
13626; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13627; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
13628; GFX10-CU-NEXT:    s_endpgm
13629;
13630; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
13631; SKIP-CACHE-INV:       ; %bb.0: ; %entry
13632; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13633; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
13634; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13635; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
13636; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
13637; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
13638; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
13639; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
13640; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
13641; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
13642; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13643; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
13644; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
13645; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
13646; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13647; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
13648; SKIP-CACHE-INV-NEXT:    s_endpgm
13649;
13650; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
13651; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
13652; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13653; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13654; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13655; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13656; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13657; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13658; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13659; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13660; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
13661; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13662; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13663; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
13664;
13665; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
13666; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
13667; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13668; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13669; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13670; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13671; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13672; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13673; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13674; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13675; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
13676; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13677; GFX90A-TGSPLIT-NEXT:    s_endpgm
13678;
13679; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
13680; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
13681; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13682; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13683; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13684; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13685; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13686; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
13687; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13688; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
13689; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13690; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
13691; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13692; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13693; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
13694;
13695; GFX940-TGSPLIT-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
13696; GFX940-TGSPLIT:       ; %bb.0: ; %entry
13697; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13698; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13699; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13700; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13701; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13702; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
13703; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13704; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
13705; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13706; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
13707; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13708; GFX940-TGSPLIT-NEXT:    s_endpgm
13709;
13710; GFX11-WGP-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
13711; GFX11-WGP:       ; %bb.0: ; %entry
13712; GFX11-WGP-NEXT:    s_clause 0x1
13713; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13714; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13715; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13716; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
13717; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
13718; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
13719; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
13720; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13721; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
13722; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13723; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13724; GFX11-WGP-NEXT:    buffer_gl0_inv
13725; GFX11-WGP-NEXT:    buffer_gl1_inv
13726; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13727; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
13728; GFX11-WGP-NEXT:    s_endpgm
13729;
13730; GFX11-CU-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
13731; GFX11-CU:       ; %bb.0: ; %entry
13732; GFX11-CU-NEXT:    s_clause 0x1
13733; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13734; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13735; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13736; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
13737; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
13738; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
13739; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
13740; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13741; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
13742; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13743; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13744; GFX11-CU-NEXT:    buffer_gl0_inv
13745; GFX11-CU-NEXT:    buffer_gl1_inv
13746; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13747; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
13748; GFX11-CU-NEXT:    s_endpgm
13749    i32* %out, i32 %in, i32 %old) {
13750entry:
13751  %gep = getelementptr i32, i32* %out, i32 4
13752  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release acquire
13753  %val0 = extractvalue { i32, i1 } %val, 0
13754  store i32 %val0, i32* %out, align 4
13755  ret void
13756}
13757
13758define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_ret_cmpxchg(
13759; GFX7-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
13760; GFX7:       ; %bb.0: ; %entry
13761; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13762; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
13763; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13764; GFX7-NEXT:    s_add_u32 s4, s0, 16
13765; GFX7-NEXT:    s_addc_u32 s5, s1, 0
13766; GFX7-NEXT:    v_mov_b32_e32 v0, s4
13767; GFX7-NEXT:    v_mov_b32_e32 v2, s2
13768; GFX7-NEXT:    v_mov_b32_e32 v1, s5
13769; GFX7-NEXT:    v_mov_b32_e32 v3, s3
13770; GFX7-NEXT:    s_waitcnt vmcnt(0)
13771; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13772; GFX7-NEXT:    s_waitcnt vmcnt(0)
13773; GFX7-NEXT:    buffer_wbinvl1_vol
13774; GFX7-NEXT:    v_mov_b32_e32 v0, s0
13775; GFX7-NEXT:    v_mov_b32_e32 v1, s1
13776; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13777; GFX7-NEXT:    flat_store_dword v[0:1], v2
13778; GFX7-NEXT:    s_endpgm
13779;
13780; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
13781; GFX10-WGP:       ; %bb.0: ; %entry
13782; GFX10-WGP-NEXT:    s_clause 0x1
13783; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13784; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13785; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13786; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
13787; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
13788; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
13789; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
13790; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
13791; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
13792; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13793; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
13794; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13795; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13796; GFX10-WGP-NEXT:    buffer_gl0_inv
13797; GFX10-WGP-NEXT:    buffer_gl1_inv
13798; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
13799; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
13800; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13801; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
13802; GFX10-WGP-NEXT:    s_endpgm
13803;
13804; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
13805; GFX10-CU:       ; %bb.0: ; %entry
13806; GFX10-CU-NEXT:    s_clause 0x1
13807; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13808; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13809; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13810; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
13811; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
13812; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
13813; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
13814; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
13815; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
13816; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13817; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
13818; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13819; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
13820; GFX10-CU-NEXT:    buffer_gl0_inv
13821; GFX10-CU-NEXT:    buffer_gl1_inv
13822; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
13823; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
13824; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
13825; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
13826; GFX10-CU-NEXT:    s_endpgm
13827;
13828; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
13829; SKIP-CACHE-INV:       ; %bb.0: ; %entry
13830; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13831; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
13832; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13833; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
13834; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
13835; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
13836; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
13837; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
13838; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
13839; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
13840; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13841; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
13842; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
13843; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
13844; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
13845; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
13846; SKIP-CACHE-INV-NEXT:    s_endpgm
13847;
13848; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
13849; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
13850; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13851; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13852; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13853; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13854; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13855; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13856; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13857; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13858; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
13859; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13860; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13861; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
13862;
13863; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
13864; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
13865; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13866; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13867; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13868; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
13869; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
13870; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13871; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
13872; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13873; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
13874; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13875; GFX90A-TGSPLIT-NEXT:    s_endpgm
13876;
13877; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
13878; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
13879; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13880; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13881; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13882; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13883; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13884; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
13885; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13886; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
13887; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13888; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
13889; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13890; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13891; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
13892;
13893; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
13894; GFX940-TGSPLIT:       ; %bb.0: ; %entry
13895; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
13896; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
13897; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
13898; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
13899; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
13900; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
13901; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13902; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
13903; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
13904; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
13905; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
13906; GFX940-TGSPLIT-NEXT:    s_endpgm
13907;
13908; GFX11-WGP-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
13909; GFX11-WGP:       ; %bb.0: ; %entry
13910; GFX11-WGP-NEXT:    s_clause 0x1
13911; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13912; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13913; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13914; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
13915; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
13916; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
13917; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
13918; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13919; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
13920; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13921; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
13922; GFX11-WGP-NEXT:    buffer_gl0_inv
13923; GFX11-WGP-NEXT:    buffer_gl1_inv
13924; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13925; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
13926; GFX11-WGP-NEXT:    s_endpgm
13927;
13928; GFX11-CU-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
13929; GFX11-CU:       ; %bb.0: ; %entry
13930; GFX11-CU-NEXT:    s_clause 0x1
13931; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
13932; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
13933; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13934; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
13935; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
13936; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
13937; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
13938; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13939; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
13940; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
13941; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
13942; GFX11-CU-NEXT:    buffer_gl0_inv
13943; GFX11-CU-NEXT:    buffer_gl1_inv
13944; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
13945; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
13946; GFX11-CU-NEXT:    s_endpgm
13947    i32* %out, i32 %in, i32 %old) {
13948entry:
13949  %gep = getelementptr i32, i32* %out, i32 4
13950  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel acquire
13951  %val0 = extractvalue { i32, i1 } %val, 0
13952  store i32 %val0, i32* %out, align 4
13953  ret void
13954}
13955
13956define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_ret_cmpxchg(
13957; GFX7-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
13958; GFX7:       ; %bb.0: ; %entry
13959; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13960; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
13961; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13962; GFX7-NEXT:    s_add_u32 s4, s0, 16
13963; GFX7-NEXT:    s_addc_u32 s5, s1, 0
13964; GFX7-NEXT:    v_mov_b32_e32 v0, s4
13965; GFX7-NEXT:    v_mov_b32_e32 v2, s2
13966; GFX7-NEXT:    v_mov_b32_e32 v1, s5
13967; GFX7-NEXT:    v_mov_b32_e32 v3, s3
13968; GFX7-NEXT:    s_waitcnt vmcnt(0)
13969; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13970; GFX7-NEXT:    s_waitcnt vmcnt(0)
13971; GFX7-NEXT:    buffer_wbinvl1_vol
13972; GFX7-NEXT:    v_mov_b32_e32 v0, s0
13973; GFX7-NEXT:    v_mov_b32_e32 v1, s1
13974; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
13975; GFX7-NEXT:    flat_store_dword v[0:1], v2
13976; GFX7-NEXT:    s_endpgm
13977;
13978; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
13979; GFX10-WGP:       ; %bb.0: ; %entry
13980; GFX10-WGP-NEXT:    s_clause 0x1
13981; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
13982; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
13983; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13984; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
13985; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
13986; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
13987; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
13988; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
13989; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
13990; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13991; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
13992; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
13993; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
13994; GFX10-WGP-NEXT:    buffer_gl0_inv
13995; GFX10-WGP-NEXT:    buffer_gl1_inv
13996; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
13997; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
13998; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
13999; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
14000; GFX10-WGP-NEXT:    s_endpgm
14001;
14002; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
14003; GFX10-CU:       ; %bb.0: ; %entry
14004; GFX10-CU-NEXT:    s_clause 0x1
14005; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14006; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14007; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14008; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
14009; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
14010; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
14011; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
14012; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
14013; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
14014; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14015; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14016; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14017; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14018; GFX10-CU-NEXT:    buffer_gl0_inv
14019; GFX10-CU-NEXT:    buffer_gl1_inv
14020; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
14021; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
14022; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14023; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
14024; GFX10-CU-NEXT:    s_endpgm
14025;
14026; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
14027; SKIP-CACHE-INV:       ; %bb.0: ; %entry
14028; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14029; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
14030; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14031; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
14032; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
14033; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
14034; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
14035; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
14036; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
14037; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14038; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14039; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14040; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
14041; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
14042; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14043; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
14044; SKIP-CACHE-INV-NEXT:    s_endpgm
14045;
14046; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
14047; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
14048; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14049; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14050; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14051; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14052; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14053; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14054; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14055; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14056; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
14057; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14058; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14059; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
14060;
14061; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
14062; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
14063; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14064; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14065; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14066; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14067; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14068; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14069; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14070; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14071; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
14072; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14073; GFX90A-TGSPLIT-NEXT:    s_endpgm
14074;
14075; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
14076; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
14077; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14078; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14079; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14080; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14081; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14082; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
14083; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14084; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
14085; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14086; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
14087; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14088; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14089; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
14090;
14091; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
14092; GFX940-TGSPLIT:       ; %bb.0: ; %entry
14093; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14094; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14095; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14096; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14097; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14098; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
14099; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14100; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
14101; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14102; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
14103; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14104; GFX940-TGSPLIT-NEXT:    s_endpgm
14105;
14106; GFX11-WGP-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
14107; GFX11-WGP:       ; %bb.0: ; %entry
14108; GFX11-WGP-NEXT:    s_clause 0x1
14109; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14110; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14111; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14112; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
14113; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
14114; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
14115; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
14116; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14117; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14118; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14119; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14120; GFX11-WGP-NEXT:    buffer_gl0_inv
14121; GFX11-WGP-NEXT:    buffer_gl1_inv
14122; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14123; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
14124; GFX11-WGP-NEXT:    s_endpgm
14125;
14126; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
14127; GFX11-CU:       ; %bb.0: ; %entry
14128; GFX11-CU-NEXT:    s_clause 0x1
14129; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14130; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14131; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14132; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
14133; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
14134; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
14135; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
14136; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14137; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14138; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14139; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14140; GFX11-CU-NEXT:    buffer_gl0_inv
14141; GFX11-CU-NEXT:    buffer_gl1_inv
14142; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14143; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
14144; GFX11-CU-NEXT:    s_endpgm
14145    i32* %out, i32 %in, i32 %old) {
14146entry:
14147  %gep = getelementptr i32, i32* %out, i32 4
14148  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst acquire
14149  %val0 = extractvalue { i32, i1 } %val, 0
14150  store i32 %val0, i32* %out, align 4
14151  ret void
14152}
14153
14154define amdgpu_kernel void @flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg(
14155; GFX7-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
14156; GFX7:       ; %bb.0: ; %entry
14157; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14158; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
14159; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14160; GFX7-NEXT:    s_add_u32 s4, s0, 16
14161; GFX7-NEXT:    s_addc_u32 s5, s1, 0
14162; GFX7-NEXT:    v_mov_b32_e32 v0, s4
14163; GFX7-NEXT:    v_mov_b32_e32 v2, s2
14164; GFX7-NEXT:    v_mov_b32_e32 v1, s5
14165; GFX7-NEXT:    v_mov_b32_e32 v3, s3
14166; GFX7-NEXT:    s_waitcnt vmcnt(0)
14167; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14168; GFX7-NEXT:    s_waitcnt vmcnt(0)
14169; GFX7-NEXT:    buffer_wbinvl1_vol
14170; GFX7-NEXT:    v_mov_b32_e32 v0, s0
14171; GFX7-NEXT:    v_mov_b32_e32 v1, s1
14172; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14173; GFX7-NEXT:    flat_store_dword v[0:1], v2
14174; GFX7-NEXT:    s_endpgm
14175;
14176; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
14177; GFX10-WGP:       ; %bb.0: ; %entry
14178; GFX10-WGP-NEXT:    s_clause 0x1
14179; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14180; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14181; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14182; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
14183; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
14184; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
14185; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
14186; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
14187; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
14188; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14189; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14190; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14191; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14192; GFX10-WGP-NEXT:    buffer_gl0_inv
14193; GFX10-WGP-NEXT:    buffer_gl1_inv
14194; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
14195; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
14196; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14197; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
14198; GFX10-WGP-NEXT:    s_endpgm
14199;
14200; GFX10-CU-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
14201; GFX10-CU:       ; %bb.0: ; %entry
14202; GFX10-CU-NEXT:    s_clause 0x1
14203; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14204; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14205; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14206; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
14207; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
14208; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
14209; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
14210; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
14211; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
14212; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14213; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14214; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14215; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14216; GFX10-CU-NEXT:    buffer_gl0_inv
14217; GFX10-CU-NEXT:    buffer_gl1_inv
14218; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
14219; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
14220; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14221; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
14222; GFX10-CU-NEXT:    s_endpgm
14223;
14224; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
14225; SKIP-CACHE-INV:       ; %bb.0: ; %entry
14226; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14227; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
14228; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14229; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
14230; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
14231; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
14232; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
14233; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
14234; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
14235; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14236; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14237; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14238; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
14239; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
14240; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14241; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
14242; SKIP-CACHE-INV-NEXT:    s_endpgm
14243;
14244; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
14245; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
14246; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14247; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14248; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14249; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14250; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14251; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14252; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14253; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14254; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
14255; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14256; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14257; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
14258;
14259; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
14260; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
14261; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14262; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14263; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14264; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14265; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14266; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14267; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14268; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14269; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
14270; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14271; GFX90A-TGSPLIT-NEXT:    s_endpgm
14272;
14273; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
14274; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
14275; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14276; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14277; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14278; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14279; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14280; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
14281; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14282; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
14283; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14284; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
14285; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14286; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14287; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
14288;
14289; GFX940-TGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
14290; GFX940-TGSPLIT:       ; %bb.0: ; %entry
14291; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14292; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14293; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14294; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14295; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14296; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
14297; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14298; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
14299; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14300; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
14301; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14302; GFX940-TGSPLIT-NEXT:    s_endpgm
14303;
14304; GFX11-WGP-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
14305; GFX11-WGP:       ; %bb.0: ; %entry
14306; GFX11-WGP-NEXT:    s_clause 0x1
14307; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14308; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14309; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14310; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
14311; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
14312; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
14313; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
14314; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14315; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14316; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14317; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14318; GFX11-WGP-NEXT:    buffer_gl0_inv
14319; GFX11-WGP-NEXT:    buffer_gl1_inv
14320; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14321; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
14322; GFX11-WGP-NEXT:    s_endpgm
14323;
14324; GFX11-CU-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
14325; GFX11-CU:       ; %bb.0: ; %entry
14326; GFX11-CU-NEXT:    s_clause 0x1
14327; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14328; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14329; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14330; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
14331; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
14332; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
14333; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
14334; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14335; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14336; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14337; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14338; GFX11-CU-NEXT:    buffer_gl0_inv
14339; GFX11-CU-NEXT:    buffer_gl1_inv
14340; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14341; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
14342; GFX11-CU-NEXT:    s_endpgm
14343    i32* %out, i32 %in, i32 %old) {
14344entry:
14345  %gep = getelementptr i32, i32* %out, i32 4
14346  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") monotonic seq_cst
14347  %val0 = extractvalue { i32, i1 } %val, 0
14348  store i32 %val0, i32* %out, align 4
14349  ret void
14350}
14351
14352define amdgpu_kernel void @flat_agent_one_as_acquire_seq_cst_ret_cmpxchg(
14353; GFX7-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
14354; GFX7:       ; %bb.0: ; %entry
14355; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14356; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
14357; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14358; GFX7-NEXT:    s_add_u32 s4, s0, 16
14359; GFX7-NEXT:    s_addc_u32 s5, s1, 0
14360; GFX7-NEXT:    v_mov_b32_e32 v0, s4
14361; GFX7-NEXT:    v_mov_b32_e32 v2, s2
14362; GFX7-NEXT:    v_mov_b32_e32 v1, s5
14363; GFX7-NEXT:    v_mov_b32_e32 v3, s3
14364; GFX7-NEXT:    s_waitcnt vmcnt(0)
14365; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14366; GFX7-NEXT:    s_waitcnt vmcnt(0)
14367; GFX7-NEXT:    buffer_wbinvl1_vol
14368; GFX7-NEXT:    v_mov_b32_e32 v0, s0
14369; GFX7-NEXT:    v_mov_b32_e32 v1, s1
14370; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14371; GFX7-NEXT:    flat_store_dword v[0:1], v2
14372; GFX7-NEXT:    s_endpgm
14373;
14374; GFX10-WGP-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
14375; GFX10-WGP:       ; %bb.0: ; %entry
14376; GFX10-WGP-NEXT:    s_clause 0x1
14377; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14378; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14379; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14380; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
14381; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
14382; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
14383; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
14384; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
14385; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
14386; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14387; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14388; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14389; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14390; GFX10-WGP-NEXT:    buffer_gl0_inv
14391; GFX10-WGP-NEXT:    buffer_gl1_inv
14392; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
14393; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
14394; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14395; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
14396; GFX10-WGP-NEXT:    s_endpgm
14397;
14398; GFX10-CU-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
14399; GFX10-CU:       ; %bb.0: ; %entry
14400; GFX10-CU-NEXT:    s_clause 0x1
14401; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14402; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14403; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14404; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
14405; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
14406; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
14407; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
14408; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
14409; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
14410; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14411; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14412; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14413; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14414; GFX10-CU-NEXT:    buffer_gl0_inv
14415; GFX10-CU-NEXT:    buffer_gl1_inv
14416; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
14417; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
14418; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14419; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
14420; GFX10-CU-NEXT:    s_endpgm
14421;
14422; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
14423; SKIP-CACHE-INV:       ; %bb.0: ; %entry
14424; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14425; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
14426; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14427; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
14428; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
14429; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
14430; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
14431; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
14432; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
14433; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14434; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14435; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14436; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
14437; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
14438; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14439; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
14440; SKIP-CACHE-INV-NEXT:    s_endpgm
14441;
14442; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
14443; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
14444; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14445; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14446; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14447; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14448; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14449; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14450; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14451; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14452; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
14453; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14454; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14455; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
14456;
14457; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
14458; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
14459; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14460; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14461; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14462; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14463; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14464; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14465; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14466; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14467; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
14468; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14469; GFX90A-TGSPLIT-NEXT:    s_endpgm
14470;
14471; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
14472; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
14473; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14474; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14475; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14476; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14477; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14478; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
14479; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14480; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
14481; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14482; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
14483; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14484; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14485; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
14486;
14487; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
14488; GFX940-TGSPLIT:       ; %bb.0: ; %entry
14489; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14490; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14491; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14492; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14493; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14494; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
14495; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14496; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
14497; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14498; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
14499; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14500; GFX940-TGSPLIT-NEXT:    s_endpgm
14501;
14502; GFX11-WGP-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
14503; GFX11-WGP:       ; %bb.0: ; %entry
14504; GFX11-WGP-NEXT:    s_clause 0x1
14505; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14506; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14507; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14508; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
14509; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
14510; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
14511; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
14512; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14513; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14514; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14515; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14516; GFX11-WGP-NEXT:    buffer_gl0_inv
14517; GFX11-WGP-NEXT:    buffer_gl1_inv
14518; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14519; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
14520; GFX11-WGP-NEXT:    s_endpgm
14521;
14522; GFX11-CU-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
14523; GFX11-CU:       ; %bb.0: ; %entry
14524; GFX11-CU-NEXT:    s_clause 0x1
14525; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14526; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14527; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14528; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
14529; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
14530; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
14531; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
14532; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14533; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14534; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14535; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14536; GFX11-CU-NEXT:    buffer_gl0_inv
14537; GFX11-CU-NEXT:    buffer_gl1_inv
14538; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14539; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
14540; GFX11-CU-NEXT:    s_endpgm
14541    i32* %out, i32 %in, i32 %old) {
14542entry:
14543  %gep = getelementptr i32, i32* %out, i32 4
14544  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire seq_cst
14545  %val0 = extractvalue { i32, i1 } %val, 0
14546  store i32 %val0, i32* %out, align 4
14547  ret void
14548}
14549
14550define amdgpu_kernel void @flat_agent_one_as_release_seq_cst_ret_cmpxchg(
14551; GFX7-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
14552; GFX7:       ; %bb.0: ; %entry
14553; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14554; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
14555; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14556; GFX7-NEXT:    s_add_u32 s4, s0, 16
14557; GFX7-NEXT:    s_addc_u32 s5, s1, 0
14558; GFX7-NEXT:    v_mov_b32_e32 v0, s4
14559; GFX7-NEXT:    v_mov_b32_e32 v2, s2
14560; GFX7-NEXT:    v_mov_b32_e32 v1, s5
14561; GFX7-NEXT:    v_mov_b32_e32 v3, s3
14562; GFX7-NEXT:    s_waitcnt vmcnt(0)
14563; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14564; GFX7-NEXT:    s_waitcnt vmcnt(0)
14565; GFX7-NEXT:    buffer_wbinvl1_vol
14566; GFX7-NEXT:    v_mov_b32_e32 v0, s0
14567; GFX7-NEXT:    v_mov_b32_e32 v1, s1
14568; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14569; GFX7-NEXT:    flat_store_dword v[0:1], v2
14570; GFX7-NEXT:    s_endpgm
14571;
14572; GFX10-WGP-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
14573; GFX10-WGP:       ; %bb.0: ; %entry
14574; GFX10-WGP-NEXT:    s_clause 0x1
14575; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14576; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14577; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14578; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
14579; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
14580; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
14581; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
14582; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
14583; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
14584; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14585; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14586; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14587; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14588; GFX10-WGP-NEXT:    buffer_gl0_inv
14589; GFX10-WGP-NEXT:    buffer_gl1_inv
14590; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
14591; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
14592; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14593; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
14594; GFX10-WGP-NEXT:    s_endpgm
14595;
14596; GFX10-CU-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
14597; GFX10-CU:       ; %bb.0: ; %entry
14598; GFX10-CU-NEXT:    s_clause 0x1
14599; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14600; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14601; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14602; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
14603; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
14604; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
14605; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
14606; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
14607; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
14608; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14609; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14610; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14611; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14612; GFX10-CU-NEXT:    buffer_gl0_inv
14613; GFX10-CU-NEXT:    buffer_gl1_inv
14614; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
14615; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
14616; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14617; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
14618; GFX10-CU-NEXT:    s_endpgm
14619;
14620; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
14621; SKIP-CACHE-INV:       ; %bb.0: ; %entry
14622; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14623; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
14624; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14625; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
14626; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
14627; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
14628; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
14629; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
14630; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
14631; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14632; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14633; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14634; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
14635; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
14636; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14637; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
14638; SKIP-CACHE-INV-NEXT:    s_endpgm
14639;
14640; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
14641; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
14642; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14643; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14644; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14645; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14646; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14647; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14648; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14649; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14650; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
14651; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14652; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14653; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
14654;
14655; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
14656; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
14657; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14658; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14659; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14660; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14661; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14662; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14663; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14664; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14665; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
14666; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14667; GFX90A-TGSPLIT-NEXT:    s_endpgm
14668;
14669; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
14670; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
14671; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14672; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14673; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14674; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14675; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14676; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
14677; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14678; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
14679; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14680; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
14681; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14682; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14683; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
14684;
14685; GFX940-TGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
14686; GFX940-TGSPLIT:       ; %bb.0: ; %entry
14687; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14688; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14689; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14690; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14691; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14692; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
14693; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14694; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
14695; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14696; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
14697; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14698; GFX940-TGSPLIT-NEXT:    s_endpgm
14699;
14700; GFX11-WGP-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
14701; GFX11-WGP:       ; %bb.0: ; %entry
14702; GFX11-WGP-NEXT:    s_clause 0x1
14703; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14704; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14705; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14706; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
14707; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
14708; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
14709; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
14710; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14711; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14712; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14713; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14714; GFX11-WGP-NEXT:    buffer_gl0_inv
14715; GFX11-WGP-NEXT:    buffer_gl1_inv
14716; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14717; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
14718; GFX11-WGP-NEXT:    s_endpgm
14719;
14720; GFX11-CU-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
14721; GFX11-CU:       ; %bb.0: ; %entry
14722; GFX11-CU-NEXT:    s_clause 0x1
14723; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14724; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14725; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14726; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
14727; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
14728; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
14729; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
14730; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14731; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14732; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14733; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14734; GFX11-CU-NEXT:    buffer_gl0_inv
14735; GFX11-CU-NEXT:    buffer_gl1_inv
14736; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14737; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
14738; GFX11-CU-NEXT:    s_endpgm
14739    i32* %out, i32 %in, i32 %old) {
14740entry:
14741  %gep = getelementptr i32, i32* %out, i32 4
14742  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release seq_cst
14743  %val0 = extractvalue { i32, i1 } %val, 0
14744  store i32 %val0, i32* %out, align 4
14745  ret void
14746}
14747
14748define amdgpu_kernel void @flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg(
14749; GFX7-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
14750; GFX7:       ; %bb.0: ; %entry
14751; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14752; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
14753; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14754; GFX7-NEXT:    s_add_u32 s4, s0, 16
14755; GFX7-NEXT:    s_addc_u32 s5, s1, 0
14756; GFX7-NEXT:    v_mov_b32_e32 v0, s4
14757; GFX7-NEXT:    v_mov_b32_e32 v2, s2
14758; GFX7-NEXT:    v_mov_b32_e32 v1, s5
14759; GFX7-NEXT:    v_mov_b32_e32 v3, s3
14760; GFX7-NEXT:    s_waitcnt vmcnt(0)
14761; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14762; GFX7-NEXT:    s_waitcnt vmcnt(0)
14763; GFX7-NEXT:    buffer_wbinvl1_vol
14764; GFX7-NEXT:    v_mov_b32_e32 v0, s0
14765; GFX7-NEXT:    v_mov_b32_e32 v1, s1
14766; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14767; GFX7-NEXT:    flat_store_dword v[0:1], v2
14768; GFX7-NEXT:    s_endpgm
14769;
14770; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
14771; GFX10-WGP:       ; %bb.0: ; %entry
14772; GFX10-WGP-NEXT:    s_clause 0x1
14773; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14774; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14775; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14776; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
14777; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
14778; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
14779; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
14780; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
14781; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
14782; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14783; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14784; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14785; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14786; GFX10-WGP-NEXT:    buffer_gl0_inv
14787; GFX10-WGP-NEXT:    buffer_gl1_inv
14788; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
14789; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
14790; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14791; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
14792; GFX10-WGP-NEXT:    s_endpgm
14793;
14794; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
14795; GFX10-CU:       ; %bb.0: ; %entry
14796; GFX10-CU-NEXT:    s_clause 0x1
14797; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14798; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14799; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14800; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
14801; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
14802; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
14803; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
14804; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
14805; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
14806; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14807; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14808; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14809; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
14810; GFX10-CU-NEXT:    buffer_gl0_inv
14811; GFX10-CU-NEXT:    buffer_gl1_inv
14812; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
14813; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
14814; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14815; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
14816; GFX10-CU-NEXT:    s_endpgm
14817;
14818; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
14819; SKIP-CACHE-INV:       ; %bb.0: ; %entry
14820; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14821; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
14822; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14823; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
14824; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
14825; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
14826; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
14827; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
14828; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
14829; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14830; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14831; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
14832; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
14833; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
14834; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
14835; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
14836; SKIP-CACHE-INV-NEXT:    s_endpgm
14837;
14838; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
14839; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
14840; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14841; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14842; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14843; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14844; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14845; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14846; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14847; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14848; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
14849; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14850; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14851; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
14852;
14853; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
14854; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
14855; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14856; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14857; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14858; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
14859; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
14860; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14861; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
14862; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14863; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
14864; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14865; GFX90A-TGSPLIT-NEXT:    s_endpgm
14866;
14867; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
14868; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
14869; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14870; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14871; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14872; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14873; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14874; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
14875; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14876; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
14877; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14878; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
14879; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14880; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14881; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
14882;
14883; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
14884; GFX940-TGSPLIT:       ; %bb.0: ; %entry
14885; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
14886; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
14887; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
14888; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
14889; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
14890; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
14891; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14892; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
14893; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
14894; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
14895; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
14896; GFX940-TGSPLIT-NEXT:    s_endpgm
14897;
14898; GFX11-WGP-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
14899; GFX11-WGP:       ; %bb.0: ; %entry
14900; GFX11-WGP-NEXT:    s_clause 0x1
14901; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14902; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14903; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14904; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
14905; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
14906; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
14907; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
14908; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14909; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14910; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14911; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
14912; GFX11-WGP-NEXT:    buffer_gl0_inv
14913; GFX11-WGP-NEXT:    buffer_gl1_inv
14914; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14915; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
14916; GFX11-WGP-NEXT:    s_endpgm
14917;
14918; GFX11-CU-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
14919; GFX11-CU:       ; %bb.0: ; %entry
14920; GFX11-CU-NEXT:    s_clause 0x1
14921; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
14922; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
14923; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14924; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
14925; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
14926; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
14927; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
14928; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14929; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
14930; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
14931; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
14932; GFX11-CU-NEXT:    buffer_gl0_inv
14933; GFX11-CU-NEXT:    buffer_gl1_inv
14934; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
14935; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
14936; GFX11-CU-NEXT:    s_endpgm
14937    i32* %out, i32 %in, i32 %old) {
14938entry:
14939  %gep = getelementptr i32, i32* %out, i32 4
14940  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel seq_cst
14941  %val0 = extractvalue { i32, i1 } %val, 0
14942  store i32 %val0, i32* %out, align 4
14943  ret void
14944}
14945
14946define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg(
14947; GFX7-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
14948; GFX7:       ; %bb.0: ; %entry
14949; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14950; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
14951; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14952; GFX7-NEXT:    s_add_u32 s4, s0, 16
14953; GFX7-NEXT:    s_addc_u32 s5, s1, 0
14954; GFX7-NEXT:    v_mov_b32_e32 v0, s4
14955; GFX7-NEXT:    v_mov_b32_e32 v2, s2
14956; GFX7-NEXT:    v_mov_b32_e32 v1, s5
14957; GFX7-NEXT:    v_mov_b32_e32 v3, s3
14958; GFX7-NEXT:    s_waitcnt vmcnt(0)
14959; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14960; GFX7-NEXT:    s_waitcnt vmcnt(0)
14961; GFX7-NEXT:    buffer_wbinvl1_vol
14962; GFX7-NEXT:    v_mov_b32_e32 v0, s0
14963; GFX7-NEXT:    v_mov_b32_e32 v1, s1
14964; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14965; GFX7-NEXT:    flat_store_dword v[0:1], v2
14966; GFX7-NEXT:    s_endpgm
14967;
14968; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
14969; GFX10-WGP:       ; %bb.0: ; %entry
14970; GFX10-WGP-NEXT:    s_clause 0x1
14971; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14972; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14973; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14974; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
14975; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
14976; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
14977; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
14978; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
14979; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
14980; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14981; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
14982; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
14983; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
14984; GFX10-WGP-NEXT:    buffer_gl0_inv
14985; GFX10-WGP-NEXT:    buffer_gl1_inv
14986; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
14987; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
14988; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
14989; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
14990; GFX10-WGP-NEXT:    s_endpgm
14991;
14992; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
14993; GFX10-CU:       ; %bb.0: ; %entry
14994; GFX10-CU-NEXT:    s_clause 0x1
14995; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
14996; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
14997; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
14998; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
14999; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
15000; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
15001; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
15002; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
15003; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
15004; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
15005; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
15006; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
15007; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
15008; GFX10-CU-NEXT:    buffer_gl0_inv
15009; GFX10-CU-NEXT:    buffer_gl1_inv
15010; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
15011; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
15012; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
15013; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
15014; GFX10-CU-NEXT:    s_endpgm
15015;
15016; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
15017; SKIP-CACHE-INV:       ; %bb.0: ; %entry
15018; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
15019; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
15020; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
15021; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
15022; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
15023; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
15024; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
15025; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
15026; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
15027; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
15028; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
15029; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
15030; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
15031; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
15032; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
15033; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
15034; SKIP-CACHE-INV-NEXT:    s_endpgm
15035;
15036; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
15037; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
15038; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
15039; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
15040; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
15041; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
15042; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
15043; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
15044; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
15045; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
15046; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
15047; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
15048; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
15049; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
15050;
15051; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
15052; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
15053; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
15054; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
15055; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
15056; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
15057; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
15058; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
15059; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
15060; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
15061; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
15062; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
15063; GFX90A-TGSPLIT-NEXT:    s_endpgm
15064;
15065; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
15066; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
15067; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
15068; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
15069; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
15070; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
15071; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
15072; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
15073; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
15074; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
15075; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
15076; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
15077; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
15078; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
15079; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
15080;
15081; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
15082; GFX940-TGSPLIT:       ; %bb.0: ; %entry
15083; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
15084; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
15085; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
15086; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
15087; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
15088; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
15089; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
15090; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
15091; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
15092; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
15093; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
15094; GFX940-TGSPLIT-NEXT:    s_endpgm
15095;
15096; GFX11-WGP-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
15097; GFX11-WGP:       ; %bb.0: ; %entry
15098; GFX11-WGP-NEXT:    s_clause 0x1
15099; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
15100; GFX11-WGP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
15101; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
15102; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
15103; GFX11-WGP-NEXT:    v_mov_b32_e32 v3, s1
15104; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
15105; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
15106; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
15107; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
15108; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
15109; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
15110; GFX11-WGP-NEXT:    buffer_gl0_inv
15111; GFX11-WGP-NEXT:    buffer_gl1_inv
15112; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
15113; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
15114; GFX11-WGP-NEXT:    s_endpgm
15115;
15116; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
15117; GFX11-CU:       ; %bb.0: ; %entry
15118; GFX11-CU-NEXT:    s_clause 0x1
15119; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
15120; GFX11-CU-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
15121; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
15122; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
15123; GFX11-CU-NEXT:    v_mov_b32_e32 v3, s1
15124; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
15125; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
15126; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
15127; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
15128; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
15129; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
15130; GFX11-CU-NEXT:    buffer_gl0_inv
15131; GFX11-CU-NEXT:    buffer_gl1_inv
15132; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
15133; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
15134; GFX11-CU-NEXT:    s_endpgm
15135    i32* %out, i32 %in, i32 %old) {
15136entry:
15137  %gep = getelementptr i32, i32* %out, i32 4
15138  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst seq_cst
15139  %val0 = extractvalue { i32, i1 } %val, 0
15140  store i32 %val0, i32* %out, align 4
15141  ret void
15142}
15143
15144