1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
3; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
4; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
5; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
6; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-NOTTGSPLIT %s
7; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-TGSPLIT %s
8
9define amdgpu_kernel void @flat_agent_unordered_load(
10; GFX7-LABEL: flat_agent_unordered_load:
11; GFX7:       ; %bb.0: ; %entry
12; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
13; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14; GFX7-NEXT:    v_mov_b32_e32 v0, s0
15; GFX7-NEXT:    v_mov_b32_e32 v1, s1
16; GFX7-NEXT:    flat_load_dword v2, v[0:1]
17; GFX7-NEXT:    v_mov_b32_e32 v0, s2
18; GFX7-NEXT:    v_mov_b32_e32 v1, s3
19; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
20; GFX7-NEXT:    flat_store_dword v[0:1], v2
21; GFX7-NEXT:    s_endpgm
22;
23; GFX10-WGP-LABEL: flat_agent_unordered_load:
24; GFX10-WGP:       ; %bb.0: ; %entry
25; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
26; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
27; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
28; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
29; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
30; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
31; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
32; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
33; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
34; GFX10-WGP-NEXT:    s_endpgm
35;
36; GFX10-CU-LABEL: flat_agent_unordered_load:
37; GFX10-CU:       ; %bb.0: ; %entry
38; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
39; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
40; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
41; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
42; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
43; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
44; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
45; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
46; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
47; GFX10-CU-NEXT:    s_endpgm
48;
49; SKIP-CACHE-INV-LABEL: flat_agent_unordered_load:
50; SKIP-CACHE-INV:       ; %bb.0: ; %entry
51; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
52; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
53; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
54; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
55; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
56; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
57; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
58; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
59; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
60; SKIP-CACHE-INV-NEXT:    s_endpgm
61;
62; GFX90A-NOTTGSPLIT-LABEL: flat_agent_unordered_load:
63; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
64; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
65; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
66; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
67; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
68; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
69; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
70; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
71; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
72; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
73; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
74;
75; GFX90A-TGSPLIT-LABEL: flat_agent_unordered_load:
76; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
77; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
78; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
79; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
80; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
81; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
82; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
83; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
84; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
85; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
86; GFX90A-TGSPLIT-NEXT:    s_endpgm
87    i32* %in, i32* %out) {
88entry:
89  %val = load atomic i32, i32* %in syncscope("agent") unordered, align 4
90  store i32 %val, i32* %out
91  ret void
92}
93
94define amdgpu_kernel void @flat_agent_monotonic_load(
95; GFX7-LABEL: flat_agent_monotonic_load:
96; GFX7:       ; %bb.0: ; %entry
97; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
98; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
99; GFX7-NEXT:    v_mov_b32_e32 v0, s0
100; GFX7-NEXT:    v_mov_b32_e32 v1, s1
101; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
102; GFX7-NEXT:    v_mov_b32_e32 v0, s2
103; GFX7-NEXT:    v_mov_b32_e32 v1, s3
104; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
105; GFX7-NEXT:    flat_store_dword v[0:1], v2
106; GFX7-NEXT:    s_endpgm
107;
108; GFX10-WGP-LABEL: flat_agent_monotonic_load:
109; GFX10-WGP:       ; %bb.0: ; %entry
110; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
111; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
112; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
113; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
114; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
115; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
116; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
117; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
118; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
119; GFX10-WGP-NEXT:    s_endpgm
120;
121; GFX10-CU-LABEL: flat_agent_monotonic_load:
122; GFX10-CU:       ; %bb.0: ; %entry
123; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
124; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
125; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
126; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
127; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
128; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
129; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
130; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
131; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
132; GFX10-CU-NEXT:    s_endpgm
133;
134; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_load:
135; SKIP-CACHE-INV:       ; %bb.0: ; %entry
136; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
137; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
138; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
139; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
140; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
141; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
142; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
143; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
144; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
145; SKIP-CACHE-INV-NEXT:    s_endpgm
146;
147; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_load:
148; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
149; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
150; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
151; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
152; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
153; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
154; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
155; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
156; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
157; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
158; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
159;
160; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_load:
161; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
162; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
163; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
164; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
165; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
166; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
167; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
168; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
169; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
170; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
171; GFX90A-TGSPLIT-NEXT:    s_endpgm
172    i32* %in, i32* %out) {
173entry:
174  %val = load atomic i32, i32* %in syncscope("agent") monotonic, align 4
175  store i32 %val, i32* %out
176  ret void
177}
178
179define amdgpu_kernel void @flat_agent_acquire_load(
180; GFX7-LABEL: flat_agent_acquire_load:
181; GFX7:       ; %bb.0: ; %entry
182; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
183; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
184; GFX7-NEXT:    v_mov_b32_e32 v0, s0
185; GFX7-NEXT:    v_mov_b32_e32 v1, s1
186; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
187; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
188; GFX7-NEXT:    buffer_wbinvl1_vol
189; GFX7-NEXT:    v_mov_b32_e32 v0, s2
190; GFX7-NEXT:    v_mov_b32_e32 v1, s3
191; GFX7-NEXT:    flat_store_dword v[0:1], v2
192; GFX7-NEXT:    s_endpgm
193;
194; GFX10-WGP-LABEL: flat_agent_acquire_load:
195; GFX10-WGP:       ; %bb.0: ; %entry
196; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
197; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
198; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
199; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
200; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
201; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
202; GFX10-WGP-NEXT:    buffer_gl0_inv
203; GFX10-WGP-NEXT:    buffer_gl1_inv
204; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
205; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
206; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
207; GFX10-WGP-NEXT:    s_endpgm
208;
209; GFX10-CU-LABEL: flat_agent_acquire_load:
210; GFX10-CU:       ; %bb.0: ; %entry
211; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
212; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
213; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
214; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
215; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
216; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
217; GFX10-CU-NEXT:    buffer_gl0_inv
218; GFX10-CU-NEXT:    buffer_gl1_inv
219; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
220; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
221; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
222; GFX10-CU-NEXT:    s_endpgm
223;
224; SKIP-CACHE-INV-LABEL: flat_agent_acquire_load:
225; SKIP-CACHE-INV:       ; %bb.0: ; %entry
226; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
227; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
228; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
229; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
230; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
231; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
232; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
233; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
234; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
235; SKIP-CACHE-INV-NEXT:    s_endpgm
236;
237; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_load:
238; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
239; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
240; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
241; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
242; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
243; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
244; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
245; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
246; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
247; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
248; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
249; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
250;
251; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_load:
252; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
253; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
254; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
255; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
256; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
257; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
258; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
259; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
260; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
261; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
262; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
263; GFX90A-TGSPLIT-NEXT:    s_endpgm
264    i32* %in, i32* %out) {
265entry:
266  %val = load atomic i32, i32* %in syncscope("agent") acquire, align 4
267  store i32 %val, i32* %out
268  ret void
269}
270
271define amdgpu_kernel void @flat_agent_seq_cst_load(
272; GFX7-LABEL: flat_agent_seq_cst_load:
273; GFX7:       ; %bb.0: ; %entry
274; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
275; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
276; GFX7-NEXT:    v_mov_b32_e32 v0, s0
277; GFX7-NEXT:    v_mov_b32_e32 v1, s1
278; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
279; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
280; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
281; GFX7-NEXT:    buffer_wbinvl1_vol
282; GFX7-NEXT:    v_mov_b32_e32 v0, s2
283; GFX7-NEXT:    v_mov_b32_e32 v1, s3
284; GFX7-NEXT:    flat_store_dword v[0:1], v2
285; GFX7-NEXT:    s_endpgm
286;
287; GFX10-WGP-LABEL: flat_agent_seq_cst_load:
288; GFX10-WGP:       ; %bb.0: ; %entry
289; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
290; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
291; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
292; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
293; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
294; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
295; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
296; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
297; GFX10-WGP-NEXT:    buffer_gl0_inv
298; GFX10-WGP-NEXT:    buffer_gl1_inv
299; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
300; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
301; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
302; GFX10-WGP-NEXT:    s_endpgm
303;
304; GFX10-CU-LABEL: flat_agent_seq_cst_load:
305; GFX10-CU:       ; %bb.0: ; %entry
306; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
307; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
308; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
309; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
310; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
311; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
312; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
313; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
314; GFX10-CU-NEXT:    buffer_gl0_inv
315; GFX10-CU-NEXT:    buffer_gl1_inv
316; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
317; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
318; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
319; GFX10-CU-NEXT:    s_endpgm
320;
321; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_load:
322; SKIP-CACHE-INV:       ; %bb.0: ; %entry
323; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
324; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
325; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
326; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
327; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
328; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
329; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
330; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
331; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
332; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
333; SKIP-CACHE-INV-NEXT:    s_endpgm
334;
335; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_load:
336; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
337; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
338; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
339; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
340; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
341; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
342; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
343; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
344; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
345; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
346; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
347; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
348; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
349;
350; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_load:
351; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
352; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
353; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
354; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
355; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
356; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
357; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
358; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
359; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
360; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
361; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
362; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
363; GFX90A-TGSPLIT-NEXT:    s_endpgm
364    i32* %in, i32* %out) {
365entry:
366  %val = load atomic i32, i32* %in syncscope("agent") seq_cst, align 4
367  store i32 %val, i32* %out
368  ret void
369}
370
371define amdgpu_kernel void @flat_agent_unordered_store(
372; GFX7-LABEL: flat_agent_unordered_store:
373; GFX7:       ; %bb.0: ; %entry
374; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
375; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
376; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
377; GFX7-NEXT:    v_mov_b32_e32 v0, s0
378; GFX7-NEXT:    v_mov_b32_e32 v1, s1
379; GFX7-NEXT:    v_mov_b32_e32 v2, s2
380; GFX7-NEXT:    flat_store_dword v[0:1], v2
381; GFX7-NEXT:    s_endpgm
382;
383; GFX10-WGP-LABEL: flat_agent_unordered_store:
384; GFX10-WGP:       ; %bb.0: ; %entry
385; GFX10-WGP-NEXT:    s_clause 0x1
386; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
387; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
388; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
389; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
390; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
391; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
392; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
393; GFX10-WGP-NEXT:    s_endpgm
394;
395; GFX10-CU-LABEL: flat_agent_unordered_store:
396; GFX10-CU:       ; %bb.0: ; %entry
397; GFX10-CU-NEXT:    s_clause 0x1
398; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
399; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
400; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
401; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
402; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
403; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
404; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
405; GFX10-CU-NEXT:    s_endpgm
406;
407; SKIP-CACHE-INV-LABEL: flat_agent_unordered_store:
408; SKIP-CACHE-INV:       ; %bb.0: ; %entry
409; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
410; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
411; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
412; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
413; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
414; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
415; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
416; SKIP-CACHE-INV-NEXT:    s_endpgm
417;
418; GFX90A-NOTTGSPLIT-LABEL: flat_agent_unordered_store:
419; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
420; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
421; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
422; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
423; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
424; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
425; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
426; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
427;
428; GFX90A-TGSPLIT-LABEL: flat_agent_unordered_store:
429; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
430; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
431; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
432; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
433; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
434; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
435; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
436; GFX90A-TGSPLIT-NEXT:    s_endpgm
437    i32 %in, i32* %out) {
438entry:
439  store atomic i32 %in, i32* %out syncscope("agent") unordered, align 4
440  ret void
441}
442
443define amdgpu_kernel void @flat_agent_monotonic_store(
444; GFX7-LABEL: flat_agent_monotonic_store:
445; GFX7:       ; %bb.0: ; %entry
446; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
447; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
448; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
449; GFX7-NEXT:    v_mov_b32_e32 v0, s0
450; GFX7-NEXT:    v_mov_b32_e32 v1, s1
451; GFX7-NEXT:    v_mov_b32_e32 v2, s2
452; GFX7-NEXT:    flat_store_dword v[0:1], v2
453; GFX7-NEXT:    s_endpgm
454;
455; GFX10-WGP-LABEL: flat_agent_monotonic_store:
456; GFX10-WGP:       ; %bb.0: ; %entry
457; GFX10-WGP-NEXT:    s_clause 0x1
458; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
459; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
460; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
461; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
462; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
463; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
464; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
465; GFX10-WGP-NEXT:    s_endpgm
466;
467; GFX10-CU-LABEL: flat_agent_monotonic_store:
468; GFX10-CU:       ; %bb.0: ; %entry
469; GFX10-CU-NEXT:    s_clause 0x1
470; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
471; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
472; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
473; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
474; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
475; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
476; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
477; GFX10-CU-NEXT:    s_endpgm
478;
479; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_store:
480; SKIP-CACHE-INV:       ; %bb.0: ; %entry
481; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
482; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
483; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
484; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
485; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
486; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
487; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
488; SKIP-CACHE-INV-NEXT:    s_endpgm
489;
490; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_store:
491; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
492; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
493; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
494; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
495; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
496; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
497; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
498; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
499;
500; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_store:
501; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
502; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
503; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
504; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
505; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
506; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
507; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
508; GFX90A-TGSPLIT-NEXT:    s_endpgm
509    i32 %in, i32* %out) {
510entry:
511  store atomic i32 %in, i32* %out syncscope("agent") monotonic, align 4
512  ret void
513}
514
515define amdgpu_kernel void @flat_agent_release_store(
516; GFX7-LABEL: flat_agent_release_store:
517; GFX7:       ; %bb.0: ; %entry
518; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
519; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
520; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
521; GFX7-NEXT:    v_mov_b32_e32 v0, s0
522; GFX7-NEXT:    v_mov_b32_e32 v1, s1
523; GFX7-NEXT:    v_mov_b32_e32 v2, s2
524; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
525; GFX7-NEXT:    flat_store_dword v[0:1], v2
526; GFX7-NEXT:    s_endpgm
527;
528; GFX10-WGP-LABEL: flat_agent_release_store:
529; GFX10-WGP:       ; %bb.0: ; %entry
530; GFX10-WGP-NEXT:    s_clause 0x1
531; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
532; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
533; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
534; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
535; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
536; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
537; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
538; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
539; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
540; GFX10-WGP-NEXT:    s_endpgm
541;
542; GFX10-CU-LABEL: flat_agent_release_store:
543; GFX10-CU:       ; %bb.0: ; %entry
544; GFX10-CU-NEXT:    s_clause 0x1
545; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
546; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
547; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
548; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
549; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
550; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
551; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
552; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
553; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
554; GFX10-CU-NEXT:    s_endpgm
555;
556; SKIP-CACHE-INV-LABEL: flat_agent_release_store:
557; SKIP-CACHE-INV:       ; %bb.0: ; %entry
558; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
559; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
560; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
561; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
562; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
563; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
564; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
565; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
566; SKIP-CACHE-INV-NEXT:    s_endpgm
567;
568; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_store:
569; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
570; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
571; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
572; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
573; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
574; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
575; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
576; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
577; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
578;
579; GFX90A-TGSPLIT-LABEL: flat_agent_release_store:
580; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
581; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
582; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
583; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
584; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
585; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
586; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
587; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
588; GFX90A-TGSPLIT-NEXT:    s_endpgm
589    i32 %in, i32* %out) {
590entry:
591  store atomic i32 %in, i32* %out syncscope("agent") release, align 4
592  ret void
593}
594
595define amdgpu_kernel void @flat_agent_seq_cst_store(
596; GFX7-LABEL: flat_agent_seq_cst_store:
597; GFX7:       ; %bb.0: ; %entry
598; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
599; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
600; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
601; GFX7-NEXT:    v_mov_b32_e32 v0, s0
602; GFX7-NEXT:    v_mov_b32_e32 v1, s1
603; GFX7-NEXT:    v_mov_b32_e32 v2, s2
604; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
605; GFX7-NEXT:    flat_store_dword v[0:1], v2
606; GFX7-NEXT:    s_endpgm
607;
608; GFX10-WGP-LABEL: flat_agent_seq_cst_store:
609; GFX10-WGP:       ; %bb.0: ; %entry
610; GFX10-WGP-NEXT:    s_clause 0x1
611; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
612; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
613; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
614; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
615; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
616; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
617; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
618; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
619; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
620; GFX10-WGP-NEXT:    s_endpgm
621;
622; GFX10-CU-LABEL: flat_agent_seq_cst_store:
623; GFX10-CU:       ; %bb.0: ; %entry
624; GFX10-CU-NEXT:    s_clause 0x1
625; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
626; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
627; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
628; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
629; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
630; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
631; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
632; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
633; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
634; GFX10-CU-NEXT:    s_endpgm
635;
636; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_store:
637; SKIP-CACHE-INV:       ; %bb.0: ; %entry
638; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
639; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
640; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
641; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
642; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
643; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
644; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
645; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
646; SKIP-CACHE-INV-NEXT:    s_endpgm
647;
648; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_store:
649; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
650; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
651; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
652; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
653; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
654; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
655; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
656; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
657; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
658;
659; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_store:
660; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
661; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
662; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
663; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
664; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
665; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
666; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
667; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
668; GFX90A-TGSPLIT-NEXT:    s_endpgm
669    i32 %in, i32* %out) {
670entry:
671  store atomic i32 %in, i32* %out syncscope("agent") seq_cst, align 4
672  ret void
673}
674
675define amdgpu_kernel void @flat_agent_monotonic_atomicrmw(
676; GFX7-LABEL: flat_agent_monotonic_atomicrmw:
677; GFX7:       ; %bb.0: ; %entry
678; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
679; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
680; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
681; GFX7-NEXT:    v_mov_b32_e32 v0, s0
682; GFX7-NEXT:    v_mov_b32_e32 v1, s1
683; GFX7-NEXT:    v_mov_b32_e32 v2, s2
684; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
685; GFX7-NEXT:    s_endpgm
686;
687; GFX10-WGP-LABEL: flat_agent_monotonic_atomicrmw:
688; GFX10-WGP:       ; %bb.0: ; %entry
689; GFX10-WGP-NEXT:    s_clause 0x1
690; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
691; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
692; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
693; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
694; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
695; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
696; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
697; GFX10-WGP-NEXT:    s_endpgm
698;
699; GFX10-CU-LABEL: flat_agent_monotonic_atomicrmw:
700; GFX10-CU:       ; %bb.0: ; %entry
701; GFX10-CU-NEXT:    s_clause 0x1
702; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
703; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
704; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
705; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
706; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
707; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
708; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
709; GFX10-CU-NEXT:    s_endpgm
710;
711; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_atomicrmw:
712; SKIP-CACHE-INV:       ; %bb.0: ; %entry
713; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
714; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
715; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
716; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
717; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
718; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
719; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
720; SKIP-CACHE-INV-NEXT:    s_endpgm
721;
722; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_atomicrmw:
723; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
724; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
725; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
726; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
727; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
728; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
729; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
730; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
731;
732; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_atomicrmw:
733; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
734; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
735; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
736; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
737; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
738; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
739; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
740; GFX90A-TGSPLIT-NEXT:    s_endpgm
741    i32* %out, i32 %in) {
742entry:
743  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") monotonic
744  ret void
745}
746
747define amdgpu_kernel void @flat_agent_acquire_atomicrmw(
748; GFX7-LABEL: flat_agent_acquire_atomicrmw:
749; GFX7:       ; %bb.0: ; %entry
750; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
751; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
752; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
753; GFX7-NEXT:    v_mov_b32_e32 v0, s0
754; GFX7-NEXT:    v_mov_b32_e32 v1, s1
755; GFX7-NEXT:    v_mov_b32_e32 v2, s2
756; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
757; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
758; GFX7-NEXT:    buffer_wbinvl1_vol
759; GFX7-NEXT:    s_endpgm
760;
761; GFX10-WGP-LABEL: flat_agent_acquire_atomicrmw:
762; GFX10-WGP:       ; %bb.0: ; %entry
763; GFX10-WGP-NEXT:    s_clause 0x1
764; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
765; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
766; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
767; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
768; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
769; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
770; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
771; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
772; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
773; GFX10-WGP-NEXT:    buffer_gl0_inv
774; GFX10-WGP-NEXT:    buffer_gl1_inv
775; GFX10-WGP-NEXT:    s_endpgm
776;
777; GFX10-CU-LABEL: flat_agent_acquire_atomicrmw:
778; GFX10-CU:       ; %bb.0: ; %entry
779; GFX10-CU-NEXT:    s_clause 0x1
780; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
781; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
782; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
783; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
784; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
785; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
786; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
787; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
788; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
789; GFX10-CU-NEXT:    buffer_gl0_inv
790; GFX10-CU-NEXT:    buffer_gl1_inv
791; GFX10-CU-NEXT:    s_endpgm
792;
793; SKIP-CACHE-INV-LABEL: flat_agent_acquire_atomicrmw:
794; SKIP-CACHE-INV:       ; %bb.0: ; %entry
795; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
796; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
797; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
798; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
799; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
800; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
801; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
802; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
803; SKIP-CACHE-INV-NEXT:    s_endpgm
804;
805; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_atomicrmw:
806; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
807; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
808; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
809; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
810; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
811; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
812; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
813; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
814; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
815; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
816;
817; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_atomicrmw:
818; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
819; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
820; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
821; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
822; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
823; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
824; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
825; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
826; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
827; GFX90A-TGSPLIT-NEXT:    s_endpgm
828    i32* %out, i32 %in) {
829entry:
830  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") acquire
831  ret void
832}
833
834define amdgpu_kernel void @flat_agent_release_atomicrmw(
835; GFX7-LABEL: flat_agent_release_atomicrmw:
836; GFX7:       ; %bb.0: ; %entry
837; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
838; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
839; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
840; GFX7-NEXT:    v_mov_b32_e32 v0, s0
841; GFX7-NEXT:    v_mov_b32_e32 v1, s1
842; GFX7-NEXT:    v_mov_b32_e32 v2, s2
843; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
844; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
845; GFX7-NEXT:    s_endpgm
846;
847; GFX10-WGP-LABEL: flat_agent_release_atomicrmw:
848; GFX10-WGP:       ; %bb.0: ; %entry
849; GFX10-WGP-NEXT:    s_clause 0x1
850; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
851; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
852; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
853; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
854; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
855; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
856; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
857; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
858; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
859; GFX10-WGP-NEXT:    s_endpgm
860;
861; GFX10-CU-LABEL: flat_agent_release_atomicrmw:
862; GFX10-CU:       ; %bb.0: ; %entry
863; GFX10-CU-NEXT:    s_clause 0x1
864; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
865; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
866; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
867; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
868; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
869; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
870; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
871; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
872; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
873; GFX10-CU-NEXT:    s_endpgm
874;
875; SKIP-CACHE-INV-LABEL: flat_agent_release_atomicrmw:
876; SKIP-CACHE-INV:       ; %bb.0: ; %entry
877; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
878; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
879; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
880; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
881; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
882; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
883; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
884; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
885; SKIP-CACHE-INV-NEXT:    s_endpgm
886;
887; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_atomicrmw:
888; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
889; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
890; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
891; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
892; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
893; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
894; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
895; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
896; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
897;
898; GFX90A-TGSPLIT-LABEL: flat_agent_release_atomicrmw:
899; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
900; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
901; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
902; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
903; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
904; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
905; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
906; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
907; GFX90A-TGSPLIT-NEXT:    s_endpgm
908    i32* %out, i32 %in) {
909entry:
910  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") release
911  ret void
912}
913
914define amdgpu_kernel void @flat_agent_acq_rel_atomicrmw(
915; GFX7-LABEL: flat_agent_acq_rel_atomicrmw:
916; GFX7:       ; %bb.0: ; %entry
917; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
918; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
919; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
920; GFX7-NEXT:    v_mov_b32_e32 v0, s0
921; GFX7-NEXT:    v_mov_b32_e32 v1, s1
922; GFX7-NEXT:    v_mov_b32_e32 v2, s2
923; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
924; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
925; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
926; GFX7-NEXT:    buffer_wbinvl1_vol
927; GFX7-NEXT:    s_endpgm
928;
929; GFX10-WGP-LABEL: flat_agent_acq_rel_atomicrmw:
930; GFX10-WGP:       ; %bb.0: ; %entry
931; GFX10-WGP-NEXT:    s_clause 0x1
932; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
933; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
934; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
935; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
936; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
937; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
938; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
939; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
940; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
941; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
942; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
943; GFX10-WGP-NEXT:    buffer_gl0_inv
944; GFX10-WGP-NEXT:    buffer_gl1_inv
945; GFX10-WGP-NEXT:    s_endpgm
946;
947; GFX10-CU-LABEL: flat_agent_acq_rel_atomicrmw:
948; GFX10-CU:       ; %bb.0: ; %entry
949; GFX10-CU-NEXT:    s_clause 0x1
950; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
951; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
952; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
953; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
954; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
955; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
956; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
957; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
958; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
959; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
960; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
961; GFX10-CU-NEXT:    buffer_gl0_inv
962; GFX10-CU-NEXT:    buffer_gl1_inv
963; GFX10-CU-NEXT:    s_endpgm
964;
965; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_atomicrmw:
966; SKIP-CACHE-INV:       ; %bb.0: ; %entry
967; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
968; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
969; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
970; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
971; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
972; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
973; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
974; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
975; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
976; SKIP-CACHE-INV-NEXT:    s_endpgm
977;
978; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_atomicrmw:
979; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
980; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
981; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
982; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
983; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
984; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
985; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
986; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
987; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
988; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
989; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
990;
991; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_atomicrmw:
992; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
993; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
994; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
995; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
996; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
997; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
998; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
999; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1000; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1001; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1002; GFX90A-TGSPLIT-NEXT:    s_endpgm
1003    i32* %out, i32 %in) {
1004entry:
1005  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") acq_rel
1006  ret void
1007}
1008
1009define amdgpu_kernel void @flat_agent_seq_cst_atomicrmw(
1010; GFX7-LABEL: flat_agent_seq_cst_atomicrmw:
1011; GFX7:       ; %bb.0: ; %entry
1012; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1013; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1014; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1015; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1016; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1017; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1018; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1019; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1020; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1021; GFX7-NEXT:    buffer_wbinvl1_vol
1022; GFX7-NEXT:    s_endpgm
1023;
1024; GFX10-WGP-LABEL: flat_agent_seq_cst_atomicrmw:
1025; GFX10-WGP:       ; %bb.0: ; %entry
1026; GFX10-WGP-NEXT:    s_clause 0x1
1027; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1028; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1029; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1030; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1031; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1032; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1033; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1034; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1035; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1036; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1037; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1038; GFX10-WGP-NEXT:    buffer_gl0_inv
1039; GFX10-WGP-NEXT:    buffer_gl1_inv
1040; GFX10-WGP-NEXT:    s_endpgm
1041;
1042; GFX10-CU-LABEL: flat_agent_seq_cst_atomicrmw:
1043; GFX10-CU:       ; %bb.0: ; %entry
1044; GFX10-CU-NEXT:    s_clause 0x1
1045; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1046; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1047; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1048; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1049; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1050; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1051; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1052; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1053; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1054; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1055; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1056; GFX10-CU-NEXT:    buffer_gl0_inv
1057; GFX10-CU-NEXT:    buffer_gl1_inv
1058; GFX10-CU-NEXT:    s_endpgm
1059;
1060; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_atomicrmw:
1061; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1062; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1063; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1064; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1065; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1066; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1067; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1068; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1069; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1070; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1071; SKIP-CACHE-INV-NEXT:    s_endpgm
1072;
1073; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_atomicrmw:
1074; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1075; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1076; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1077; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1078; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1079; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1080; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1081; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1082; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1083; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1084; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1085;
1086; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_atomicrmw:
1087; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1088; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1089; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1090; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1091; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1092; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1093; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1094; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1095; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1096; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1097; GFX90A-TGSPLIT-NEXT:    s_endpgm
1098    i32* %out, i32 %in) {
1099entry:
1100  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") seq_cst
1101  ret void
1102}
1103
1104define amdgpu_kernel void @flat_agent_acquire_ret_atomicrmw(
1105; GFX7-LABEL: flat_agent_acquire_ret_atomicrmw:
1106; GFX7:       ; %bb.0: ; %entry
1107; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1108; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1109; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1110; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1111; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1112; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1113; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1114; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1115; GFX7-NEXT:    buffer_wbinvl1_vol
1116; GFX7-NEXT:    flat_store_dword v[0:1], v2
1117; GFX7-NEXT:    s_endpgm
1118;
1119; GFX10-WGP-LABEL: flat_agent_acquire_ret_atomicrmw:
1120; GFX10-WGP:       ; %bb.0: ; %entry
1121; GFX10-WGP-NEXT:    s_clause 0x1
1122; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1123; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1124; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1125; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1126; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1127; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1128; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1129; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1130; GFX10-WGP-NEXT:    buffer_gl0_inv
1131; GFX10-WGP-NEXT:    buffer_gl1_inv
1132; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1133; GFX10-WGP-NEXT:    s_endpgm
1134;
1135; GFX10-CU-LABEL: flat_agent_acquire_ret_atomicrmw:
1136; GFX10-CU:       ; %bb.0: ; %entry
1137; GFX10-CU-NEXT:    s_clause 0x1
1138; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1139; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1140; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1141; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1142; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1143; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1144; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1145; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1146; GFX10-CU-NEXT:    buffer_gl0_inv
1147; GFX10-CU-NEXT:    buffer_gl1_inv
1148; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1149; GFX10-CU-NEXT:    s_endpgm
1150;
1151; SKIP-CACHE-INV-LABEL: flat_agent_acquire_ret_atomicrmw:
1152; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1153; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1154; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1155; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1156; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1157; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1158; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1159; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1160; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1161; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1162; SKIP-CACHE-INV-NEXT:    s_endpgm
1163;
1164; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_ret_atomicrmw:
1165; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1166; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1167; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1168; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1169; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1170; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1171; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1172; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1173; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1174; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1175; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1176;
1177; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_ret_atomicrmw:
1178; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1179; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1180; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1181; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1182; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1183; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1184; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1185; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1186; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1187; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1188; GFX90A-TGSPLIT-NEXT:    s_endpgm
1189    i32* %out, i32 %in) {
1190entry:
1191  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") acquire
1192  store i32 %val, i32* %out, align 4
1193  ret void
1194}
1195
1196define amdgpu_kernel void @flat_agent_acq_rel_ret_atomicrmw(
1197; GFX7-LABEL: flat_agent_acq_rel_ret_atomicrmw:
1198; GFX7:       ; %bb.0: ; %entry
1199; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1200; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1201; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1202; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1203; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1204; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1205; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1206; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1207; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1208; GFX7-NEXT:    buffer_wbinvl1_vol
1209; GFX7-NEXT:    flat_store_dword v[0:1], v2
1210; GFX7-NEXT:    s_endpgm
1211;
1212; GFX10-WGP-LABEL: flat_agent_acq_rel_ret_atomicrmw:
1213; GFX10-WGP:       ; %bb.0: ; %entry
1214; GFX10-WGP-NEXT:    s_clause 0x1
1215; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1216; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1217; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1218; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1219; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1220; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1221; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1222; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1223; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1224; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1225; GFX10-WGP-NEXT:    buffer_gl0_inv
1226; GFX10-WGP-NEXT:    buffer_gl1_inv
1227; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1228; GFX10-WGP-NEXT:    s_endpgm
1229;
1230; GFX10-CU-LABEL: flat_agent_acq_rel_ret_atomicrmw:
1231; GFX10-CU:       ; %bb.0: ; %entry
1232; GFX10-CU-NEXT:    s_clause 0x1
1233; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1234; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1235; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1236; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1237; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1238; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1239; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1240; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1241; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1242; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1243; GFX10-CU-NEXT:    buffer_gl0_inv
1244; GFX10-CU-NEXT:    buffer_gl1_inv
1245; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1246; GFX10-CU-NEXT:    s_endpgm
1247;
1248; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_ret_atomicrmw:
1249; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1250; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1251; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1252; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1253; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1254; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1255; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1256; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1257; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1258; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1259; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1260; SKIP-CACHE-INV-NEXT:    s_endpgm
1261;
1262; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_ret_atomicrmw:
1263; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1264; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1265; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1266; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1267; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1268; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1269; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1270; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1271; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1272; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1273; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1274; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1275;
1276; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_ret_atomicrmw:
1277; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1278; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1279; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1280; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1281; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1282; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1283; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1284; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1285; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1286; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1287; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1288; GFX90A-TGSPLIT-NEXT:    s_endpgm
1289    i32* %out, i32 %in) {
1290entry:
1291  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") acq_rel
1292  store i32 %val, i32* %out, align 4
1293  ret void
1294}
1295
1296define amdgpu_kernel void @flat_agent_seq_cst_ret_atomicrmw(
1297; GFX7-LABEL: flat_agent_seq_cst_ret_atomicrmw:
1298; GFX7:       ; %bb.0: ; %entry
1299; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1300; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1301; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1302; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1303; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1304; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1305; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1306; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1307; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1308; GFX7-NEXT:    buffer_wbinvl1_vol
1309; GFX7-NEXT:    flat_store_dword v[0:1], v2
1310; GFX7-NEXT:    s_endpgm
1311;
1312; GFX10-WGP-LABEL: flat_agent_seq_cst_ret_atomicrmw:
1313; GFX10-WGP:       ; %bb.0: ; %entry
1314; GFX10-WGP-NEXT:    s_clause 0x1
1315; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1316; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1317; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1318; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1319; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1320; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1321; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1322; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1323; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1324; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1325; GFX10-WGP-NEXT:    buffer_gl0_inv
1326; GFX10-WGP-NEXT:    buffer_gl1_inv
1327; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1328; GFX10-WGP-NEXT:    s_endpgm
1329;
1330; GFX10-CU-LABEL: flat_agent_seq_cst_ret_atomicrmw:
1331; GFX10-CU:       ; %bb.0: ; %entry
1332; GFX10-CU-NEXT:    s_clause 0x1
1333; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1334; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1335; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1336; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1337; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1338; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1339; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1340; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1341; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1342; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1343; GFX10-CU-NEXT:    buffer_gl0_inv
1344; GFX10-CU-NEXT:    buffer_gl1_inv
1345; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1346; GFX10-CU-NEXT:    s_endpgm
1347;
1348; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_ret_atomicrmw:
1349; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1350; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1351; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1352; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1353; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1354; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1355; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1356; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1357; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1358; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1359; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1360; SKIP-CACHE-INV-NEXT:    s_endpgm
1361;
1362; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_ret_atomicrmw:
1363; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1364; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1365; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1366; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1367; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1368; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1369; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1370; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1371; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1372; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1373; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1374; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1375;
1376; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_ret_atomicrmw:
1377; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1378; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1379; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1380; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1381; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1382; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1383; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1384; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1385; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1386; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1387; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1388; GFX90A-TGSPLIT-NEXT:    s_endpgm
1389    i32* %out, i32 %in) {
1390entry:
1391  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") seq_cst
1392  store i32 %val, i32* %out, align 4
1393  ret void
1394}
1395
1396define amdgpu_kernel void @flat_agent_monotonic_monotonic_cmpxchg(
1397; GFX7-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
1398; GFX7:       ; %bb.0: ; %entry
1399; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1400; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1401; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1402; GFX7-NEXT:    s_add_u32 s0, s0, 16
1403; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1404; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1405; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1406; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1407; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1408; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1409; GFX7-NEXT:    s_endpgm
1410;
1411; GFX10-WGP-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
1412; GFX10-WGP:       ; %bb.0: ; %entry
1413; GFX10-WGP-NEXT:    s_clause 0x1
1414; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1415; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1416; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1417; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1418; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1419; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1420; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1421; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1422; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1423; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1424; GFX10-WGP-NEXT:    s_endpgm
1425;
1426; GFX10-CU-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
1427; GFX10-CU:       ; %bb.0: ; %entry
1428; GFX10-CU-NEXT:    s_clause 0x1
1429; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1430; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1431; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1432; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1433; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1434; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1435; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1436; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1437; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1438; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1439; GFX10-CU-NEXT:    s_endpgm
1440;
1441; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
1442; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1443; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1444; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
1445; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1446; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1447; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1448; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1449; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1450; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1451; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1452; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1453; SKIP-CACHE-INV-NEXT:    s_endpgm
1454;
1455; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
1456; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1457; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1458; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1459; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1460; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1461; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1462; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1463; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1464;
1465; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
1466; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1467; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1468; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1469; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1470; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1471; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1472; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1473; GFX90A-TGSPLIT-NEXT:    s_endpgm
1474    i32* %out, i32 %in, i32 %old) {
1475entry:
1476  %gep = getelementptr i32, i32* %out, i32 4
1477  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") monotonic monotonic
1478  ret void
1479}
1480
1481define amdgpu_kernel void @flat_agent_acquire_monotonic_cmpxchg(
1482; GFX7-LABEL: flat_agent_acquire_monotonic_cmpxchg:
1483; GFX7:       ; %bb.0: ; %entry
1484; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1485; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1486; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1487; GFX7-NEXT:    s_add_u32 s0, s0, 16
1488; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1489; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1490; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1491; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1492; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1493; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1494; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1495; GFX7-NEXT:    buffer_wbinvl1_vol
1496; GFX7-NEXT:    s_endpgm
1497;
1498; GFX10-WGP-LABEL: flat_agent_acquire_monotonic_cmpxchg:
1499; GFX10-WGP:       ; %bb.0: ; %entry
1500; GFX10-WGP-NEXT:    s_clause 0x1
1501; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1502; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1503; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1504; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1505; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1506; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1507; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1508; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1509; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1510; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1511; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1512; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1513; GFX10-WGP-NEXT:    buffer_gl0_inv
1514; GFX10-WGP-NEXT:    buffer_gl1_inv
1515; GFX10-WGP-NEXT:    s_endpgm
1516;
1517; GFX10-CU-LABEL: flat_agent_acquire_monotonic_cmpxchg:
1518; GFX10-CU:       ; %bb.0: ; %entry
1519; GFX10-CU-NEXT:    s_clause 0x1
1520; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1521; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1522; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1523; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1524; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1525; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1526; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1527; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1528; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1529; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1530; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1531; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1532; GFX10-CU-NEXT:    buffer_gl0_inv
1533; GFX10-CU-NEXT:    buffer_gl1_inv
1534; GFX10-CU-NEXT:    s_endpgm
1535;
1536; SKIP-CACHE-INV-LABEL: flat_agent_acquire_monotonic_cmpxchg:
1537; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1538; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1539; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
1540; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1541; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1542; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1543; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1544; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1545; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1546; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1547; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1548; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1549; SKIP-CACHE-INV-NEXT:    s_endpgm
1550;
1551; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_monotonic_cmpxchg:
1552; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1553; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1554; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1555; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1556; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1557; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1558; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1559; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1560; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1561; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1562;
1563; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_monotonic_cmpxchg:
1564; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1565; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1566; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1567; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1568; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1569; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1570; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1571; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1572; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1573; GFX90A-TGSPLIT-NEXT:    s_endpgm
1574    i32* %out, i32 %in, i32 %old) {
1575entry:
1576  %gep = getelementptr i32, i32* %out, i32 4
1577  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire monotonic
1578  ret void
1579}
1580
1581define amdgpu_kernel void @flat_agent_release_monotonic_cmpxchg(
1582; GFX7-LABEL: flat_agent_release_monotonic_cmpxchg:
1583; GFX7:       ; %bb.0: ; %entry
1584; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1585; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1586; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1587; GFX7-NEXT:    s_add_u32 s0, s0, 16
1588; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1589; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1590; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1591; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1592; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1593; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1594; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1595; GFX7-NEXT:    s_endpgm
1596;
1597; GFX10-WGP-LABEL: flat_agent_release_monotonic_cmpxchg:
1598; GFX10-WGP:       ; %bb.0: ; %entry
1599; GFX10-WGP-NEXT:    s_clause 0x1
1600; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1601; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1602; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1603; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1604; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1605; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1606; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1607; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1608; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1609; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1610; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1611; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1612; GFX10-WGP-NEXT:    s_endpgm
1613;
1614; GFX10-CU-LABEL: flat_agent_release_monotonic_cmpxchg:
1615; GFX10-CU:       ; %bb.0: ; %entry
1616; GFX10-CU-NEXT:    s_clause 0x1
1617; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1618; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1619; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1620; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1621; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1622; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1623; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1624; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1625; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1626; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1627; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1628; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1629; GFX10-CU-NEXT:    s_endpgm
1630;
1631; SKIP-CACHE-INV-LABEL: flat_agent_release_monotonic_cmpxchg:
1632; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1633; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1634; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
1635; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1636; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1637; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1638; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1639; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1640; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1641; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1642; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1643; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1644; SKIP-CACHE-INV-NEXT:    s_endpgm
1645;
1646; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_monotonic_cmpxchg:
1647; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1648; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1649; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1650; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1651; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1652; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1653; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1654; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1655; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1656;
1657; GFX90A-TGSPLIT-LABEL: flat_agent_release_monotonic_cmpxchg:
1658; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1659; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1660; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1661; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1662; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1663; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1664; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1665; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1666; GFX90A-TGSPLIT-NEXT:    s_endpgm
1667    i32* %out, i32 %in, i32 %old) {
1668entry:
1669  %gep = getelementptr i32, i32* %out, i32 4
1670  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release monotonic
1671  ret void
1672}
1673
1674define amdgpu_kernel void @flat_agent_acq_rel_monotonic_cmpxchg(
1675; GFX7-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
1676; GFX7:       ; %bb.0: ; %entry
1677; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1678; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1679; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1680; GFX7-NEXT:    s_add_u32 s0, s0, 16
1681; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1682; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1683; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1684; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1685; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1686; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1687; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1688; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1689; GFX7-NEXT:    buffer_wbinvl1_vol
1690; GFX7-NEXT:    s_endpgm
1691;
1692; GFX10-WGP-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
1693; GFX10-WGP:       ; %bb.0: ; %entry
1694; GFX10-WGP-NEXT:    s_clause 0x1
1695; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1696; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1697; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1698; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1699; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1700; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1701; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1702; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1703; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1704; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1705; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1706; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1707; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1708; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1709; GFX10-WGP-NEXT:    buffer_gl0_inv
1710; GFX10-WGP-NEXT:    buffer_gl1_inv
1711; GFX10-WGP-NEXT:    s_endpgm
1712;
1713; GFX10-CU-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
1714; GFX10-CU:       ; %bb.0: ; %entry
1715; GFX10-CU-NEXT:    s_clause 0x1
1716; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1717; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1718; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1719; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1720; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1721; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1722; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1723; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1724; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1725; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1726; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1727; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1728; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1729; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1730; GFX10-CU-NEXT:    buffer_gl0_inv
1731; GFX10-CU-NEXT:    buffer_gl1_inv
1732; GFX10-CU-NEXT:    s_endpgm
1733;
1734; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
1735; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1736; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1737; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
1738; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1739; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1740; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1741; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1742; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1743; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1744; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1745; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1746; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1747; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1748; SKIP-CACHE-INV-NEXT:    s_endpgm
1749;
1750; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
1751; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1752; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1753; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1754; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1755; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1756; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1757; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1758; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1759; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1760; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1761; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1762;
1763; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
1764; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1765; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1766; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1767; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1768; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1769; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1770; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1771; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1772; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1773; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1774; GFX90A-TGSPLIT-NEXT:    s_endpgm
1775    i32* %out, i32 %in, i32 %old) {
1776entry:
1777  %gep = getelementptr i32, i32* %out, i32 4
1778  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel monotonic
1779  ret void
1780}
1781
1782define amdgpu_kernel void @flat_agent_seq_cst_monotonic_cmpxchg(
1783; GFX7-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
1784; GFX7:       ; %bb.0: ; %entry
1785; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1786; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1787; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1788; GFX7-NEXT:    s_add_u32 s0, s0, 16
1789; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1790; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1791; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1792; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1793; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1794; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1795; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1796; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1797; GFX7-NEXT:    buffer_wbinvl1_vol
1798; GFX7-NEXT:    s_endpgm
1799;
1800; GFX10-WGP-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
1801; GFX10-WGP:       ; %bb.0: ; %entry
1802; GFX10-WGP-NEXT:    s_clause 0x1
1803; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1804; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1805; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1806; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1807; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1808; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1809; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1810; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1811; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1812; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1813; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1814; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1815; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1816; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1817; GFX10-WGP-NEXT:    buffer_gl0_inv
1818; GFX10-WGP-NEXT:    buffer_gl1_inv
1819; GFX10-WGP-NEXT:    s_endpgm
1820;
1821; GFX10-CU-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
1822; GFX10-CU:       ; %bb.0: ; %entry
1823; GFX10-CU-NEXT:    s_clause 0x1
1824; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1825; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1826; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1827; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1828; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1829; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1830; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1831; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1832; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1833; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1834; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1835; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1836; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1837; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1838; GFX10-CU-NEXT:    buffer_gl0_inv
1839; GFX10-CU-NEXT:    buffer_gl1_inv
1840; GFX10-CU-NEXT:    s_endpgm
1841;
1842; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
1843; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1844; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1845; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
1846; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1847; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1848; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1849; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1850; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1851; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1852; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1853; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1854; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1855; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1856; SKIP-CACHE-INV-NEXT:    s_endpgm
1857;
1858; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
1859; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1860; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1861; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1862; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1863; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1864; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1865; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1866; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1867; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1868; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1869; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1870;
1871; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
1872; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1873; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1874; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1875; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1876; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1877; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1878; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1879; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1880; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1881; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1882; GFX90A-TGSPLIT-NEXT:    s_endpgm
1883    i32* %out, i32 %in, i32 %old) {
1884entry:
1885  %gep = getelementptr i32, i32* %out, i32 4
1886  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst monotonic
1887  ret void
1888}
1889
1890define amdgpu_kernel void @flat_agent_monotonic_acquire_cmpxchg(
1891; GFX7-LABEL: flat_agent_monotonic_acquire_cmpxchg:
1892; GFX7:       ; %bb.0: ; %entry
1893; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1894; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1895; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1896; GFX7-NEXT:    s_add_u32 s0, s0, 16
1897; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1898; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1899; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1900; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1901; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1902; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1903; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1904; GFX7-NEXT:    buffer_wbinvl1_vol
1905; GFX7-NEXT:    s_endpgm
1906;
1907; GFX10-WGP-LABEL: flat_agent_monotonic_acquire_cmpxchg:
1908; GFX10-WGP:       ; %bb.0: ; %entry
1909; GFX10-WGP-NEXT:    s_clause 0x1
1910; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1911; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1912; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1913; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1914; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1915; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1916; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1917; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1918; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1919; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1920; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1921; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1922; GFX10-WGP-NEXT:    buffer_gl0_inv
1923; GFX10-WGP-NEXT:    buffer_gl1_inv
1924; GFX10-WGP-NEXT:    s_endpgm
1925;
1926; GFX10-CU-LABEL: flat_agent_monotonic_acquire_cmpxchg:
1927; GFX10-CU:       ; %bb.0: ; %entry
1928; GFX10-CU-NEXT:    s_clause 0x1
1929; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1930; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1931; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1932; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1933; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1934; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1935; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1936; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1937; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1938; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1939; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1940; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1941; GFX10-CU-NEXT:    buffer_gl0_inv
1942; GFX10-CU-NEXT:    buffer_gl1_inv
1943; GFX10-CU-NEXT:    s_endpgm
1944;
1945; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_acquire_cmpxchg:
1946; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1947; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1948; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
1949; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1950; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1951; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1952; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1953; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1954; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1955; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1956; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1957; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1958; SKIP-CACHE-INV-NEXT:    s_endpgm
1959;
1960; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_acquire_cmpxchg:
1961; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1962; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1963; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1964; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1965; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1966; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1967; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1968; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1969; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1970; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1971;
1972; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_acquire_cmpxchg:
1973; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1974; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1975; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1976; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1977; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1978; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1979; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1980; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1981; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1982; GFX90A-TGSPLIT-NEXT:    s_endpgm
1983    i32* %out, i32 %in, i32 %old) {
1984entry:
1985  %gep = getelementptr i32, i32* %out, i32 4
1986  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") monotonic acquire
1987  ret void
1988}
1989
1990define amdgpu_kernel void @flat_agent_acquire_acquire_cmpxchg(
1991; GFX7-LABEL: flat_agent_acquire_acquire_cmpxchg:
1992; GFX7:       ; %bb.0: ; %entry
1993; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1994; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1995; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1996; GFX7-NEXT:    s_add_u32 s0, s0, 16
1997; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1998; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1999; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2000; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2001; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2002; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2003; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2004; GFX7-NEXT:    buffer_wbinvl1_vol
2005; GFX7-NEXT:    s_endpgm
2006;
2007; GFX10-WGP-LABEL: flat_agent_acquire_acquire_cmpxchg:
2008; GFX10-WGP:       ; %bb.0: ; %entry
2009; GFX10-WGP-NEXT:    s_clause 0x1
2010; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2011; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2012; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2013; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2014; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2015; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2016; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2017; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2018; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2019; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2020; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2021; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2022; GFX10-WGP-NEXT:    buffer_gl0_inv
2023; GFX10-WGP-NEXT:    buffer_gl1_inv
2024; GFX10-WGP-NEXT:    s_endpgm
2025;
2026; GFX10-CU-LABEL: flat_agent_acquire_acquire_cmpxchg:
2027; GFX10-CU:       ; %bb.0: ; %entry
2028; GFX10-CU-NEXT:    s_clause 0x1
2029; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2030; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2031; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2032; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2033; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2034; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2035; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2036; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2037; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2038; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2039; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2040; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2041; GFX10-CU-NEXT:    buffer_gl0_inv
2042; GFX10-CU-NEXT:    buffer_gl1_inv
2043; GFX10-CU-NEXT:    s_endpgm
2044;
2045; SKIP-CACHE-INV-LABEL: flat_agent_acquire_acquire_cmpxchg:
2046; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2047; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2048; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2049; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2050; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2051; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2052; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2053; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2054; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2055; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2056; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2057; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2058; SKIP-CACHE-INV-NEXT:    s_endpgm
2059;
2060; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_acquire_cmpxchg:
2061; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2062; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2063; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2064; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2065; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2066; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2067; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2068; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2069; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2070; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2071;
2072; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_acquire_cmpxchg:
2073; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2074; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2075; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2076; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2077; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2078; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2079; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2080; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2081; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2082; GFX90A-TGSPLIT-NEXT:    s_endpgm
2083    i32* %out, i32 %in, i32 %old) {
2084entry:
2085  %gep = getelementptr i32, i32* %out, i32 4
2086  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire acquire
2087  ret void
2088}
2089
2090define amdgpu_kernel void @flat_agent_release_acquire_cmpxchg(
2091; GFX7-LABEL: flat_agent_release_acquire_cmpxchg:
2092; GFX7:       ; %bb.0: ; %entry
2093; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2094; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2095; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2096; GFX7-NEXT:    s_add_u32 s0, s0, 16
2097; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2098; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2099; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2100; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2101; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2102; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2103; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2104; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2105; GFX7-NEXT:    buffer_wbinvl1_vol
2106; GFX7-NEXT:    s_endpgm
2107;
2108; GFX10-WGP-LABEL: flat_agent_release_acquire_cmpxchg:
2109; GFX10-WGP:       ; %bb.0: ; %entry
2110; GFX10-WGP-NEXT:    s_clause 0x1
2111; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2112; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2113; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2114; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2115; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2116; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2117; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2118; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2119; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2120; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2121; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2122; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2123; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2124; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2125; GFX10-WGP-NEXT:    buffer_gl0_inv
2126; GFX10-WGP-NEXT:    buffer_gl1_inv
2127; GFX10-WGP-NEXT:    s_endpgm
2128;
2129; GFX10-CU-LABEL: flat_agent_release_acquire_cmpxchg:
2130; GFX10-CU:       ; %bb.0: ; %entry
2131; GFX10-CU-NEXT:    s_clause 0x1
2132; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2133; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2134; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2135; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2136; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2137; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2138; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2139; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2140; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2141; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2142; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2143; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2144; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2145; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2146; GFX10-CU-NEXT:    buffer_gl0_inv
2147; GFX10-CU-NEXT:    buffer_gl1_inv
2148; GFX10-CU-NEXT:    s_endpgm
2149;
2150; SKIP-CACHE-INV-LABEL: flat_agent_release_acquire_cmpxchg:
2151; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2152; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2153; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2154; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2155; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2156; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2157; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2158; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2159; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2160; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2161; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2162; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2163; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2164; SKIP-CACHE-INV-NEXT:    s_endpgm
2165;
2166; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_acquire_cmpxchg:
2167; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2168; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2169; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2170; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2171; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2172; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2173; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2174; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2175; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2176; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2177; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2178;
2179; GFX90A-TGSPLIT-LABEL: flat_agent_release_acquire_cmpxchg:
2180; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2181; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2182; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2183; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2184; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2185; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2186; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2187; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2188; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2189; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2190; GFX90A-TGSPLIT-NEXT:    s_endpgm
2191    i32* %out, i32 %in, i32 %old) {
2192entry:
2193  %gep = getelementptr i32, i32* %out, i32 4
2194  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release acquire
2195  ret void
2196}
2197
2198define amdgpu_kernel void @flat_agent_acq_rel_acquire_cmpxchg(
2199; GFX7-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
2200; GFX7:       ; %bb.0: ; %entry
2201; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2202; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2203; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2204; GFX7-NEXT:    s_add_u32 s0, s0, 16
2205; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2206; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2207; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2208; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2209; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2210; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2211; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2212; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2213; GFX7-NEXT:    buffer_wbinvl1_vol
2214; GFX7-NEXT:    s_endpgm
2215;
2216; GFX10-WGP-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
2217; GFX10-WGP:       ; %bb.0: ; %entry
2218; GFX10-WGP-NEXT:    s_clause 0x1
2219; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2220; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2221; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2222; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2223; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2224; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2225; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2226; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2227; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2228; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2229; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2230; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2231; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2232; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2233; GFX10-WGP-NEXT:    buffer_gl0_inv
2234; GFX10-WGP-NEXT:    buffer_gl1_inv
2235; GFX10-WGP-NEXT:    s_endpgm
2236;
2237; GFX10-CU-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
2238; GFX10-CU:       ; %bb.0: ; %entry
2239; GFX10-CU-NEXT:    s_clause 0x1
2240; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2241; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2242; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2243; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2244; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2245; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2246; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2247; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2248; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2249; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2250; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2251; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2252; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2253; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2254; GFX10-CU-NEXT:    buffer_gl0_inv
2255; GFX10-CU-NEXT:    buffer_gl1_inv
2256; GFX10-CU-NEXT:    s_endpgm
2257;
2258; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
2259; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2260; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2261; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2262; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2263; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2264; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2265; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2266; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2267; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2268; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2269; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2270; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2271; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2272; SKIP-CACHE-INV-NEXT:    s_endpgm
2273;
2274; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
2275; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2276; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2277; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2278; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2279; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2280; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2281; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2282; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2283; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2284; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2285; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2286;
2287; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
2288; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2289; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2290; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2291; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2292; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2293; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2294; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2295; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2296; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2297; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2298; GFX90A-TGSPLIT-NEXT:    s_endpgm
2299    i32* %out, i32 %in, i32 %old) {
2300entry:
2301  %gep = getelementptr i32, i32* %out, i32 4
2302  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel acquire
2303  ret void
2304}
2305
2306define amdgpu_kernel void @flat_agent_seq_cst_acquire_cmpxchg(
2307; GFX7-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
2308; GFX7:       ; %bb.0: ; %entry
2309; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2310; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2311; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2312; GFX7-NEXT:    s_add_u32 s0, s0, 16
2313; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2314; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2315; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2316; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2317; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2318; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2319; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2320; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2321; GFX7-NEXT:    buffer_wbinvl1_vol
2322; GFX7-NEXT:    s_endpgm
2323;
2324; GFX10-WGP-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
2325; GFX10-WGP:       ; %bb.0: ; %entry
2326; GFX10-WGP-NEXT:    s_clause 0x1
2327; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2328; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2329; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2330; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2331; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2332; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2333; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2334; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2335; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2336; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2337; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2338; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2339; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2340; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2341; GFX10-WGP-NEXT:    buffer_gl0_inv
2342; GFX10-WGP-NEXT:    buffer_gl1_inv
2343; GFX10-WGP-NEXT:    s_endpgm
2344;
2345; GFX10-CU-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
2346; GFX10-CU:       ; %bb.0: ; %entry
2347; GFX10-CU-NEXT:    s_clause 0x1
2348; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2349; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2350; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2351; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2352; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2353; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2354; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2355; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2356; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2357; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2358; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2359; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2360; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2361; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2362; GFX10-CU-NEXT:    buffer_gl0_inv
2363; GFX10-CU-NEXT:    buffer_gl1_inv
2364; GFX10-CU-NEXT:    s_endpgm
2365;
2366; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
2367; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2368; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2369; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2370; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2371; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2372; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2373; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2374; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2375; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2376; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2377; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2378; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2379; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2380; SKIP-CACHE-INV-NEXT:    s_endpgm
2381;
2382; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
2383; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2384; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2385; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2386; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2387; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2388; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2389; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2390; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2391; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2392; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2393; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2394;
2395; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
2396; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2397; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2398; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2399; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2400; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2401; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2402; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2403; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2404; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2405; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2406; GFX90A-TGSPLIT-NEXT:    s_endpgm
2407    i32* %out, i32 %in, i32 %old) {
2408entry:
2409  %gep = getelementptr i32, i32* %out, i32 4
2410  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst acquire
2411  ret void
2412}
2413
2414define amdgpu_kernel void @flat_agent_monotonic_seq_cst_cmpxchg(
2415; GFX7-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
2416; GFX7:       ; %bb.0: ; %entry
2417; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2418; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2419; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2420; GFX7-NEXT:    s_add_u32 s0, s0, 16
2421; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2422; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2423; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2424; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2425; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2426; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2427; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2428; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2429; GFX7-NEXT:    buffer_wbinvl1_vol
2430; GFX7-NEXT:    s_endpgm
2431;
2432; GFX10-WGP-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
2433; GFX10-WGP:       ; %bb.0: ; %entry
2434; GFX10-WGP-NEXT:    s_clause 0x1
2435; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2436; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2437; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2438; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2439; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2440; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2441; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2442; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2443; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2444; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2445; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2446; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2447; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2448; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2449; GFX10-WGP-NEXT:    buffer_gl0_inv
2450; GFX10-WGP-NEXT:    buffer_gl1_inv
2451; GFX10-WGP-NEXT:    s_endpgm
2452;
2453; GFX10-CU-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
2454; GFX10-CU:       ; %bb.0: ; %entry
2455; GFX10-CU-NEXT:    s_clause 0x1
2456; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2457; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2458; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2459; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2460; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2461; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2462; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2463; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2464; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2465; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2466; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2467; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2468; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2469; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2470; GFX10-CU-NEXT:    buffer_gl0_inv
2471; GFX10-CU-NEXT:    buffer_gl1_inv
2472; GFX10-CU-NEXT:    s_endpgm
2473;
2474; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
2475; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2476; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2477; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2478; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2479; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2480; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2481; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2482; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2483; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2484; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2485; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2486; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2487; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2488; SKIP-CACHE-INV-NEXT:    s_endpgm
2489;
2490; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
2491; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2492; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2493; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2494; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2495; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2496; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2497; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2498; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2499; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2500; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2501; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2502;
2503; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
2504; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2505; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2506; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2507; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2508; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2509; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2510; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2511; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2512; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2513; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2514; GFX90A-TGSPLIT-NEXT:    s_endpgm
2515    i32* %out, i32 %in, i32 %old) {
2516entry:
2517  %gep = getelementptr i32, i32* %out, i32 4
2518  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") monotonic seq_cst
2519  ret void
2520}
2521
2522define amdgpu_kernel void @flat_agent_acquire_seq_cst_cmpxchg(
2523; GFX7-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
2524; GFX7:       ; %bb.0: ; %entry
2525; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2526; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2527; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2528; GFX7-NEXT:    s_add_u32 s0, s0, 16
2529; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2530; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2531; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2532; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2533; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2534; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2535; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2536; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2537; GFX7-NEXT:    buffer_wbinvl1_vol
2538; GFX7-NEXT:    s_endpgm
2539;
2540; GFX10-WGP-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
2541; GFX10-WGP:       ; %bb.0: ; %entry
2542; GFX10-WGP-NEXT:    s_clause 0x1
2543; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2544; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2545; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2546; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2547; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2548; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2549; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2550; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2551; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2552; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2553; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2554; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2555; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2556; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2557; GFX10-WGP-NEXT:    buffer_gl0_inv
2558; GFX10-WGP-NEXT:    buffer_gl1_inv
2559; GFX10-WGP-NEXT:    s_endpgm
2560;
2561; GFX10-CU-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
2562; GFX10-CU:       ; %bb.0: ; %entry
2563; GFX10-CU-NEXT:    s_clause 0x1
2564; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2565; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2566; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2567; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2568; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2569; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2570; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2571; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2572; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2573; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2574; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2575; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2576; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2577; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2578; GFX10-CU-NEXT:    buffer_gl0_inv
2579; GFX10-CU-NEXT:    buffer_gl1_inv
2580; GFX10-CU-NEXT:    s_endpgm
2581;
2582; SKIP-CACHE-INV-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
2583; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2584; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2585; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2586; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2587; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2588; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2589; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2590; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2591; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2592; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2593; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2594; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2595; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2596; SKIP-CACHE-INV-NEXT:    s_endpgm
2597;
2598; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
2599; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2600; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2601; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2602; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2603; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2604; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2605; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2606; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2607; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2608; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2609; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2610;
2611; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
2612; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2613; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2614; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2615; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2616; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2617; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2618; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2619; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2620; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2621; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2622; GFX90A-TGSPLIT-NEXT:    s_endpgm
2623    i32* %out, i32 %in, i32 %old) {
2624entry:
2625  %gep = getelementptr i32, i32* %out, i32 4
2626  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire seq_cst
2627  ret void
2628}
2629
2630define amdgpu_kernel void @flat_agent_release_seq_cst_cmpxchg(
2631; GFX7-LABEL: flat_agent_release_seq_cst_cmpxchg:
2632; GFX7:       ; %bb.0: ; %entry
2633; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2634; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2635; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2636; GFX7-NEXT:    s_add_u32 s0, s0, 16
2637; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2638; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2639; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2640; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2641; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2642; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2643; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2644; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2645; GFX7-NEXT:    buffer_wbinvl1_vol
2646; GFX7-NEXT:    s_endpgm
2647;
2648; GFX10-WGP-LABEL: flat_agent_release_seq_cst_cmpxchg:
2649; GFX10-WGP:       ; %bb.0: ; %entry
2650; GFX10-WGP-NEXT:    s_clause 0x1
2651; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2652; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2653; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2654; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2655; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2656; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2657; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2658; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2659; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2660; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2661; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2662; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2663; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2664; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2665; GFX10-WGP-NEXT:    buffer_gl0_inv
2666; GFX10-WGP-NEXT:    buffer_gl1_inv
2667; GFX10-WGP-NEXT:    s_endpgm
2668;
2669; GFX10-CU-LABEL: flat_agent_release_seq_cst_cmpxchg:
2670; GFX10-CU:       ; %bb.0: ; %entry
2671; GFX10-CU-NEXT:    s_clause 0x1
2672; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2673; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2674; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2675; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2676; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2677; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2678; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2679; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2680; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2681; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2682; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2683; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2684; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2685; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2686; GFX10-CU-NEXT:    buffer_gl0_inv
2687; GFX10-CU-NEXT:    buffer_gl1_inv
2688; GFX10-CU-NEXT:    s_endpgm
2689;
2690; SKIP-CACHE-INV-LABEL: flat_agent_release_seq_cst_cmpxchg:
2691; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2692; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2693; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2694; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2695; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2696; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2697; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2698; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2699; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2700; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2701; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2702; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2703; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2704; SKIP-CACHE-INV-NEXT:    s_endpgm
2705;
2706; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_seq_cst_cmpxchg:
2707; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2708; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2709; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2710; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2711; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2712; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2713; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2714; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2715; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2716; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2717; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2718;
2719; GFX90A-TGSPLIT-LABEL: flat_agent_release_seq_cst_cmpxchg:
2720; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2721; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2722; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2723; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2724; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2725; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2726; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2727; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2728; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2729; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2730; GFX90A-TGSPLIT-NEXT:    s_endpgm
2731    i32* %out, i32 %in, i32 %old) {
2732entry:
2733  %gep = getelementptr i32, i32* %out, i32 4
2734  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release seq_cst
2735  ret void
2736}
2737
2738define amdgpu_kernel void @flat_agent_acq_rel_seq_cst_cmpxchg(
2739; GFX7-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
2740; GFX7:       ; %bb.0: ; %entry
2741; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2742; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2743; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2744; GFX7-NEXT:    s_add_u32 s0, s0, 16
2745; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2746; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2747; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2748; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2749; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2750; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2751; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2752; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2753; GFX7-NEXT:    buffer_wbinvl1_vol
2754; GFX7-NEXT:    s_endpgm
2755;
2756; GFX10-WGP-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
2757; GFX10-WGP:       ; %bb.0: ; %entry
2758; GFX10-WGP-NEXT:    s_clause 0x1
2759; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2760; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2761; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2762; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2763; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2764; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2765; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2766; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2767; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2768; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2769; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2770; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2771; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2772; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2773; GFX10-WGP-NEXT:    buffer_gl0_inv
2774; GFX10-WGP-NEXT:    buffer_gl1_inv
2775; GFX10-WGP-NEXT:    s_endpgm
2776;
2777; GFX10-CU-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
2778; GFX10-CU:       ; %bb.0: ; %entry
2779; GFX10-CU-NEXT:    s_clause 0x1
2780; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2781; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2782; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2783; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2784; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2785; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2786; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2787; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2788; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2789; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2790; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2791; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2792; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2793; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2794; GFX10-CU-NEXT:    buffer_gl0_inv
2795; GFX10-CU-NEXT:    buffer_gl1_inv
2796; GFX10-CU-NEXT:    s_endpgm
2797;
2798; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
2799; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2800; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2801; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2802; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2803; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2804; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2805; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2806; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2807; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2808; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2809; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2810; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2811; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2812; SKIP-CACHE-INV-NEXT:    s_endpgm
2813;
2814; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
2815; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2816; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2817; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2818; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2819; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2820; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2821; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2822; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2823; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2824; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2825; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2826;
2827; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
2828; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2829; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2830; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2831; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2832; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2833; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2834; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2835; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2836; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2837; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2838; GFX90A-TGSPLIT-NEXT:    s_endpgm
2839    i32* %out, i32 %in, i32 %old) {
2840entry:
2841  %gep = getelementptr i32, i32* %out, i32 4
2842  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel seq_cst
2843  ret void
2844}
2845
2846define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_cmpxchg(
2847; GFX7-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
2848; GFX7:       ; %bb.0: ; %entry
2849; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2850; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2851; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2852; GFX7-NEXT:    s_add_u32 s0, s0, 16
2853; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2854; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2855; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2856; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2857; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2858; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2859; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2860; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2861; GFX7-NEXT:    buffer_wbinvl1_vol
2862; GFX7-NEXT:    s_endpgm
2863;
2864; GFX10-WGP-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
2865; GFX10-WGP:       ; %bb.0: ; %entry
2866; GFX10-WGP-NEXT:    s_clause 0x1
2867; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2868; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2869; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2870; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2871; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2872; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2873; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2874; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2875; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2876; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2877; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2878; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2879; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2880; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2881; GFX10-WGP-NEXT:    buffer_gl0_inv
2882; GFX10-WGP-NEXT:    buffer_gl1_inv
2883; GFX10-WGP-NEXT:    s_endpgm
2884;
2885; GFX10-CU-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
2886; GFX10-CU:       ; %bb.0: ; %entry
2887; GFX10-CU-NEXT:    s_clause 0x1
2888; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2889; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2890; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2891; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2892; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2893; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2894; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2895; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2896; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2897; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2898; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2899; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2900; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2901; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2902; GFX10-CU-NEXT:    buffer_gl0_inv
2903; GFX10-CU-NEXT:    buffer_gl1_inv
2904; GFX10-CU-NEXT:    s_endpgm
2905;
2906; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
2907; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2908; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2909; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2910; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2911; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2912; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2913; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2914; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2915; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2916; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2917; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2918; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2919; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2920; SKIP-CACHE-INV-NEXT:    s_endpgm
2921;
2922; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
2923; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2924; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2925; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2926; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2927; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2928; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2929; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2930; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2931; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2932; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2933; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2934;
2935; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
2936; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2937; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2938; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2939; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2940; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2941; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2942; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2943; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2944; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2945; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2946; GFX90A-TGSPLIT-NEXT:    s_endpgm
2947    i32* %out, i32 %in, i32 %old) {
2948entry:
2949  %gep = getelementptr i32, i32* %out, i32 4
2950  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
2951  ret void
2952}
2953
2954define amdgpu_kernel void @flat_agent_monotonic_monotonic_ret_cmpxchg(
2955; GFX7-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
2956; GFX7:       ; %bb.0: ; %entry
2957; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2958; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2959; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2960; GFX7-NEXT:    s_add_u32 s4, s0, 16
2961; GFX7-NEXT:    s_addc_u32 s5, s1, 0
2962; GFX7-NEXT:    v_mov_b32_e32 v0, s4
2963; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2964; GFX7-NEXT:    v_mov_b32_e32 v1, s5
2965; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2966; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2967; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2968; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2969; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2970; GFX7-NEXT:    flat_store_dword v[0:1], v2
2971; GFX7-NEXT:    s_endpgm
2972;
2973; GFX10-WGP-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
2974; GFX10-WGP:       ; %bb.0: ; %entry
2975; GFX10-WGP-NEXT:    s_clause 0x1
2976; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2977; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2978; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2979; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
2980; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
2981; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
2982; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2983; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
2984; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2985; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2986; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2987; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2988; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2989; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
2990; GFX10-WGP-NEXT:    s_endpgm
2991;
2992; GFX10-CU-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
2993; GFX10-CU:       ; %bb.0: ; %entry
2994; GFX10-CU-NEXT:    s_clause 0x1
2995; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2996; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2997; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2998; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
2999; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3000; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3001; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3002; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3003; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3004; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3005; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3006; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3007; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3008; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3009; GFX10-CU-NEXT:    s_endpgm
3010;
3011; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
3012; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3013; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3014; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3015; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3016; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3017; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3018; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3019; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3020; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3021; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3022; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3023; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3024; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3025; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3026; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3027; SKIP-CACHE-INV-NEXT:    s_endpgm
3028;
3029; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
3030; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3031; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3032; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3033; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3034; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3035; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3036; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3037; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3038; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3039; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3040;
3041; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
3042; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3043; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3044; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3045; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3046; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3047; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3048; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3049; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3050; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3051; GFX90A-TGSPLIT-NEXT:    s_endpgm
3052    i32* %out, i32 %in, i32 %old) {
3053entry:
3054  %gep = getelementptr i32, i32* %out, i32 4
3055  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") monotonic monotonic
3056  %val0 = extractvalue { i32, i1 } %val, 0
3057  store i32 %val0, i32* %out, align 4
3058  ret void
3059}
3060
3061define amdgpu_kernel void @flat_agent_acquire_monotonic_ret_cmpxchg(
3062; GFX7-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
3063; GFX7:       ; %bb.0: ; %entry
3064; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3065; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3066; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3067; GFX7-NEXT:    s_add_u32 s4, s0, 16
3068; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3069; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3070; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3071; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3072; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3073; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3074; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3075; GFX7-NEXT:    buffer_wbinvl1_vol
3076; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3077; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3078; GFX7-NEXT:    flat_store_dword v[0:1], v2
3079; GFX7-NEXT:    s_endpgm
3080;
3081; GFX10-WGP-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
3082; GFX10-WGP:       ; %bb.0: ; %entry
3083; GFX10-WGP-NEXT:    s_clause 0x1
3084; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3085; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3086; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3087; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3088; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3089; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3090; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3091; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
3092; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3093; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3094; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3095; GFX10-WGP-NEXT:    buffer_gl0_inv
3096; GFX10-WGP-NEXT:    buffer_gl1_inv
3097; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3098; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3099; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3100; GFX10-WGP-NEXT:    s_endpgm
3101;
3102; GFX10-CU-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
3103; GFX10-CU:       ; %bb.0: ; %entry
3104; GFX10-CU-NEXT:    s_clause 0x1
3105; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3106; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3107; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3108; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3109; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3110; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3111; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3112; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3113; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3114; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3115; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3116; GFX10-CU-NEXT:    buffer_gl0_inv
3117; GFX10-CU-NEXT:    buffer_gl1_inv
3118; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3119; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3120; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3121; GFX10-CU-NEXT:    s_endpgm
3122;
3123; SKIP-CACHE-INV-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
3124; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3125; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3126; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3127; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3128; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3129; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3130; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3131; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3132; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3133; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3134; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3135; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3136; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3137; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3138; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3139; SKIP-CACHE-INV-NEXT:    s_endpgm
3140;
3141; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
3142; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3143; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3144; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3145; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3146; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3147; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3148; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3149; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3150; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3151; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3152; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3153;
3154; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
3155; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3156; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3157; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3158; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3159; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3160; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3161; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3162; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3163; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3164; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3165; GFX90A-TGSPLIT-NEXT:    s_endpgm
3166    i32* %out, i32 %in, i32 %old) {
3167entry:
3168  %gep = getelementptr i32, i32* %out, i32 4
3169  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire monotonic
3170  %val0 = extractvalue { i32, i1 } %val, 0
3171  store i32 %val0, i32* %out, align 4
3172  ret void
3173}
3174
3175define amdgpu_kernel void @flat_agent_release_monotonic_ret_cmpxchg(
3176; GFX7-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
3177; GFX7:       ; %bb.0: ; %entry
3178; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3179; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3180; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3181; GFX7-NEXT:    s_add_u32 s4, s0, 16
3182; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3183; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3184; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3185; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3186; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3187; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3188; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3189; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3190; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3191; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3192; GFX7-NEXT:    flat_store_dword v[0:1], v2
3193; GFX7-NEXT:    s_endpgm
3194;
3195; GFX10-WGP-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
3196; GFX10-WGP:       ; %bb.0: ; %entry
3197; GFX10-WGP-NEXT:    s_clause 0x1
3198; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3199; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3200; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3201; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3202; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3203; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3204; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3205; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
3206; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3207; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3208; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3209; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3210; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3211; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3212; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3213; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3214; GFX10-WGP-NEXT:    s_endpgm
3215;
3216; GFX10-CU-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
3217; GFX10-CU:       ; %bb.0: ; %entry
3218; GFX10-CU-NEXT:    s_clause 0x1
3219; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3220; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3221; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3222; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3223; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3224; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3225; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3226; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3227; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3228; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3229; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3230; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3231; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3232; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3233; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3234; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3235; GFX10-CU-NEXT:    s_endpgm
3236;
3237; SKIP-CACHE-INV-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
3238; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3239; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3240; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3241; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3242; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3243; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3244; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3245; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3246; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3247; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3248; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3249; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3250; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3251; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3252; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3253; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3254; SKIP-CACHE-INV-NEXT:    s_endpgm
3255;
3256; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
3257; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3258; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3259; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3260; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3261; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3262; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3263; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3264; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3265; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3266; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3267; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3268;
3269; GFX90A-TGSPLIT-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
3270; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3271; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3272; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3273; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3274; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3275; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3276; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3277; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3278; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3279; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3280; GFX90A-TGSPLIT-NEXT:    s_endpgm
3281    i32* %out, i32 %in, i32 %old) {
3282entry:
3283  %gep = getelementptr i32, i32* %out, i32 4
3284  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release monotonic
3285  %val0 = extractvalue { i32, i1 } %val, 0
3286  store i32 %val0, i32* %out, align 4
3287  ret void
3288}
3289
3290define amdgpu_kernel void @flat_agent_acq_rel_monotonic_ret_cmpxchg(
3291; GFX7-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
3292; GFX7:       ; %bb.0: ; %entry
3293; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3294; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3295; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3296; GFX7-NEXT:    s_add_u32 s4, s0, 16
3297; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3298; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3299; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3300; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3301; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3302; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3303; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3304; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3305; GFX7-NEXT:    buffer_wbinvl1_vol
3306; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3307; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3308; GFX7-NEXT:    flat_store_dword v[0:1], v2
3309; GFX7-NEXT:    s_endpgm
3310;
3311; GFX10-WGP-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
3312; GFX10-WGP:       ; %bb.0: ; %entry
3313; GFX10-WGP-NEXT:    s_clause 0x1
3314; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3315; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3316; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3317; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3318; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3319; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3320; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3321; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
3322; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3323; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3324; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3325; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3326; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3327; GFX10-WGP-NEXT:    buffer_gl0_inv
3328; GFX10-WGP-NEXT:    buffer_gl1_inv
3329; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3330; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3331; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3332; GFX10-WGP-NEXT:    s_endpgm
3333;
3334; GFX10-CU-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
3335; GFX10-CU:       ; %bb.0: ; %entry
3336; GFX10-CU-NEXT:    s_clause 0x1
3337; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3338; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3339; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3340; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3341; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3342; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3343; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3344; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3345; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3346; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3347; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3348; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3349; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3350; GFX10-CU-NEXT:    buffer_gl0_inv
3351; GFX10-CU-NEXT:    buffer_gl1_inv
3352; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3353; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3354; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3355; GFX10-CU-NEXT:    s_endpgm
3356;
3357; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
3358; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3359; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3360; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3361; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3362; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3363; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3364; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3365; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3366; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3367; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3368; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3369; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3370; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3371; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3372; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3373; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3374; SKIP-CACHE-INV-NEXT:    s_endpgm
3375;
3376; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
3377; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3378; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3379; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3380; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3381; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3382; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3383; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3384; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3385; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3386; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3387; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3388; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3389;
3390; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
3391; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3392; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3393; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3394; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3395; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3396; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3397; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3398; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3399; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3400; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3401; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3402; GFX90A-TGSPLIT-NEXT:    s_endpgm
3403    i32* %out, i32 %in, i32 %old) {
3404entry:
3405  %gep = getelementptr i32, i32* %out, i32 4
3406  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel monotonic
3407  %val0 = extractvalue { i32, i1 } %val, 0
3408  store i32 %val0, i32* %out, align 4
3409  ret void
3410}
3411
3412define amdgpu_kernel void @flat_agent_seq_cst_monotonic_ret_cmpxchg(
3413; GFX7-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
3414; GFX7:       ; %bb.0: ; %entry
3415; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3416; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3417; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3418; GFX7-NEXT:    s_add_u32 s4, s0, 16
3419; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3420; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3421; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3422; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3423; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3424; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3425; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3426; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3427; GFX7-NEXT:    buffer_wbinvl1_vol
3428; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3429; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3430; GFX7-NEXT:    flat_store_dword v[0:1], v2
3431; GFX7-NEXT:    s_endpgm
3432;
3433; GFX10-WGP-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
3434; GFX10-WGP:       ; %bb.0: ; %entry
3435; GFX10-WGP-NEXT:    s_clause 0x1
3436; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3437; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3438; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3439; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3440; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3441; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3442; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3443; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
3444; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3445; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3446; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3447; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3448; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3449; GFX10-WGP-NEXT:    buffer_gl0_inv
3450; GFX10-WGP-NEXT:    buffer_gl1_inv
3451; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3452; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3453; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3454; GFX10-WGP-NEXT:    s_endpgm
3455;
3456; GFX10-CU-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
3457; GFX10-CU:       ; %bb.0: ; %entry
3458; GFX10-CU-NEXT:    s_clause 0x1
3459; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3460; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3461; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3462; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3463; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3464; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3465; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3466; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3467; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3468; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3469; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3470; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3471; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3472; GFX10-CU-NEXT:    buffer_gl0_inv
3473; GFX10-CU-NEXT:    buffer_gl1_inv
3474; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3475; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3476; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3477; GFX10-CU-NEXT:    s_endpgm
3478;
3479; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
3480; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3481; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3482; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3483; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3484; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3485; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3486; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3487; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3488; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3489; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3490; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3491; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3492; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3493; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3494; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3495; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3496; SKIP-CACHE-INV-NEXT:    s_endpgm
3497;
3498; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
3499; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3500; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3501; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3502; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3503; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3504; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3505; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3506; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3507; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3508; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3509; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3510; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3511;
3512; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
3513; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3514; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3515; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3516; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3517; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3518; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3519; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3520; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3521; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3522; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3523; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3524; GFX90A-TGSPLIT-NEXT:    s_endpgm
3525    i32* %out, i32 %in, i32 %old) {
3526entry:
3527  %gep = getelementptr i32, i32* %out, i32 4
3528  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst monotonic
3529  %val0 = extractvalue { i32, i1 } %val, 0
3530  store i32 %val0, i32* %out, align 4
3531  ret void
3532}
3533
3534define amdgpu_kernel void @flat_agent_monotonic_acquire_ret_cmpxchg(
3535; GFX7-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
3536; GFX7:       ; %bb.0: ; %entry
3537; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3538; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3539; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3540; GFX7-NEXT:    s_add_u32 s4, s0, 16
3541; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3542; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3543; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3544; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3545; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3546; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3547; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3548; GFX7-NEXT:    buffer_wbinvl1_vol
3549; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3550; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3551; GFX7-NEXT:    flat_store_dword v[0:1], v2
3552; GFX7-NEXT:    s_endpgm
3553;
3554; GFX10-WGP-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
3555; GFX10-WGP:       ; %bb.0: ; %entry
3556; GFX10-WGP-NEXT:    s_clause 0x1
3557; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3558; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3559; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3560; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3561; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3562; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3563; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3564; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
3565; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3566; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3567; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3568; GFX10-WGP-NEXT:    buffer_gl0_inv
3569; GFX10-WGP-NEXT:    buffer_gl1_inv
3570; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3571; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3572; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3573; GFX10-WGP-NEXT:    s_endpgm
3574;
3575; GFX10-CU-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
3576; GFX10-CU:       ; %bb.0: ; %entry
3577; GFX10-CU-NEXT:    s_clause 0x1
3578; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3579; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3580; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3581; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3582; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3583; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3584; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3585; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3586; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3587; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3588; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3589; GFX10-CU-NEXT:    buffer_gl0_inv
3590; GFX10-CU-NEXT:    buffer_gl1_inv
3591; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3592; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3593; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3594; GFX10-CU-NEXT:    s_endpgm
3595;
3596; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
3597; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3598; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3599; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3600; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3601; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3602; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3603; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3604; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3605; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3606; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3607; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3608; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3609; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3610; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3611; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3612; SKIP-CACHE-INV-NEXT:    s_endpgm
3613;
3614; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
3615; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3616; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3617; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3618; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3619; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3620; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3621; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3622; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3623; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3624; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3625; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3626;
3627; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
3628; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3629; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3630; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3631; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3632; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3633; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3634; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3635; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3636; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3637; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3638; GFX90A-TGSPLIT-NEXT:    s_endpgm
3639    i32* %out, i32 %in, i32 %old) {
3640entry:
3641  %gep = getelementptr i32, i32* %out, i32 4
3642  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") monotonic acquire
3643  %val0 = extractvalue { i32, i1 } %val, 0
3644  store i32 %val0, i32* %out, align 4
3645  ret void
3646}
3647
3648define amdgpu_kernel void @flat_agent_acquire_acquire_ret_cmpxchg(
3649; GFX7-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
3650; GFX7:       ; %bb.0: ; %entry
3651; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3652; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3653; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3654; GFX7-NEXT:    s_add_u32 s4, s0, 16
3655; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3656; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3657; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3658; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3659; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3660; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3661; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3662; GFX7-NEXT:    buffer_wbinvl1_vol
3663; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3664; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3665; GFX7-NEXT:    flat_store_dword v[0:1], v2
3666; GFX7-NEXT:    s_endpgm
3667;
3668; GFX10-WGP-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
3669; GFX10-WGP:       ; %bb.0: ; %entry
3670; GFX10-WGP-NEXT:    s_clause 0x1
3671; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3672; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3673; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3674; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3675; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3676; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3677; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3678; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
3679; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3680; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3681; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3682; GFX10-WGP-NEXT:    buffer_gl0_inv
3683; GFX10-WGP-NEXT:    buffer_gl1_inv
3684; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3685; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3686; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3687; GFX10-WGP-NEXT:    s_endpgm
3688;
3689; GFX10-CU-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
3690; GFX10-CU:       ; %bb.0: ; %entry
3691; GFX10-CU-NEXT:    s_clause 0x1
3692; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3693; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3694; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3695; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3696; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3697; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3698; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3699; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3700; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3701; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3702; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3703; GFX10-CU-NEXT:    buffer_gl0_inv
3704; GFX10-CU-NEXT:    buffer_gl1_inv
3705; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3706; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3707; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3708; GFX10-CU-NEXT:    s_endpgm
3709;
3710; SKIP-CACHE-INV-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
3711; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3712; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3713; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3714; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3715; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3716; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3717; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3718; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3719; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3720; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3721; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3722; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3723; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3724; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3725; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3726; SKIP-CACHE-INV-NEXT:    s_endpgm
3727;
3728; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
3729; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3730; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3731; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3732; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3733; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3734; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3735; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3736; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3737; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3738; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3739; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3740;
3741; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
3742; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3743; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3744; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3745; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3746; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3747; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3748; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3749; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3750; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3751; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3752; GFX90A-TGSPLIT-NEXT:    s_endpgm
3753    i32* %out, i32 %in, i32 %old) {
3754entry:
3755  %gep = getelementptr i32, i32* %out, i32 4
3756  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire acquire
3757  %val0 = extractvalue { i32, i1 } %val, 0
3758  store i32 %val0, i32* %out, align 4
3759  ret void
3760}
3761
3762define amdgpu_kernel void @flat_agent_release_acquire_ret_cmpxchg(
3763; GFX7-LABEL: flat_agent_release_acquire_ret_cmpxchg:
3764; GFX7:       ; %bb.0: ; %entry
3765; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3766; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3767; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3768; GFX7-NEXT:    s_add_u32 s4, s0, 16
3769; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3770; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3771; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3772; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3773; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3774; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3775; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3776; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3777; GFX7-NEXT:    buffer_wbinvl1_vol
3778; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3779; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3780; GFX7-NEXT:    flat_store_dword v[0:1], v2
3781; GFX7-NEXT:    s_endpgm
3782;
3783; GFX10-WGP-LABEL: flat_agent_release_acquire_ret_cmpxchg:
3784; GFX10-WGP:       ; %bb.0: ; %entry
3785; GFX10-WGP-NEXT:    s_clause 0x1
3786; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3787; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3788; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3789; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3790; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3791; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3792; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3793; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
3794; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3795; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3796; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3797; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3798; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3799; GFX10-WGP-NEXT:    buffer_gl0_inv
3800; GFX10-WGP-NEXT:    buffer_gl1_inv
3801; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3802; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3803; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3804; GFX10-WGP-NEXT:    s_endpgm
3805;
3806; GFX10-CU-LABEL: flat_agent_release_acquire_ret_cmpxchg:
3807; GFX10-CU:       ; %bb.0: ; %entry
3808; GFX10-CU-NEXT:    s_clause 0x1
3809; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3810; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3811; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3812; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3813; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3814; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3815; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3816; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3817; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3818; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3819; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3820; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3821; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3822; GFX10-CU-NEXT:    buffer_gl0_inv
3823; GFX10-CU-NEXT:    buffer_gl1_inv
3824; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3825; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3826; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3827; GFX10-CU-NEXT:    s_endpgm
3828;
3829; SKIP-CACHE-INV-LABEL: flat_agent_release_acquire_ret_cmpxchg:
3830; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3831; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3832; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3833; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3834; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3835; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3836; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3837; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3838; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3839; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3840; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3841; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3842; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3843; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3844; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3845; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3846; SKIP-CACHE-INV-NEXT:    s_endpgm
3847;
3848; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_acquire_ret_cmpxchg:
3849; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3850; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3851; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3852; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3853; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3854; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3855; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3856; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3857; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3858; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3859; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3860; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3861;
3862; GFX90A-TGSPLIT-LABEL: flat_agent_release_acquire_ret_cmpxchg:
3863; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3864; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3865; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3866; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3867; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3868; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3869; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3870; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3871; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3872; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3873; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3874; GFX90A-TGSPLIT-NEXT:    s_endpgm
3875    i32* %out, i32 %in, i32 %old) {
3876entry:
3877  %gep = getelementptr i32, i32* %out, i32 4
3878  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release acquire
3879  %val0 = extractvalue { i32, i1 } %val, 0
3880  store i32 %val0, i32* %out, align 4
3881  ret void
3882}
3883
3884define amdgpu_kernel void @flat_agent_acq_rel_acquire_ret_cmpxchg(
3885; GFX7-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
3886; GFX7:       ; %bb.0: ; %entry
3887; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3888; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3889; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3890; GFX7-NEXT:    s_add_u32 s4, s0, 16
3891; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3892; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3893; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3894; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3895; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3896; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3897; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3898; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3899; GFX7-NEXT:    buffer_wbinvl1_vol
3900; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3901; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3902; GFX7-NEXT:    flat_store_dword v[0:1], v2
3903; GFX7-NEXT:    s_endpgm
3904;
3905; GFX10-WGP-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
3906; GFX10-WGP:       ; %bb.0: ; %entry
3907; GFX10-WGP-NEXT:    s_clause 0x1
3908; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3909; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3910; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3911; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3912; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3913; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3914; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3915; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
3916; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3917; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3918; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3919; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3920; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3921; GFX10-WGP-NEXT:    buffer_gl0_inv
3922; GFX10-WGP-NEXT:    buffer_gl1_inv
3923; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3924; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3925; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3926; GFX10-WGP-NEXT:    s_endpgm
3927;
3928; GFX10-CU-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
3929; GFX10-CU:       ; %bb.0: ; %entry
3930; GFX10-CU-NEXT:    s_clause 0x1
3931; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3932; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3933; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3934; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3935; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3936; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3937; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3938; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3939; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3940; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3941; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3942; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3943; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3944; GFX10-CU-NEXT:    buffer_gl0_inv
3945; GFX10-CU-NEXT:    buffer_gl1_inv
3946; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3947; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3948; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3949; GFX10-CU-NEXT:    s_endpgm
3950;
3951; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
3952; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3953; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3954; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3955; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3956; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3957; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3958; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3959; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3960; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3961; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3962; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3963; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3964; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3965; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3966; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3967; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3968; SKIP-CACHE-INV-NEXT:    s_endpgm
3969;
3970; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
3971; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3972; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3973; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3974; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3975; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3976; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3977; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3978; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3979; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3980; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3981; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3982; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3983;
3984; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
3985; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3986; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3987; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3988; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3989; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3990; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3991; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3992; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3993; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3994; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3995; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3996; GFX90A-TGSPLIT-NEXT:    s_endpgm
3997    i32* %out, i32 %in, i32 %old) {
3998entry:
3999  %gep = getelementptr i32, i32* %out, i32 4
4000  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel acquire
4001  %val0 = extractvalue { i32, i1 } %val, 0
4002  store i32 %val0, i32* %out, align 4
4003  ret void
4004}
4005
4006define amdgpu_kernel void @flat_agent_seq_cst_acquire_ret_cmpxchg(
4007; GFX7-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
4008; GFX7:       ; %bb.0: ; %entry
4009; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4010; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4011; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4012; GFX7-NEXT:    s_add_u32 s4, s0, 16
4013; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4014; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4015; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4016; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4017; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4018; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4019; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4020; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4021; GFX7-NEXT:    buffer_wbinvl1_vol
4022; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4023; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4024; GFX7-NEXT:    flat_store_dword v[0:1], v2
4025; GFX7-NEXT:    s_endpgm
4026;
4027; GFX10-WGP-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
4028; GFX10-WGP:       ; %bb.0: ; %entry
4029; GFX10-WGP-NEXT:    s_clause 0x1
4030; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4031; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4032; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4033; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4034; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4035; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4036; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4037; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4038; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4039; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4040; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4041; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4042; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4043; GFX10-WGP-NEXT:    buffer_gl0_inv
4044; GFX10-WGP-NEXT:    buffer_gl1_inv
4045; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4046; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4047; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4048; GFX10-WGP-NEXT:    s_endpgm
4049;
4050; GFX10-CU-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
4051; GFX10-CU:       ; %bb.0: ; %entry
4052; GFX10-CU-NEXT:    s_clause 0x1
4053; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4054; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4055; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4056; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4057; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4058; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4059; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4060; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4061; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4062; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4063; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4064; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4065; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4066; GFX10-CU-NEXT:    buffer_gl0_inv
4067; GFX10-CU-NEXT:    buffer_gl1_inv
4068; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4069; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4070; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4071; GFX10-CU-NEXT:    s_endpgm
4072;
4073; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
4074; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4075; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4076; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4077; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4078; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4079; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4080; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4081; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4082; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4083; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4084; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4085; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4086; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4087; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4088; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4089; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4090; SKIP-CACHE-INV-NEXT:    s_endpgm
4091;
4092; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
4093; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4094; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4095; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4096; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4097; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4098; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4099; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4100; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4101; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4102; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4103; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4104; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4105;
4106; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
4107; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4108; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4109; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4110; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4111; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4112; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4113; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4114; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4115; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4116; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4117; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4118; GFX90A-TGSPLIT-NEXT:    s_endpgm
4119    i32* %out, i32 %in, i32 %old) {
4120entry:
4121  %gep = getelementptr i32, i32* %out, i32 4
4122  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst acquire
4123  %val0 = extractvalue { i32, i1 } %val, 0
4124  store i32 %val0, i32* %out, align 4
4125  ret void
4126}
4127
4128define amdgpu_kernel void @flat_agent_monotonic_seq_cst_ret_cmpxchg(
4129; GFX7-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
4130; GFX7:       ; %bb.0: ; %entry
4131; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4132; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4133; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4134; GFX7-NEXT:    s_add_u32 s4, s0, 16
4135; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4136; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4137; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4138; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4139; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4140; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4141; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4142; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4143; GFX7-NEXT:    buffer_wbinvl1_vol
4144; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4145; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4146; GFX7-NEXT:    flat_store_dword v[0:1], v2
4147; GFX7-NEXT:    s_endpgm
4148;
4149; GFX10-WGP-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
4150; GFX10-WGP:       ; %bb.0: ; %entry
4151; GFX10-WGP-NEXT:    s_clause 0x1
4152; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4153; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4154; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4155; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4156; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4157; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4158; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4159; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4160; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4161; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4162; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4163; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4164; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4165; GFX10-WGP-NEXT:    buffer_gl0_inv
4166; GFX10-WGP-NEXT:    buffer_gl1_inv
4167; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4168; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4169; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4170; GFX10-WGP-NEXT:    s_endpgm
4171;
4172; GFX10-CU-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
4173; GFX10-CU:       ; %bb.0: ; %entry
4174; GFX10-CU-NEXT:    s_clause 0x1
4175; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4176; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4177; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4178; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4179; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4180; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4181; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4182; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4183; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4184; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4185; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4186; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4187; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4188; GFX10-CU-NEXT:    buffer_gl0_inv
4189; GFX10-CU-NEXT:    buffer_gl1_inv
4190; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4191; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4192; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4193; GFX10-CU-NEXT:    s_endpgm
4194;
4195; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
4196; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4197; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4198; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4199; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4200; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4201; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4202; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4203; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4204; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4205; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4206; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4207; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4208; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4209; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4210; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4211; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4212; SKIP-CACHE-INV-NEXT:    s_endpgm
4213;
4214; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
4215; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4216; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4217; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4218; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4219; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4220; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4221; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4222; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4223; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4224; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4225; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4226; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4227;
4228; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
4229; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4230; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4231; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4232; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4233; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4234; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4235; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4236; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4237; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4238; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4239; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4240; GFX90A-TGSPLIT-NEXT:    s_endpgm
4241    i32* %out, i32 %in, i32 %old) {
4242entry:
4243  %gep = getelementptr i32, i32* %out, i32 4
4244  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") monotonic seq_cst
4245  %val0 = extractvalue { i32, i1 } %val, 0
4246  store i32 %val0, i32* %out, align 4
4247  ret void
4248}
4249
4250define amdgpu_kernel void @flat_agent_acquire_seq_cst_ret_cmpxchg(
4251; GFX7-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
4252; GFX7:       ; %bb.0: ; %entry
4253; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4254; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4255; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4256; GFX7-NEXT:    s_add_u32 s4, s0, 16
4257; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4258; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4259; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4260; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4261; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4262; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4263; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4264; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4265; GFX7-NEXT:    buffer_wbinvl1_vol
4266; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4267; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4268; GFX7-NEXT:    flat_store_dword v[0:1], v2
4269; GFX7-NEXT:    s_endpgm
4270;
4271; GFX10-WGP-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
4272; GFX10-WGP:       ; %bb.0: ; %entry
4273; GFX10-WGP-NEXT:    s_clause 0x1
4274; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4275; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4276; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4277; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4278; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4279; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4280; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4281; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4282; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4283; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4284; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4285; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4286; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4287; GFX10-WGP-NEXT:    buffer_gl0_inv
4288; GFX10-WGP-NEXT:    buffer_gl1_inv
4289; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4290; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4291; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4292; GFX10-WGP-NEXT:    s_endpgm
4293;
4294; GFX10-CU-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
4295; GFX10-CU:       ; %bb.0: ; %entry
4296; GFX10-CU-NEXT:    s_clause 0x1
4297; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4298; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4299; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4300; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4301; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4302; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4303; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4304; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4305; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4306; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4307; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4308; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4309; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4310; GFX10-CU-NEXT:    buffer_gl0_inv
4311; GFX10-CU-NEXT:    buffer_gl1_inv
4312; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4313; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4314; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4315; GFX10-CU-NEXT:    s_endpgm
4316;
4317; SKIP-CACHE-INV-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
4318; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4319; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4320; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4321; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4322; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4323; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4324; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4325; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4326; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4327; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4328; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4329; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4330; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4331; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4332; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4333; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4334; SKIP-CACHE-INV-NEXT:    s_endpgm
4335;
4336; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
4337; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4338; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4339; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4340; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4341; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4342; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4343; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4344; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4345; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4346; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4347; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4348; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4349;
4350; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
4351; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4352; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4353; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4354; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4355; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4356; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4357; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4358; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4359; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4360; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4361; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4362; GFX90A-TGSPLIT-NEXT:    s_endpgm
4363    i32* %out, i32 %in, i32 %old) {
4364entry:
4365  %gep = getelementptr i32, i32* %out, i32 4
4366  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire seq_cst
4367  %val0 = extractvalue { i32, i1 } %val, 0
4368  store i32 %val0, i32* %out, align 4
4369  ret void
4370}
4371
4372define amdgpu_kernel void @flat_agent_release_seq_cst_ret_cmpxchg(
4373; GFX7-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
4374; GFX7:       ; %bb.0: ; %entry
4375; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4376; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4377; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4378; GFX7-NEXT:    s_add_u32 s4, s0, 16
4379; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4380; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4381; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4382; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4383; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4384; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4385; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4386; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4387; GFX7-NEXT:    buffer_wbinvl1_vol
4388; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4389; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4390; GFX7-NEXT:    flat_store_dword v[0:1], v2
4391; GFX7-NEXT:    s_endpgm
4392;
4393; GFX10-WGP-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
4394; GFX10-WGP:       ; %bb.0: ; %entry
4395; GFX10-WGP-NEXT:    s_clause 0x1
4396; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4397; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4398; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4399; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4400; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4401; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4402; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4403; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4404; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4405; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4406; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4407; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4408; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4409; GFX10-WGP-NEXT:    buffer_gl0_inv
4410; GFX10-WGP-NEXT:    buffer_gl1_inv
4411; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4412; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4413; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4414; GFX10-WGP-NEXT:    s_endpgm
4415;
4416; GFX10-CU-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
4417; GFX10-CU:       ; %bb.0: ; %entry
4418; GFX10-CU-NEXT:    s_clause 0x1
4419; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4420; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4421; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4422; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4423; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4424; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4425; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4426; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4427; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4428; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4429; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4430; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4431; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4432; GFX10-CU-NEXT:    buffer_gl0_inv
4433; GFX10-CU-NEXT:    buffer_gl1_inv
4434; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4435; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4436; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4437; GFX10-CU-NEXT:    s_endpgm
4438;
4439; SKIP-CACHE-INV-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
4440; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4441; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4442; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4443; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4444; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4445; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4446; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4447; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4448; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4449; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4450; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4451; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4452; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4453; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4454; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4455; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4456; SKIP-CACHE-INV-NEXT:    s_endpgm
4457;
4458; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
4459; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4460; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4461; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4462; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4463; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4464; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4465; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4466; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4467; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4468; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4469; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4470; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4471;
4472; GFX90A-TGSPLIT-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
4473; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4474; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4475; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4476; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4477; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4478; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4479; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4480; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4481; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4482; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4483; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4484; GFX90A-TGSPLIT-NEXT:    s_endpgm
4485    i32* %out, i32 %in, i32 %old) {
4486entry:
4487  %gep = getelementptr i32, i32* %out, i32 4
4488  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release seq_cst
4489  %val0 = extractvalue { i32, i1 } %val, 0
4490  store i32 %val0, i32* %out, align 4
4491  ret void
4492}
4493
4494define amdgpu_kernel void @flat_agent_acq_rel_seq_cst_ret_cmpxchg(
4495; GFX7-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
4496; GFX7:       ; %bb.0: ; %entry
4497; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4498; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4499; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4500; GFX7-NEXT:    s_add_u32 s4, s0, 16
4501; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4502; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4503; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4504; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4505; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4506; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4507; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4508; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4509; GFX7-NEXT:    buffer_wbinvl1_vol
4510; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4511; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4512; GFX7-NEXT:    flat_store_dword v[0:1], v2
4513; GFX7-NEXT:    s_endpgm
4514;
4515; GFX10-WGP-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
4516; GFX10-WGP:       ; %bb.0: ; %entry
4517; GFX10-WGP-NEXT:    s_clause 0x1
4518; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4519; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4520; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4521; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4522; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4523; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4524; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4525; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4526; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4527; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4528; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4529; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4530; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4531; GFX10-WGP-NEXT:    buffer_gl0_inv
4532; GFX10-WGP-NEXT:    buffer_gl1_inv
4533; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4534; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4535; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4536; GFX10-WGP-NEXT:    s_endpgm
4537;
4538; GFX10-CU-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
4539; GFX10-CU:       ; %bb.0: ; %entry
4540; GFX10-CU-NEXT:    s_clause 0x1
4541; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4542; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4543; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4544; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4545; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4546; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4547; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4548; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4549; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4550; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4551; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4552; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4553; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4554; GFX10-CU-NEXT:    buffer_gl0_inv
4555; GFX10-CU-NEXT:    buffer_gl1_inv
4556; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4557; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4558; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4559; GFX10-CU-NEXT:    s_endpgm
4560;
4561; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
4562; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4563; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4564; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4565; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4566; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4567; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4568; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4569; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4570; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4571; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4572; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4573; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4574; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4575; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4576; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4577; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4578; SKIP-CACHE-INV-NEXT:    s_endpgm
4579;
4580; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
4581; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4582; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4583; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4584; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4585; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4586; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4587; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4588; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4589; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4590; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4591; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4592; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4593;
4594; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
4595; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4596; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4597; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4598; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4599; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4600; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4601; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4602; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4603; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4604; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4605; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4606; GFX90A-TGSPLIT-NEXT:    s_endpgm
4607    i32* %out, i32 %in, i32 %old) {
4608entry:
4609  %gep = getelementptr i32, i32* %out, i32 4
4610  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel seq_cst
4611  %val0 = extractvalue { i32, i1 } %val, 0
4612  store i32 %val0, i32* %out, align 4
4613  ret void
4614}
4615
4616define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_ret_cmpxchg(
4617; GFX7-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
4618; GFX7:       ; %bb.0: ; %entry
4619; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4620; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4621; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4622; GFX7-NEXT:    s_add_u32 s4, s0, 16
4623; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4624; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4625; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4626; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4627; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4628; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4629; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4630; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4631; GFX7-NEXT:    buffer_wbinvl1_vol
4632; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4633; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4634; GFX7-NEXT:    flat_store_dword v[0:1], v2
4635; GFX7-NEXT:    s_endpgm
4636;
4637; GFX10-WGP-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
4638; GFX10-WGP:       ; %bb.0: ; %entry
4639; GFX10-WGP-NEXT:    s_clause 0x1
4640; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4641; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4642; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4643; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4644; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4645; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4646; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4647; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4648; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4649; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4650; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4651; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4652; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4653; GFX10-WGP-NEXT:    buffer_gl0_inv
4654; GFX10-WGP-NEXT:    buffer_gl1_inv
4655; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4656; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4657; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4658; GFX10-WGP-NEXT:    s_endpgm
4659;
4660; GFX10-CU-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
4661; GFX10-CU:       ; %bb.0: ; %entry
4662; GFX10-CU-NEXT:    s_clause 0x1
4663; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4664; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4665; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4666; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4667; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4668; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4669; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4670; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4671; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4672; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4673; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4674; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4675; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4676; GFX10-CU-NEXT:    buffer_gl0_inv
4677; GFX10-CU-NEXT:    buffer_gl1_inv
4678; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4679; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4680; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4681; GFX10-CU-NEXT:    s_endpgm
4682;
4683; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
4684; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4685; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4686; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4687; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4688; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4689; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4690; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4691; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4692; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4693; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4694; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4695; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4696; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4697; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4698; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4699; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4700; SKIP-CACHE-INV-NEXT:    s_endpgm
4701;
4702; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
4703; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4704; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4705; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4706; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4707; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4708; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4709; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4710; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4711; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4712; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4713; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4714; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4715;
4716; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
4717; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4718; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4719; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4720; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4721; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4722; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4723; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4724; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4725; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4726; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4727; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4728; GFX90A-TGSPLIT-NEXT:    s_endpgm
4729    i32* %out, i32 %in, i32 %old) {
4730entry:
4731  %gep = getelementptr i32, i32* %out, i32 4
4732  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
4733  %val0 = extractvalue { i32, i1 } %val, 0
4734  store i32 %val0, i32* %out, align 4
4735  ret void
4736}
4737
4738define amdgpu_kernel void @flat_agent_one_as_unordered_load(
4739; GFX7-LABEL: flat_agent_one_as_unordered_load:
4740; GFX7:       ; %bb.0: ; %entry
4741; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4742; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4743; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4744; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4745; GFX7-NEXT:    flat_load_dword v2, v[0:1]
4746; GFX7-NEXT:    v_mov_b32_e32 v0, s2
4747; GFX7-NEXT:    v_mov_b32_e32 v1, s3
4748; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4749; GFX7-NEXT:    flat_store_dword v[0:1], v2
4750; GFX7-NEXT:    s_endpgm
4751;
4752; GFX10-WGP-LABEL: flat_agent_one_as_unordered_load:
4753; GFX10-WGP:       ; %bb.0: ; %entry
4754; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4755; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4756; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4757; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4758; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
4759; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
4760; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
4761; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4762; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4763; GFX10-WGP-NEXT:    s_endpgm
4764;
4765; GFX10-CU-LABEL: flat_agent_one_as_unordered_load:
4766; GFX10-CU:       ; %bb.0: ; %entry
4767; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4768; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4769; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4770; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4771; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
4772; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
4773; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
4774; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4775; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4776; GFX10-CU-NEXT:    s_endpgm
4777;
4778; SKIP-CACHE-INV-LABEL: flat_agent_one_as_unordered_load:
4779; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4780; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4781; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4782; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4783; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
4784; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
4785; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4786; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4787; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4788; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4789; SKIP-CACHE-INV-NEXT:    s_endpgm
4790;
4791; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_unordered_load:
4792; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4793; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4794; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4795; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4796; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
4797; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
4798; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
4799; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
4800; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4801; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4802; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4803;
4804; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_unordered_load:
4805; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4806; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4807; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4808; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4809; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
4810; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
4811; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
4812; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
4813; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4814; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4815; GFX90A-TGSPLIT-NEXT:    s_endpgm
4816    i32* %in, i32* %out) {
4817entry:
4818  %val = load atomic i32, i32* %in syncscope("agent-one-as") unordered, align 4
4819  store i32 %val, i32* %out
4820  ret void
4821}
4822
4823define amdgpu_kernel void @flat_agent_one_as_monotonic_load(
4824; GFX7-LABEL: flat_agent_one_as_monotonic_load:
4825; GFX7:       ; %bb.0: ; %entry
4826; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4827; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4828; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4829; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4830; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
4831; GFX7-NEXT:    v_mov_b32_e32 v0, s2
4832; GFX7-NEXT:    v_mov_b32_e32 v1, s3
4833; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4834; GFX7-NEXT:    flat_store_dword v[0:1], v2
4835; GFX7-NEXT:    s_endpgm
4836;
4837; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_load:
4838; GFX10-WGP:       ; %bb.0: ; %entry
4839; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4840; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4841; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4842; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4843; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
4844; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
4845; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
4846; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4847; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4848; GFX10-WGP-NEXT:    s_endpgm
4849;
4850; GFX10-CU-LABEL: flat_agent_one_as_monotonic_load:
4851; GFX10-CU:       ; %bb.0: ; %entry
4852; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4853; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4854; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4855; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4856; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
4857; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
4858; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
4859; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4860; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4861; GFX10-CU-NEXT:    s_endpgm
4862;
4863; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_load:
4864; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4865; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4866; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4867; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4868; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
4869; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
4870; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4871; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4872; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4873; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4874; SKIP-CACHE-INV-NEXT:    s_endpgm
4875;
4876; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_load:
4877; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4878; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4879; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4880; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4881; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
4882; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
4883; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
4884; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
4885; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4886; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4887; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4888;
4889; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_load:
4890; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4891; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4892; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4893; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4894; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
4895; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
4896; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
4897; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
4898; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4899; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4900; GFX90A-TGSPLIT-NEXT:    s_endpgm
4901    i32* %in, i32* %out) {
4902entry:
4903  %val = load atomic i32, i32* %in syncscope("agent-one-as") monotonic, align 4
4904  store i32 %val, i32* %out
4905  ret void
4906}
4907
4908define amdgpu_kernel void @flat_agent_one_as_acquire_load(
4909; GFX7-LABEL: flat_agent_one_as_acquire_load:
4910; GFX7:       ; %bb.0: ; %entry
4911; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4912; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4913; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4914; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4915; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
4916; GFX7-NEXT:    s_waitcnt vmcnt(0)
4917; GFX7-NEXT:    buffer_wbinvl1_vol
4918; GFX7-NEXT:    v_mov_b32_e32 v0, s2
4919; GFX7-NEXT:    v_mov_b32_e32 v1, s3
4920; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4921; GFX7-NEXT:    flat_store_dword v[0:1], v2
4922; GFX7-NEXT:    s_endpgm
4923;
4924; GFX10-WGP-LABEL: flat_agent_one_as_acquire_load:
4925; GFX10-WGP:       ; %bb.0: ; %entry
4926; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4927; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4928; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4929; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4930; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
4931; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4932; GFX10-WGP-NEXT:    buffer_gl0_inv
4933; GFX10-WGP-NEXT:    buffer_gl1_inv
4934; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
4935; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
4936; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4937; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4938; GFX10-WGP-NEXT:    s_endpgm
4939;
4940; GFX10-CU-LABEL: flat_agent_one_as_acquire_load:
4941; GFX10-CU:       ; %bb.0: ; %entry
4942; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4943; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4944; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4945; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4946; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
4947; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4948; GFX10-CU-NEXT:    buffer_gl0_inv
4949; GFX10-CU-NEXT:    buffer_gl1_inv
4950; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
4951; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
4952; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4953; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4954; GFX10-CU-NEXT:    s_endpgm
4955;
4956; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_load:
4957; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4958; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
4959; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4960; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4961; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
4962; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
4963; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4964; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4965; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4966; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4967; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4968; SKIP-CACHE-INV-NEXT:    s_endpgm
4969;
4970; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_load:
4971; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4972; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4973; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4974; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4975; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
4976; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
4977; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4978; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4979; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
4980; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
4981; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4982; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4983; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4984;
4985; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_load:
4986; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4987; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4988; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4989; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
4990; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
4991; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
4992; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4993; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4994; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
4995; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
4996; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4997; GFX90A-TGSPLIT-NEXT:    s_endpgm
4998    i32* %in, i32* %out) {
4999entry:
5000  %val = load atomic i32, i32* %in syncscope("agent-one-as") acquire, align 4
5001  store i32 %val, i32* %out
5002  ret void
5003}
5004
5005define amdgpu_kernel void @flat_agent_one_as_seq_cst_load(
5006; GFX7-LABEL: flat_agent_one_as_seq_cst_load:
5007; GFX7:       ; %bb.0: ; %entry
5008; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5009; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5010; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5011; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5012; GFX7-NEXT:    s_waitcnt vmcnt(0)
5013; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
5014; GFX7-NEXT:    s_waitcnt vmcnt(0)
5015; GFX7-NEXT:    buffer_wbinvl1_vol
5016; GFX7-NEXT:    v_mov_b32_e32 v0, s2
5017; GFX7-NEXT:    v_mov_b32_e32 v1, s3
5018; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5019; GFX7-NEXT:    flat_store_dword v[0:1], v2
5020; GFX7-NEXT:    s_endpgm
5021;
5022; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_load:
5023; GFX10-WGP:       ; %bb.0: ; %entry
5024; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5025; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5026; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5027; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5028; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5029; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5030; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
5031; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5032; GFX10-WGP-NEXT:    buffer_gl0_inv
5033; GFX10-WGP-NEXT:    buffer_gl1_inv
5034; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
5035; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
5036; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5037; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5038; GFX10-WGP-NEXT:    s_endpgm
5039;
5040; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_load:
5041; GFX10-CU:       ; %bb.0: ; %entry
5042; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5043; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5044; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5045; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5046; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5047; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5048; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
5049; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5050; GFX10-CU-NEXT:    buffer_gl0_inv
5051; GFX10-CU-NEXT:    buffer_gl1_inv
5052; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
5053; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
5054; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5055; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5056; GFX10-CU-NEXT:    s_endpgm
5057;
5058; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_load:
5059; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5060; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
5061; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5062; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5063; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
5064; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5065; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
5066; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5067; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5068; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5069; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5070; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5071; SKIP-CACHE-INV-NEXT:    s_endpgm
5072;
5073; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_load:
5074; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5075; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5076; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5077; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5078; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
5079; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5080; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
5081; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5082; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5083; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
5084; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
5085; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5086; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5087; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5088;
5089; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_load:
5090; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5091; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5092; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5093; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
5094; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
5095; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5096; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
5097; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5098; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5099; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
5100; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
5101; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5102; GFX90A-TGSPLIT-NEXT:    s_endpgm
5103    i32* %in, i32* %out) {
5104entry:
5105  %val = load atomic i32, i32* %in syncscope("agent-one-as") seq_cst, align 4
5106  store i32 %val, i32* %out
5107  ret void
5108}
5109
5110define amdgpu_kernel void @flat_agent_one_as_unordered_store(
5111; GFX7-LABEL: flat_agent_one_as_unordered_store:
5112; GFX7:       ; %bb.0: ; %entry
5113; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
5114; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
5115; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5116; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5117; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5118; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5119; GFX7-NEXT:    flat_store_dword v[0:1], v2
5120; GFX7-NEXT:    s_endpgm
5121;
5122; GFX10-WGP-LABEL: flat_agent_one_as_unordered_store:
5123; GFX10-WGP:       ; %bb.0: ; %entry
5124; GFX10-WGP-NEXT:    s_clause 0x1
5125; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5126; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
5127; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5128; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5129; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5130; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5131; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5132; GFX10-WGP-NEXT:    s_endpgm
5133;
5134; GFX10-CU-LABEL: flat_agent_one_as_unordered_store:
5135; GFX10-CU:       ; %bb.0: ; %entry
5136; GFX10-CU-NEXT:    s_clause 0x1
5137; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5138; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
5139; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5140; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5141; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5142; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5143; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5144; GFX10-CU-NEXT:    s_endpgm
5145;
5146; SKIP-CACHE-INV-LABEL: flat_agent_one_as_unordered_store:
5147; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5148; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
5149; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
5150; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5151; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5152; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5153; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5154; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5155; SKIP-CACHE-INV-NEXT:    s_endpgm
5156;
5157; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_unordered_store:
5158; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5159; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5160; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
5161; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5162; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5163; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5164; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5165; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5166;
5167; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_unordered_store:
5168; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5169; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5170; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
5171; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5172; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5173; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5174; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5175; GFX90A-TGSPLIT-NEXT:    s_endpgm
5176    i32 %in, i32* %out) {
5177entry:
5178  store atomic i32 %in, i32* %out syncscope("agent-one-as") unordered, align 4
5179  ret void
5180}
5181
5182define amdgpu_kernel void @flat_agent_one_as_monotonic_store(
5183; GFX7-LABEL: flat_agent_one_as_monotonic_store:
5184; GFX7:       ; %bb.0: ; %entry
5185; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
5186; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
5187; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5188; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5189; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5190; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5191; GFX7-NEXT:    flat_store_dword v[0:1], v2
5192; GFX7-NEXT:    s_endpgm
5193;
5194; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_store:
5195; GFX10-WGP:       ; %bb.0: ; %entry
5196; GFX10-WGP-NEXT:    s_clause 0x1
5197; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5198; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
5199; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5200; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5201; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5202; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5203; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5204; GFX10-WGP-NEXT:    s_endpgm
5205;
5206; GFX10-CU-LABEL: flat_agent_one_as_monotonic_store:
5207; GFX10-CU:       ; %bb.0: ; %entry
5208; GFX10-CU-NEXT:    s_clause 0x1
5209; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5210; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
5211; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5212; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5213; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5214; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5215; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5216; GFX10-CU-NEXT:    s_endpgm
5217;
5218; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_store:
5219; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5220; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
5221; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
5222; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5223; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5224; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5225; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5226; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5227; SKIP-CACHE-INV-NEXT:    s_endpgm
5228;
5229; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_store:
5230; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5231; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5232; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
5233; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5234; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5235; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5236; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5237; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5238;
5239; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_store:
5240; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5241; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5242; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
5243; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5244; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5245; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5246; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5247; GFX90A-TGSPLIT-NEXT:    s_endpgm
5248    i32 %in, i32* %out) {
5249entry:
5250  store atomic i32 %in, i32* %out syncscope("agent-one-as") monotonic, align 4
5251  ret void
5252}
5253
5254define amdgpu_kernel void @flat_agent_one_as_release_store(
5255; GFX7-LABEL: flat_agent_one_as_release_store:
5256; GFX7:       ; %bb.0: ; %entry
5257; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
5258; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
5259; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5260; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5261; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5262; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5263; GFX7-NEXT:    s_waitcnt vmcnt(0)
5264; GFX7-NEXT:    flat_store_dword v[0:1], v2
5265; GFX7-NEXT:    s_endpgm
5266;
5267; GFX10-WGP-LABEL: flat_agent_one_as_release_store:
5268; GFX10-WGP:       ; %bb.0: ; %entry
5269; GFX10-WGP-NEXT:    s_clause 0x1
5270; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5271; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
5272; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5273; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5274; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5275; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5276; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5277; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5278; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5279; GFX10-WGP-NEXT:    s_endpgm
5280;
5281; GFX10-CU-LABEL: flat_agent_one_as_release_store:
5282; GFX10-CU:       ; %bb.0: ; %entry
5283; GFX10-CU-NEXT:    s_clause 0x1
5284; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5285; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
5286; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5287; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5288; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5289; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5290; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5291; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5292; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5293; GFX10-CU-NEXT:    s_endpgm
5294;
5295; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_store:
5296; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5297; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
5298; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
5299; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5300; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5301; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5302; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5303; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5304; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5305; SKIP-CACHE-INV-NEXT:    s_endpgm
5306;
5307; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_store:
5308; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5309; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5310; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
5311; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5312; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5313; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5314; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5315; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5316; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5317;
5318; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_store:
5319; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5320; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5321; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
5322; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5323; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5324; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5325; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5326; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5327; GFX90A-TGSPLIT-NEXT:    s_endpgm
5328    i32 %in, i32* %out) {
5329entry:
5330  store atomic i32 %in, i32* %out syncscope("agent-one-as") release, align 4
5331  ret void
5332}
5333
5334define amdgpu_kernel void @flat_agent_one_as_seq_cst_store(
5335; GFX7-LABEL: flat_agent_one_as_seq_cst_store:
5336; GFX7:       ; %bb.0: ; %entry
5337; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
5338; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
5339; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5340; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5341; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5342; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5343; GFX7-NEXT:    s_waitcnt vmcnt(0)
5344; GFX7-NEXT:    flat_store_dword v[0:1], v2
5345; GFX7-NEXT:    s_endpgm
5346;
5347; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_store:
5348; GFX10-WGP:       ; %bb.0: ; %entry
5349; GFX10-WGP-NEXT:    s_clause 0x1
5350; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5351; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
5352; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5353; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5354; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5355; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5356; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5357; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5358; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5359; GFX10-WGP-NEXT:    s_endpgm
5360;
5361; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_store:
5362; GFX10-CU:       ; %bb.0: ; %entry
5363; GFX10-CU-NEXT:    s_clause 0x1
5364; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5365; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
5366; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5367; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5368; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5369; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5370; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5371; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5372; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5373; GFX10-CU-NEXT:    s_endpgm
5374;
5375; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_store:
5376; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5377; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
5378; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
5379; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5380; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5381; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5382; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5383; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5384; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5385; SKIP-CACHE-INV-NEXT:    s_endpgm
5386;
5387; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_store:
5388; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5389; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5390; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
5391; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5392; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5393; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5394; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5395; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5396; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5397;
5398; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_store:
5399; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5400; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5401; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
5402; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5403; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5404; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5405; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5406; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5407; GFX90A-TGSPLIT-NEXT:    s_endpgm
5408    i32 %in, i32* %out) {
5409entry:
5410  store atomic i32 %in, i32* %out syncscope("agent-one-as") seq_cst, align 4
5411  ret void
5412}
5413
5414define amdgpu_kernel void @flat_agent_one_as_monotonic_atomicrmw(
5415; GFX7-LABEL: flat_agent_one_as_monotonic_atomicrmw:
5416; GFX7:       ; %bb.0: ; %entry
5417; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5418; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
5419; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5420; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5421; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5422; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5423; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
5424; GFX7-NEXT:    s_endpgm
5425;
5426; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_atomicrmw:
5427; GFX10-WGP:       ; %bb.0: ; %entry
5428; GFX10-WGP-NEXT:    s_clause 0x1
5429; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5430; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
5431; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5432; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5433; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5434; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5435; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
5436; GFX10-WGP-NEXT:    s_endpgm
5437;
5438; GFX10-CU-LABEL: flat_agent_one_as_monotonic_atomicrmw:
5439; GFX10-CU:       ; %bb.0: ; %entry
5440; GFX10-CU-NEXT:    s_clause 0x1
5441; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5442; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
5443; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5444; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5445; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5446; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5447; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
5448; GFX10-CU-NEXT:    s_endpgm
5449;
5450; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_atomicrmw:
5451; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5452; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5453; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
5454; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5455; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5456; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5457; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5458; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
5459; SKIP-CACHE-INV-NEXT:    s_endpgm
5460;
5461; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_atomicrmw:
5462; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5463; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5464; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5465; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5466; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5467; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5468; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
5469; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5470;
5471; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_atomicrmw:
5472; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5473; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5474; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5475; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5476; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5477; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5478; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
5479; GFX90A-TGSPLIT-NEXT:    s_endpgm
5480    i32* %out, i32 %in) {
5481entry:
5482  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") monotonic
5483  ret void
5484}
5485
5486define amdgpu_kernel void @flat_agent_one_as_acquire_atomicrmw(
5487; GFX7-LABEL: flat_agent_one_as_acquire_atomicrmw:
5488; GFX7:       ; %bb.0: ; %entry
5489; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5490; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
5491; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5492; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5493; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5494; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5495; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
5496; GFX7-NEXT:    s_waitcnt vmcnt(0)
5497; GFX7-NEXT:    buffer_wbinvl1_vol
5498; GFX7-NEXT:    s_endpgm
5499;
5500; GFX10-WGP-LABEL: flat_agent_one_as_acquire_atomicrmw:
5501; GFX10-WGP:       ; %bb.0: ; %entry
5502; GFX10-WGP-NEXT:    s_clause 0x1
5503; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5504; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
5505; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5506; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5507; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5508; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5509; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
5510; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5511; GFX10-WGP-NEXT:    buffer_gl0_inv
5512; GFX10-WGP-NEXT:    buffer_gl1_inv
5513; GFX10-WGP-NEXT:    s_endpgm
5514;
5515; GFX10-CU-LABEL: flat_agent_one_as_acquire_atomicrmw:
5516; GFX10-CU:       ; %bb.0: ; %entry
5517; GFX10-CU-NEXT:    s_clause 0x1
5518; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5519; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
5520; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5521; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5522; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5523; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5524; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
5525; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5526; GFX10-CU-NEXT:    buffer_gl0_inv
5527; GFX10-CU-NEXT:    buffer_gl1_inv
5528; GFX10-CU-NEXT:    s_endpgm
5529;
5530; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_atomicrmw:
5531; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5532; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5533; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
5534; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5535; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5536; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5537; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5538; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
5539; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5540; SKIP-CACHE-INV-NEXT:    s_endpgm
5541;
5542; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_atomicrmw:
5543; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5544; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5545; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5546; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5547; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5548; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5549; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
5550; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5551; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5552; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5553;
5554; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_atomicrmw:
5555; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5556; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5557; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5558; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5559; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5560; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5561; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
5562; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5563; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5564; GFX90A-TGSPLIT-NEXT:    s_endpgm
5565    i32* %out, i32 %in) {
5566entry:
5567  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") acquire
5568  ret void
5569}
5570
5571define amdgpu_kernel void @flat_agent_one_as_release_atomicrmw(
5572; GFX7-LABEL: flat_agent_one_as_release_atomicrmw:
5573; GFX7:       ; %bb.0: ; %entry
5574; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5575; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
5576; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5577; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5578; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5579; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5580; GFX7-NEXT:    s_waitcnt vmcnt(0)
5581; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
5582; GFX7-NEXT:    s_endpgm
5583;
5584; GFX10-WGP-LABEL: flat_agent_one_as_release_atomicrmw:
5585; GFX10-WGP:       ; %bb.0: ; %entry
5586; GFX10-WGP-NEXT:    s_clause 0x1
5587; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5588; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
5589; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5590; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5591; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5592; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5593; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5594; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5595; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
5596; GFX10-WGP-NEXT:    s_endpgm
5597;
5598; GFX10-CU-LABEL: flat_agent_one_as_release_atomicrmw:
5599; GFX10-CU:       ; %bb.0: ; %entry
5600; GFX10-CU-NEXT:    s_clause 0x1
5601; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5602; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
5603; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5604; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5605; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5606; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5607; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5608; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5609; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
5610; GFX10-CU-NEXT:    s_endpgm
5611;
5612; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_atomicrmw:
5613; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5614; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5615; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
5616; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5617; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5618; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5619; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5620; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5621; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
5622; SKIP-CACHE-INV-NEXT:    s_endpgm
5623;
5624; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_atomicrmw:
5625; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5626; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5627; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5628; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5629; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5630; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5631; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5632; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
5633; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5634;
5635; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_atomicrmw:
5636; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5637; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5638; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5639; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5640; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5641; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5642; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5643; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
5644; GFX90A-TGSPLIT-NEXT:    s_endpgm
5645    i32* %out, i32 %in) {
5646entry:
5647  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") release
5648  ret void
5649}
5650
5651define amdgpu_kernel void @flat_agent_one_as_acq_rel_atomicrmw(
5652; GFX7-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
5653; GFX7:       ; %bb.0: ; %entry
5654; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5655; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
5656; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5657; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5658; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5659; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5660; GFX7-NEXT:    s_waitcnt vmcnt(0)
5661; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
5662; GFX7-NEXT:    s_waitcnt vmcnt(0)
5663; GFX7-NEXT:    buffer_wbinvl1_vol
5664; GFX7-NEXT:    s_endpgm
5665;
5666; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
5667; GFX10-WGP:       ; %bb.0: ; %entry
5668; GFX10-WGP-NEXT:    s_clause 0x1
5669; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5670; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
5671; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5672; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5673; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5674; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5675; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5676; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5677; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
5678; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5679; GFX10-WGP-NEXT:    buffer_gl0_inv
5680; GFX10-WGP-NEXT:    buffer_gl1_inv
5681; GFX10-WGP-NEXT:    s_endpgm
5682;
5683; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
5684; GFX10-CU:       ; %bb.0: ; %entry
5685; GFX10-CU-NEXT:    s_clause 0x1
5686; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5687; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
5688; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5689; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5690; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5691; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5692; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5693; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5694; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
5695; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5696; GFX10-CU-NEXT:    buffer_gl0_inv
5697; GFX10-CU-NEXT:    buffer_gl1_inv
5698; GFX10-CU-NEXT:    s_endpgm
5699;
5700; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
5701; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5702; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5703; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
5704; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5705; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5706; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5707; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5708; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5709; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
5710; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5711; SKIP-CACHE-INV-NEXT:    s_endpgm
5712;
5713; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
5714; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5715; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5716; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5717; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5718; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5719; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5720; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5721; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
5722; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5723; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5724; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5725;
5726; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
5727; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5728; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5729; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5730; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5731; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5732; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5733; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5734; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
5735; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5736; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5737; GFX90A-TGSPLIT-NEXT:    s_endpgm
5738    i32* %out, i32 %in) {
5739entry:
5740  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") acq_rel
5741  ret void
5742}
5743
5744define amdgpu_kernel void @flat_agent_one_as_seq_cst_atomicrmw(
5745; GFX7-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
5746; GFX7:       ; %bb.0: ; %entry
5747; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5748; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
5749; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5750; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5751; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5752; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5753; GFX7-NEXT:    s_waitcnt vmcnt(0)
5754; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
5755; GFX7-NEXT:    s_waitcnt vmcnt(0)
5756; GFX7-NEXT:    buffer_wbinvl1_vol
5757; GFX7-NEXT:    s_endpgm
5758;
5759; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
5760; GFX10-WGP:       ; %bb.0: ; %entry
5761; GFX10-WGP-NEXT:    s_clause 0x1
5762; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5763; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
5764; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5765; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5766; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5767; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5768; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5769; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5770; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
5771; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5772; GFX10-WGP-NEXT:    buffer_gl0_inv
5773; GFX10-WGP-NEXT:    buffer_gl1_inv
5774; GFX10-WGP-NEXT:    s_endpgm
5775;
5776; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
5777; GFX10-CU:       ; %bb.0: ; %entry
5778; GFX10-CU-NEXT:    s_clause 0x1
5779; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5780; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
5781; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5782; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5783; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5784; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5785; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5786; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5787; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
5788; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5789; GFX10-CU-NEXT:    buffer_gl0_inv
5790; GFX10-CU-NEXT:    buffer_gl1_inv
5791; GFX10-CU-NEXT:    s_endpgm
5792;
5793; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
5794; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5795; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5796; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
5797; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5798; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5799; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5800; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5801; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5802; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
5803; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5804; SKIP-CACHE-INV-NEXT:    s_endpgm
5805;
5806; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
5807; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5808; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5809; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5810; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5811; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5812; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5813; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5814; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
5815; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5816; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5817; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5818;
5819; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
5820; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5821; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5822; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5823; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5824; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5825; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5826; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5827; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
5828; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5829; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5830; GFX90A-TGSPLIT-NEXT:    s_endpgm
5831    i32* %out, i32 %in) {
5832entry:
5833  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") seq_cst
5834  ret void
5835}
5836
5837define amdgpu_kernel void @flat_agent_one_as_acquire_ret_atomicrmw(
5838; GFX7-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
5839; GFX7:       ; %bb.0: ; %entry
5840; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5841; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
5842; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5843; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5844; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5845; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5846; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
5847; GFX7-NEXT:    s_waitcnt vmcnt(0)
5848; GFX7-NEXT:    buffer_wbinvl1_vol
5849; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5850; GFX7-NEXT:    flat_store_dword v[0:1], v2
5851; GFX7-NEXT:    s_endpgm
5852;
5853; GFX10-WGP-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
5854; GFX10-WGP:       ; %bb.0: ; %entry
5855; GFX10-WGP-NEXT:    s_clause 0x1
5856; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5857; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
5858; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5859; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5860; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5861; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5862; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
5863; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5864; GFX10-WGP-NEXT:    buffer_gl0_inv
5865; GFX10-WGP-NEXT:    buffer_gl1_inv
5866; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5867; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5868; GFX10-WGP-NEXT:    s_endpgm
5869;
5870; GFX10-CU-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
5871; GFX10-CU:       ; %bb.0: ; %entry
5872; GFX10-CU-NEXT:    s_clause 0x1
5873; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5874; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
5875; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5876; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5877; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5878; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5879; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
5880; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5881; GFX10-CU-NEXT:    buffer_gl0_inv
5882; GFX10-CU-NEXT:    buffer_gl1_inv
5883; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5884; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5885; GFX10-CU-NEXT:    s_endpgm
5886;
5887; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
5888; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5889; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5890; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
5891; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5892; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5893; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5894; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5895; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
5896; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5897; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5898; SKIP-CACHE-INV-NEXT:    s_endpgm
5899;
5900; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
5901; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5902; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5903; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5904; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5905; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5906; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5907; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
5908; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5909; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5910; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5911; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5912; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5913;
5914; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
5915; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5916; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5917; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5918; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5919; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5920; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
5921; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
5922; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5923; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5924; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5925; GFX90A-TGSPLIT-NEXT:    s_endpgm
5926    i32* %out, i32 %in) {
5927entry:
5928  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") acquire
5929  store i32 %val, i32* %out, align 4
5930  ret void
5931}
5932
5933define amdgpu_kernel void @flat_agent_one_as_acq_rel_ret_atomicrmw(
5934; GFX7-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
5935; GFX7:       ; %bb.0: ; %entry
5936; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5937; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
5938; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5939; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5940; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5941; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5942; GFX7-NEXT:    s_waitcnt vmcnt(0)
5943; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
5944; GFX7-NEXT:    s_waitcnt vmcnt(0)
5945; GFX7-NEXT:    buffer_wbinvl1_vol
5946; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5947; GFX7-NEXT:    flat_store_dword v[0:1], v2
5948; GFX7-NEXT:    s_endpgm
5949;
5950; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
5951; GFX10-WGP:       ; %bb.0: ; %entry
5952; GFX10-WGP-NEXT:    s_clause 0x1
5953; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5954; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
5955; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5956; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5957; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5958; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5959; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5960; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5961; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
5962; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5963; GFX10-WGP-NEXT:    buffer_gl0_inv
5964; GFX10-WGP-NEXT:    buffer_gl1_inv
5965; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5966; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5967; GFX10-WGP-NEXT:    s_endpgm
5968;
5969; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
5970; GFX10-CU:       ; %bb.0: ; %entry
5971; GFX10-CU-NEXT:    s_clause 0x1
5972; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5973; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
5974; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5975; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5976; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5977; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5978; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5979; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5980; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
5981; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5982; GFX10-CU-NEXT:    buffer_gl0_inv
5983; GFX10-CU-NEXT:    buffer_gl1_inv
5984; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5985; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5986; GFX10-CU-NEXT:    s_endpgm
5987;
5988; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
5989; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5990; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5991; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
5992; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5993; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5994; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5995; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5996; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5997; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
5998; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5999; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6000; SKIP-CACHE-INV-NEXT:    s_endpgm
6001;
6002; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
6003; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6004; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6005; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6006; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6007; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6008; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6009; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6010; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
6011; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6012; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6013; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6014; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6015; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6016;
6017; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
6018; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6019; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6020; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6021; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6022; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6023; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6024; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6025; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
6026; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6027; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6028; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6029; GFX90A-TGSPLIT-NEXT:    s_endpgm
6030    i32* %out, i32 %in) {
6031entry:
6032  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") acq_rel
6033  store i32 %val, i32* %out, align 4
6034  ret void
6035}
6036
6037define amdgpu_kernel void @flat_agent_one_as_seq_cst_ret_atomicrmw(
6038; GFX7-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
6039; GFX7:       ; %bb.0: ; %entry
6040; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6041; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
6042; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6043; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6044; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6045; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6046; GFX7-NEXT:    s_waitcnt vmcnt(0)
6047; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
6048; GFX7-NEXT:    s_waitcnt vmcnt(0)
6049; GFX7-NEXT:    buffer_wbinvl1_vol
6050; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6051; GFX7-NEXT:    flat_store_dword v[0:1], v2
6052; GFX7-NEXT:    s_endpgm
6053;
6054; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
6055; GFX10-WGP:       ; %bb.0: ; %entry
6056; GFX10-WGP-NEXT:    s_clause 0x1
6057; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6058; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
6059; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6060; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6061; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6062; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6063; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6064; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6065; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
6066; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6067; GFX10-WGP-NEXT:    buffer_gl0_inv
6068; GFX10-WGP-NEXT:    buffer_gl1_inv
6069; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6070; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6071; GFX10-WGP-NEXT:    s_endpgm
6072;
6073; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
6074; GFX10-CU:       ; %bb.0: ; %entry
6075; GFX10-CU-NEXT:    s_clause 0x1
6076; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6077; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
6078; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6079; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6080; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6081; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6082; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6083; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6084; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
6085; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6086; GFX10-CU-NEXT:    buffer_gl0_inv
6087; GFX10-CU-NEXT:    buffer_gl1_inv
6088; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6089; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6090; GFX10-CU-NEXT:    s_endpgm
6091;
6092; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
6093; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6094; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6095; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
6096; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6097; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6098; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6099; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6100; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6101; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
6102; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6103; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6104; SKIP-CACHE-INV-NEXT:    s_endpgm
6105;
6106; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
6107; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6108; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6109; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6110; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6111; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6112; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6113; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6114; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
6115; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6116; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6117; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6118; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6119; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6120;
6121; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
6122; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6123; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6124; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6125; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6126; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6127; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6128; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6129; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
6130; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6131; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6132; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6133; GFX90A-TGSPLIT-NEXT:    s_endpgm
6134    i32* %out, i32 %in) {
6135entry:
6136  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") seq_cst
6137  store i32 %val, i32* %out, align 4
6138  ret void
6139}
6140
6141define amdgpu_kernel void @flat_agent_one_as_monotonic_monotonic_cmpxchg(
6142; GFX7-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
6143; GFX7:       ; %bb.0: ; %entry
6144; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6145; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6146; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6147; GFX7-NEXT:    s_add_u32 s0, s0, 16
6148; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6149; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6150; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6151; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6152; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6153; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6154; GFX7-NEXT:    s_endpgm
6155;
6156; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
6157; GFX10-WGP:       ; %bb.0: ; %entry
6158; GFX10-WGP-NEXT:    s_clause 0x1
6159; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6160; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6161; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6162; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
6163; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
6164; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6165; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6166; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6167; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6168; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6169; GFX10-WGP-NEXT:    s_endpgm
6170;
6171; GFX10-CU-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
6172; GFX10-CU:       ; %bb.0: ; %entry
6173; GFX10-CU-NEXT:    s_clause 0x1
6174; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6175; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6176; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6177; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
6178; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
6179; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6180; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6181; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6182; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6183; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6184; GFX10-CU-NEXT:    s_endpgm
6185;
6186; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
6187; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6188; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6189; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6190; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6191; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
6192; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
6193; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6194; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6195; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6196; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6197; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6198; SKIP-CACHE-INV-NEXT:    s_endpgm
6199;
6200; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
6201; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6202; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6203; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6204; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6205; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6206; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6207; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6208; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6209;
6210; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
6211; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6212; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6213; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6214; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6215; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6216; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6217; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6218; GFX90A-TGSPLIT-NEXT:    s_endpgm
6219    i32* %out, i32 %in, i32 %old) {
6220entry:
6221  %gep = getelementptr i32, i32* %out, i32 4
6222  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") monotonic monotonic
6223  ret void
6224}
6225
6226define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_cmpxchg(
6227; GFX7-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
6228; GFX7:       ; %bb.0: ; %entry
6229; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6230; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6231; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6232; GFX7-NEXT:    s_add_u32 s0, s0, 16
6233; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6234; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6235; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6236; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6237; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6238; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6239; GFX7-NEXT:    s_waitcnt vmcnt(0)
6240; GFX7-NEXT:    buffer_wbinvl1_vol
6241; GFX7-NEXT:    s_endpgm
6242;
6243; GFX10-WGP-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
6244; GFX10-WGP:       ; %bb.0: ; %entry
6245; GFX10-WGP-NEXT:    s_clause 0x1
6246; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6247; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6248; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6249; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
6250; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
6251; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6252; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6253; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6254; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6255; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6256; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6257; GFX10-WGP-NEXT:    buffer_gl0_inv
6258; GFX10-WGP-NEXT:    buffer_gl1_inv
6259; GFX10-WGP-NEXT:    s_endpgm
6260;
6261; GFX10-CU-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
6262; GFX10-CU:       ; %bb.0: ; %entry
6263; GFX10-CU-NEXT:    s_clause 0x1
6264; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6265; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6266; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6267; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
6268; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
6269; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6270; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6271; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6272; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6273; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6274; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6275; GFX10-CU-NEXT:    buffer_gl0_inv
6276; GFX10-CU-NEXT:    buffer_gl1_inv
6277; GFX10-CU-NEXT:    s_endpgm
6278;
6279; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
6280; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6281; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6282; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6283; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6284; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
6285; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
6286; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6287; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6288; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6289; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6290; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6291; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6292; SKIP-CACHE-INV-NEXT:    s_endpgm
6293;
6294; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
6295; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6296; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6297; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6298; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6299; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6300; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6301; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6302; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6303; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6304; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6305;
6306; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
6307; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6308; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6309; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6310; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6311; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6312; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6313; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6314; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6315; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6316; GFX90A-TGSPLIT-NEXT:    s_endpgm
6317    i32* %out, i32 %in, i32 %old) {
6318entry:
6319  %gep = getelementptr i32, i32* %out, i32 4
6320  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire monotonic
6321  ret void
6322}
6323
6324define amdgpu_kernel void @flat_agent_one_as_release_monotonic_cmpxchg(
6325; GFX7-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
6326; GFX7:       ; %bb.0: ; %entry
6327; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6328; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6329; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6330; GFX7-NEXT:    s_add_u32 s0, s0, 16
6331; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6332; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6333; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6334; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6335; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6336; GFX7-NEXT:    s_waitcnt vmcnt(0)
6337; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6338; GFX7-NEXT:    s_endpgm
6339;
6340; GFX10-WGP-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
6341; GFX10-WGP:       ; %bb.0: ; %entry
6342; GFX10-WGP-NEXT:    s_clause 0x1
6343; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6344; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6345; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6346; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
6347; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
6348; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6349; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6350; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6351; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6352; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6353; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6354; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6355; GFX10-WGP-NEXT:    s_endpgm
6356;
6357; GFX10-CU-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
6358; GFX10-CU:       ; %bb.0: ; %entry
6359; GFX10-CU-NEXT:    s_clause 0x1
6360; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6361; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6362; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6363; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
6364; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
6365; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6366; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6367; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6368; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6369; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6370; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6371; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6372; GFX10-CU-NEXT:    s_endpgm
6373;
6374; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
6375; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6376; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6377; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6378; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6379; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
6380; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
6381; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6382; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6383; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6384; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6385; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6386; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6387; SKIP-CACHE-INV-NEXT:    s_endpgm
6388;
6389; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
6390; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6391; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6392; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6393; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6394; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6395; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6396; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6397; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6398; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6399;
6400; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
6401; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6402; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6403; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6404; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6405; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6406; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6407; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6408; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6409; GFX90A-TGSPLIT-NEXT:    s_endpgm
6410    i32* %out, i32 %in, i32 %old) {
6411entry:
6412  %gep = getelementptr i32, i32* %out, i32 4
6413  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release monotonic
6414  ret void
6415}
6416
6417define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_cmpxchg(
6418; GFX7-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
6419; GFX7:       ; %bb.0: ; %entry
6420; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6421; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6422; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6423; GFX7-NEXT:    s_add_u32 s0, s0, 16
6424; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6425; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6426; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6427; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6428; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6429; GFX7-NEXT:    s_waitcnt vmcnt(0)
6430; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6431; GFX7-NEXT:    s_waitcnt vmcnt(0)
6432; GFX7-NEXT:    buffer_wbinvl1_vol
6433; GFX7-NEXT:    s_endpgm
6434;
6435; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
6436; GFX10-WGP:       ; %bb.0: ; %entry
6437; GFX10-WGP-NEXT:    s_clause 0x1
6438; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6439; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6440; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6441; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
6442; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
6443; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6444; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6445; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6446; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6447; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6448; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6449; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6450; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6451; GFX10-WGP-NEXT:    buffer_gl0_inv
6452; GFX10-WGP-NEXT:    buffer_gl1_inv
6453; GFX10-WGP-NEXT:    s_endpgm
6454;
6455; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
6456; GFX10-CU:       ; %bb.0: ; %entry
6457; GFX10-CU-NEXT:    s_clause 0x1
6458; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6459; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6460; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6461; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
6462; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
6463; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6464; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6465; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6466; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6467; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6468; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6469; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6470; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6471; GFX10-CU-NEXT:    buffer_gl0_inv
6472; GFX10-CU-NEXT:    buffer_gl1_inv
6473; GFX10-CU-NEXT:    s_endpgm
6474;
6475; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
6476; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6477; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6478; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6479; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6480; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
6481; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
6482; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6483; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6484; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6485; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6486; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6487; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6488; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6489; SKIP-CACHE-INV-NEXT:    s_endpgm
6490;
6491; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
6492; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6493; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6494; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6495; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6496; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6497; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6498; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6499; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6500; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6501; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6502; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6503;
6504; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
6505; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6506; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6507; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6508; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6509; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6510; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6511; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6512; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6513; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6514; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6515; GFX90A-TGSPLIT-NEXT:    s_endpgm
6516    i32* %out, i32 %in, i32 %old) {
6517entry:
6518  %gep = getelementptr i32, i32* %out, i32 4
6519  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel monotonic
6520  ret void
6521}
6522
6523define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_cmpxchg(
6524; GFX7-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
6525; GFX7:       ; %bb.0: ; %entry
6526; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6527; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6528; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6529; GFX7-NEXT:    s_add_u32 s0, s0, 16
6530; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6531; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6532; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6533; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6534; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6535; GFX7-NEXT:    s_waitcnt vmcnt(0)
6536; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6537; GFX7-NEXT:    s_waitcnt vmcnt(0)
6538; GFX7-NEXT:    buffer_wbinvl1_vol
6539; GFX7-NEXT:    s_endpgm
6540;
6541; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
6542; GFX10-WGP:       ; %bb.0: ; %entry
6543; GFX10-WGP-NEXT:    s_clause 0x1
6544; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6545; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6546; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6547; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
6548; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
6549; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6550; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6551; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6552; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6553; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6554; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6555; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6556; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6557; GFX10-WGP-NEXT:    buffer_gl0_inv
6558; GFX10-WGP-NEXT:    buffer_gl1_inv
6559; GFX10-WGP-NEXT:    s_endpgm
6560;
6561; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
6562; GFX10-CU:       ; %bb.0: ; %entry
6563; GFX10-CU-NEXT:    s_clause 0x1
6564; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6565; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6566; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6567; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
6568; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
6569; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6570; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6571; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6572; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6573; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6574; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6575; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6576; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6577; GFX10-CU-NEXT:    buffer_gl0_inv
6578; GFX10-CU-NEXT:    buffer_gl1_inv
6579; GFX10-CU-NEXT:    s_endpgm
6580;
6581; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
6582; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6583; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6584; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6585; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6586; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
6587; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
6588; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6589; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6590; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6591; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6592; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6593; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6594; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6595; SKIP-CACHE-INV-NEXT:    s_endpgm
6596;
6597; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
6598; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6599; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6600; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6601; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6602; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6603; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6604; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6605; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6606; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6607; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6608; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6609;
6610; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
6611; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6612; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6613; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6614; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6615; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6616; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6617; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6618; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6619; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6620; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6621; GFX90A-TGSPLIT-NEXT:    s_endpgm
6622    i32* %out, i32 %in, i32 %old) {
6623entry:
6624  %gep = getelementptr i32, i32* %out, i32 4
6625  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst monotonic
6626  ret void
6627}
6628
6629define amdgpu_kernel void @flat_agent_one_as_monotonic_acquire_cmpxchg(
6630; GFX7-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
6631; GFX7:       ; %bb.0: ; %entry
6632; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6633; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6634; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6635; GFX7-NEXT:    s_add_u32 s0, s0, 16
6636; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6637; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6638; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6639; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6640; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6641; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6642; GFX7-NEXT:    s_waitcnt vmcnt(0)
6643; GFX7-NEXT:    buffer_wbinvl1_vol
6644; GFX7-NEXT:    s_endpgm
6645;
6646; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
6647; GFX10-WGP:       ; %bb.0: ; %entry
6648; GFX10-WGP-NEXT:    s_clause 0x1
6649; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6650; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6651; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6652; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
6653; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
6654; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6655; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6656; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6657; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6658; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6659; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6660; GFX10-WGP-NEXT:    buffer_gl0_inv
6661; GFX10-WGP-NEXT:    buffer_gl1_inv
6662; GFX10-WGP-NEXT:    s_endpgm
6663;
6664; GFX10-CU-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
6665; GFX10-CU:       ; %bb.0: ; %entry
6666; GFX10-CU-NEXT:    s_clause 0x1
6667; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6668; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6669; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6670; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
6671; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
6672; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6673; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6674; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6675; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6676; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6677; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6678; GFX10-CU-NEXT:    buffer_gl0_inv
6679; GFX10-CU-NEXT:    buffer_gl1_inv
6680; GFX10-CU-NEXT:    s_endpgm
6681;
6682; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
6683; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6684; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6685; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6686; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6687; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
6688; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
6689; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6690; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6691; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6692; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6693; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6694; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6695; SKIP-CACHE-INV-NEXT:    s_endpgm
6696;
6697; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
6698; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6699; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6700; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6701; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6702; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6703; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6704; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6705; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6706; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6707; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6708;
6709; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
6710; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6711; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6712; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6713; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6714; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6715; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6716; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6717; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6718; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6719; GFX90A-TGSPLIT-NEXT:    s_endpgm
6720    i32* %out, i32 %in, i32 %old) {
6721entry:
6722  %gep = getelementptr i32, i32* %out, i32 4
6723  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") monotonic acquire
6724  ret void
6725}
6726
6727define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_cmpxchg(
6728; GFX7-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
6729; GFX7:       ; %bb.0: ; %entry
6730; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6731; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6732; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6733; GFX7-NEXT:    s_add_u32 s0, s0, 16
6734; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6735; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6736; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6737; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6738; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6739; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6740; GFX7-NEXT:    s_waitcnt vmcnt(0)
6741; GFX7-NEXT:    buffer_wbinvl1_vol
6742; GFX7-NEXT:    s_endpgm
6743;
6744; GFX10-WGP-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
6745; GFX10-WGP:       ; %bb.0: ; %entry
6746; GFX10-WGP-NEXT:    s_clause 0x1
6747; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6748; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6749; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6750; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
6751; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
6752; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6753; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6754; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6755; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6756; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6757; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6758; GFX10-WGP-NEXT:    buffer_gl0_inv
6759; GFX10-WGP-NEXT:    buffer_gl1_inv
6760; GFX10-WGP-NEXT:    s_endpgm
6761;
6762; GFX10-CU-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
6763; GFX10-CU:       ; %bb.0: ; %entry
6764; GFX10-CU-NEXT:    s_clause 0x1
6765; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6766; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6767; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6768; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
6769; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
6770; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6771; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6772; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6773; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6774; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6775; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6776; GFX10-CU-NEXT:    buffer_gl0_inv
6777; GFX10-CU-NEXT:    buffer_gl1_inv
6778; GFX10-CU-NEXT:    s_endpgm
6779;
6780; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
6781; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6782; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6783; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6784; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6785; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
6786; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
6787; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6788; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6789; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6790; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6791; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6792; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6793; SKIP-CACHE-INV-NEXT:    s_endpgm
6794;
6795; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
6796; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6797; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6798; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6799; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6800; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6801; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6802; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6803; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6804; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6805; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6806;
6807; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
6808; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6809; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6810; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6811; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6812; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6813; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6814; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6815; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6816; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6817; GFX90A-TGSPLIT-NEXT:    s_endpgm
6818    i32* %out, i32 %in, i32 %old) {
6819entry:
6820  %gep = getelementptr i32, i32* %out, i32 4
6821  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire acquire
6822  ret void
6823}
6824
6825define amdgpu_kernel void @flat_agent_one_as_release_acquire_cmpxchg(
6826; GFX7-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
6827; GFX7:       ; %bb.0: ; %entry
6828; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6829; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6830; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6831; GFX7-NEXT:    s_add_u32 s0, s0, 16
6832; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6833; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6834; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6835; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6836; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6837; GFX7-NEXT:    s_waitcnt vmcnt(0)
6838; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6839; GFX7-NEXT:    s_waitcnt vmcnt(0)
6840; GFX7-NEXT:    buffer_wbinvl1_vol
6841; GFX7-NEXT:    s_endpgm
6842;
6843; GFX10-WGP-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
6844; GFX10-WGP:       ; %bb.0: ; %entry
6845; GFX10-WGP-NEXT:    s_clause 0x1
6846; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6847; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6848; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6849; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
6850; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
6851; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6852; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6853; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6854; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6855; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6856; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6857; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6858; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6859; GFX10-WGP-NEXT:    buffer_gl0_inv
6860; GFX10-WGP-NEXT:    buffer_gl1_inv
6861; GFX10-WGP-NEXT:    s_endpgm
6862;
6863; GFX10-CU-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
6864; GFX10-CU:       ; %bb.0: ; %entry
6865; GFX10-CU-NEXT:    s_clause 0x1
6866; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6867; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6868; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6869; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
6870; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
6871; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6872; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6873; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6874; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6875; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6876; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6877; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6878; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6879; GFX10-CU-NEXT:    buffer_gl0_inv
6880; GFX10-CU-NEXT:    buffer_gl1_inv
6881; GFX10-CU-NEXT:    s_endpgm
6882;
6883; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
6884; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6885; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6886; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6887; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6888; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
6889; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
6890; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6891; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6892; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6893; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6894; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6895; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6896; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6897; SKIP-CACHE-INV-NEXT:    s_endpgm
6898;
6899; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
6900; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6901; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6902; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6903; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6904; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6905; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6906; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6907; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6908; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6909; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6910; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6911;
6912; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
6913; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6914; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6915; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6916; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6917; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6918; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6919; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6920; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
6921; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6922; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6923; GFX90A-TGSPLIT-NEXT:    s_endpgm
6924    i32* %out, i32 %in, i32 %old) {
6925entry:
6926  %gep = getelementptr i32, i32* %out, i32 4
6927  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release acquire
6928  ret void
6929}
6930
6931define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_cmpxchg(
6932; GFX7-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
6933; GFX7:       ; %bb.0: ; %entry
6934; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6935; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6936; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6937; GFX7-NEXT:    s_add_u32 s0, s0, 16
6938; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6939; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6940; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6941; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6942; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6943; GFX7-NEXT:    s_waitcnt vmcnt(0)
6944; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6945; GFX7-NEXT:    s_waitcnt vmcnt(0)
6946; GFX7-NEXT:    buffer_wbinvl1_vol
6947; GFX7-NEXT:    s_endpgm
6948;
6949; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
6950; GFX10-WGP:       ; %bb.0: ; %entry
6951; GFX10-WGP-NEXT:    s_clause 0x1
6952; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6953; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6954; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6955; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
6956; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
6957; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6958; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6959; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6960; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6961; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6962; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6963; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6964; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6965; GFX10-WGP-NEXT:    buffer_gl0_inv
6966; GFX10-WGP-NEXT:    buffer_gl1_inv
6967; GFX10-WGP-NEXT:    s_endpgm
6968;
6969; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
6970; GFX10-CU:       ; %bb.0: ; %entry
6971; GFX10-CU-NEXT:    s_clause 0x1
6972; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6973; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6974; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6975; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
6976; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
6977; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6978; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6979; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6980; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6981; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6982; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6983; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6984; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6985; GFX10-CU-NEXT:    buffer_gl0_inv
6986; GFX10-CU-NEXT:    buffer_gl1_inv
6987; GFX10-CU-NEXT:    s_endpgm
6988;
6989; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
6990; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6991; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6992; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6993; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6994; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
6995; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
6996; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6997; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6998; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6999; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7000; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7001; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7002; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7003; SKIP-CACHE-INV-NEXT:    s_endpgm
7004;
7005; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
7006; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7007; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7008; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7009; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7010; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7011; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7012; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7013; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7014; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7015; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7016; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7017;
7018; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
7019; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7020; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7021; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7022; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7023; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7024; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7025; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7026; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7027; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7028; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7029; GFX90A-TGSPLIT-NEXT:    s_endpgm
7030    i32* %out, i32 %in, i32 %old) {
7031entry:
7032  %gep = getelementptr i32, i32* %out, i32 4
7033  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel acquire
7034  ret void
7035}
7036
7037define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_cmpxchg(
7038; GFX7-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
7039; GFX7:       ; %bb.0: ; %entry
7040; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7041; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7042; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7043; GFX7-NEXT:    s_add_u32 s0, s0, 16
7044; GFX7-NEXT:    s_addc_u32 s1, s1, 0
7045; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7046; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7047; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7048; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7049; GFX7-NEXT:    s_waitcnt vmcnt(0)
7050; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7051; GFX7-NEXT:    s_waitcnt vmcnt(0)
7052; GFX7-NEXT:    buffer_wbinvl1_vol
7053; GFX7-NEXT:    s_endpgm
7054;
7055; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
7056; GFX10-WGP:       ; %bb.0: ; %entry
7057; GFX10-WGP-NEXT:    s_clause 0x1
7058; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7059; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7060; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7061; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
7062; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
7063; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7064; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7065; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7066; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7067; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7068; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7069; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7070; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7071; GFX10-WGP-NEXT:    buffer_gl0_inv
7072; GFX10-WGP-NEXT:    buffer_gl1_inv
7073; GFX10-WGP-NEXT:    s_endpgm
7074;
7075; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
7076; GFX10-CU:       ; %bb.0: ; %entry
7077; GFX10-CU-NEXT:    s_clause 0x1
7078; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7079; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7080; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7081; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
7082; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
7083; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7084; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7085; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7086; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7087; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7088; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7089; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7090; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7091; GFX10-CU-NEXT:    buffer_gl0_inv
7092; GFX10-CU-NEXT:    buffer_gl1_inv
7093; GFX10-CU-NEXT:    s_endpgm
7094;
7095; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
7096; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7097; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7098; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7099; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7100; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
7101; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
7102; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7103; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7104; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7105; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7106; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7107; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7108; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7109; SKIP-CACHE-INV-NEXT:    s_endpgm
7110;
7111; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
7112; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7113; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7114; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7115; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7116; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7117; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7118; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7119; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7120; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7121; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7122; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7123;
7124; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
7125; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7126; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7127; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7128; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7129; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7130; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7131; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7132; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7133; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7134; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7135; GFX90A-TGSPLIT-NEXT:    s_endpgm
7136    i32* %out, i32 %in, i32 %old) {
7137entry:
7138  %gep = getelementptr i32, i32* %out, i32 4
7139  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst acquire
7140  ret void
7141}
7142
7143define amdgpu_kernel void @flat_agent_one_as_monotonic_seq_cst_cmpxchg(
7144; GFX7-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
7145; GFX7:       ; %bb.0: ; %entry
7146; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7147; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7148; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7149; GFX7-NEXT:    s_add_u32 s0, s0, 16
7150; GFX7-NEXT:    s_addc_u32 s1, s1, 0
7151; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7152; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7153; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7154; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7155; GFX7-NEXT:    s_waitcnt vmcnt(0)
7156; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7157; GFX7-NEXT:    s_waitcnt vmcnt(0)
7158; GFX7-NEXT:    buffer_wbinvl1_vol
7159; GFX7-NEXT:    s_endpgm
7160;
7161; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
7162; GFX10-WGP:       ; %bb.0: ; %entry
7163; GFX10-WGP-NEXT:    s_clause 0x1
7164; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7165; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7166; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7167; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
7168; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
7169; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7170; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7171; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7172; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7173; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7174; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7175; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7176; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7177; GFX10-WGP-NEXT:    buffer_gl0_inv
7178; GFX10-WGP-NEXT:    buffer_gl1_inv
7179; GFX10-WGP-NEXT:    s_endpgm
7180;
7181; GFX10-CU-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
7182; GFX10-CU:       ; %bb.0: ; %entry
7183; GFX10-CU-NEXT:    s_clause 0x1
7184; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7185; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7186; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7187; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
7188; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
7189; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7190; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7191; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7192; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7193; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7194; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7195; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7196; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7197; GFX10-CU-NEXT:    buffer_gl0_inv
7198; GFX10-CU-NEXT:    buffer_gl1_inv
7199; GFX10-CU-NEXT:    s_endpgm
7200;
7201; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
7202; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7203; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7204; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7205; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7206; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
7207; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
7208; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7209; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7210; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7211; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7212; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7213; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7214; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7215; SKIP-CACHE-INV-NEXT:    s_endpgm
7216;
7217; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
7218; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7219; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7220; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7221; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7222; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7223; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7224; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7225; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7226; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7227; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7228; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7229;
7230; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
7231; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7232; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7233; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7234; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7235; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7236; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7237; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7238; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7239; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7240; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7241; GFX90A-TGSPLIT-NEXT:    s_endpgm
7242    i32* %out, i32 %in, i32 %old) {
7243entry:
7244  %gep = getelementptr i32, i32* %out, i32 4
7245  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") monotonic seq_cst
7246  ret void
7247}
7248
7249define amdgpu_kernel void @flat_agent_one_as_acquire_seq_cst_cmpxchg(
7250; GFX7-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
7251; GFX7:       ; %bb.0: ; %entry
7252; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7253; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7254; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7255; GFX7-NEXT:    s_add_u32 s0, s0, 16
7256; GFX7-NEXT:    s_addc_u32 s1, s1, 0
7257; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7258; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7259; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7260; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7261; GFX7-NEXT:    s_waitcnt vmcnt(0)
7262; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7263; GFX7-NEXT:    s_waitcnt vmcnt(0)
7264; GFX7-NEXT:    buffer_wbinvl1_vol
7265; GFX7-NEXT:    s_endpgm
7266;
7267; GFX10-WGP-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
7268; GFX10-WGP:       ; %bb.0: ; %entry
7269; GFX10-WGP-NEXT:    s_clause 0x1
7270; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7271; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7272; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7273; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
7274; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
7275; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7276; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7277; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7278; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7279; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7280; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7281; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7282; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7283; GFX10-WGP-NEXT:    buffer_gl0_inv
7284; GFX10-WGP-NEXT:    buffer_gl1_inv
7285; GFX10-WGP-NEXT:    s_endpgm
7286;
7287; GFX10-CU-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
7288; GFX10-CU:       ; %bb.0: ; %entry
7289; GFX10-CU-NEXT:    s_clause 0x1
7290; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7291; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7292; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7293; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
7294; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
7295; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7296; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7297; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7298; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7299; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7300; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7301; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7302; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7303; GFX10-CU-NEXT:    buffer_gl0_inv
7304; GFX10-CU-NEXT:    buffer_gl1_inv
7305; GFX10-CU-NEXT:    s_endpgm
7306;
7307; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
7308; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7309; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7310; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7311; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7312; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
7313; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
7314; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7315; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7316; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7317; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7318; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7319; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7320; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7321; SKIP-CACHE-INV-NEXT:    s_endpgm
7322;
7323; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
7324; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7325; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7326; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7327; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7328; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7329; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7330; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7331; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7332; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7333; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7334; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7335;
7336; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
7337; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7338; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7339; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7340; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7341; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7342; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7343; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7344; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7345; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7346; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7347; GFX90A-TGSPLIT-NEXT:    s_endpgm
7348    i32* %out, i32 %in, i32 %old) {
7349entry:
7350  %gep = getelementptr i32, i32* %out, i32 4
7351  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire seq_cst
7352  ret void
7353}
7354
7355define amdgpu_kernel void @flat_agent_one_as_release_seq_cst_cmpxchg(
7356; GFX7-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
7357; GFX7:       ; %bb.0: ; %entry
7358; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7359; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7360; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7361; GFX7-NEXT:    s_add_u32 s0, s0, 16
7362; GFX7-NEXT:    s_addc_u32 s1, s1, 0
7363; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7364; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7365; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7366; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7367; GFX7-NEXT:    s_waitcnt vmcnt(0)
7368; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7369; GFX7-NEXT:    s_waitcnt vmcnt(0)
7370; GFX7-NEXT:    buffer_wbinvl1_vol
7371; GFX7-NEXT:    s_endpgm
7372;
7373; GFX10-WGP-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
7374; GFX10-WGP:       ; %bb.0: ; %entry
7375; GFX10-WGP-NEXT:    s_clause 0x1
7376; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7377; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7378; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7379; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
7380; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
7381; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7382; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7383; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7384; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7385; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7386; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7387; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7388; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7389; GFX10-WGP-NEXT:    buffer_gl0_inv
7390; GFX10-WGP-NEXT:    buffer_gl1_inv
7391; GFX10-WGP-NEXT:    s_endpgm
7392;
7393; GFX10-CU-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
7394; GFX10-CU:       ; %bb.0: ; %entry
7395; GFX10-CU-NEXT:    s_clause 0x1
7396; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7397; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7398; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7399; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
7400; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
7401; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7402; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7403; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7404; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7405; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7406; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7407; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7408; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7409; GFX10-CU-NEXT:    buffer_gl0_inv
7410; GFX10-CU-NEXT:    buffer_gl1_inv
7411; GFX10-CU-NEXT:    s_endpgm
7412;
7413; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
7414; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7415; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7416; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7417; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7418; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
7419; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
7420; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7421; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7422; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7423; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7424; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7425; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7426; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7427; SKIP-CACHE-INV-NEXT:    s_endpgm
7428;
7429; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
7430; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7431; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7432; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7433; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7434; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7435; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7436; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7437; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7438; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7439; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7440; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7441;
7442; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
7443; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7444; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7445; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7446; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7447; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7448; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7449; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7450; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7451; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7452; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7453; GFX90A-TGSPLIT-NEXT:    s_endpgm
7454    i32* %out, i32 %in, i32 %old) {
7455entry:
7456  %gep = getelementptr i32, i32* %out, i32 4
7457  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release seq_cst
7458  ret void
7459}
7460
7461define amdgpu_kernel void @flat_agent_one_as_acq_rel_seq_cst_cmpxchg(
7462; GFX7-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
7463; GFX7:       ; %bb.0: ; %entry
7464; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7465; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7466; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7467; GFX7-NEXT:    s_add_u32 s0, s0, 16
7468; GFX7-NEXT:    s_addc_u32 s1, s1, 0
7469; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7470; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7471; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7472; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7473; GFX7-NEXT:    s_waitcnt vmcnt(0)
7474; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7475; GFX7-NEXT:    s_waitcnt vmcnt(0)
7476; GFX7-NEXT:    buffer_wbinvl1_vol
7477; GFX7-NEXT:    s_endpgm
7478;
7479; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
7480; GFX10-WGP:       ; %bb.0: ; %entry
7481; GFX10-WGP-NEXT:    s_clause 0x1
7482; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7483; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7484; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7485; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
7486; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
7487; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7488; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7489; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7490; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7491; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7492; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7493; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7494; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7495; GFX10-WGP-NEXT:    buffer_gl0_inv
7496; GFX10-WGP-NEXT:    buffer_gl1_inv
7497; GFX10-WGP-NEXT:    s_endpgm
7498;
7499; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
7500; GFX10-CU:       ; %bb.0: ; %entry
7501; GFX10-CU-NEXT:    s_clause 0x1
7502; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7503; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7504; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7505; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
7506; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
7507; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7508; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7509; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7510; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7511; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7512; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7513; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7514; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7515; GFX10-CU-NEXT:    buffer_gl0_inv
7516; GFX10-CU-NEXT:    buffer_gl1_inv
7517; GFX10-CU-NEXT:    s_endpgm
7518;
7519; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
7520; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7521; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7522; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7523; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7524; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
7525; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
7526; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7527; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7528; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7529; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7530; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7531; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7532; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7533; SKIP-CACHE-INV-NEXT:    s_endpgm
7534;
7535; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
7536; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7537; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7538; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7539; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7540; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7541; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7542; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7543; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7544; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7545; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7546; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7547;
7548; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
7549; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7550; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7551; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7552; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7553; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7554; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7555; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7556; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7557; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7558; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7559; GFX90A-TGSPLIT-NEXT:    s_endpgm
7560    i32* %out, i32 %in, i32 %old) {
7561entry:
7562  %gep = getelementptr i32, i32* %out, i32 4
7563  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel seq_cst
7564  ret void
7565}
7566
7567define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_cmpxchg(
7568; GFX7-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
7569; GFX7:       ; %bb.0: ; %entry
7570; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7571; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7572; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7573; GFX7-NEXT:    s_add_u32 s0, s0, 16
7574; GFX7-NEXT:    s_addc_u32 s1, s1, 0
7575; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7576; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7577; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7578; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7579; GFX7-NEXT:    s_waitcnt vmcnt(0)
7580; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7581; GFX7-NEXT:    s_waitcnt vmcnt(0)
7582; GFX7-NEXT:    buffer_wbinvl1_vol
7583; GFX7-NEXT:    s_endpgm
7584;
7585; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
7586; GFX10-WGP:       ; %bb.0: ; %entry
7587; GFX10-WGP-NEXT:    s_clause 0x1
7588; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7589; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7590; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7591; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
7592; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
7593; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7594; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7595; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7596; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7597; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7598; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7599; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7600; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7601; GFX10-WGP-NEXT:    buffer_gl0_inv
7602; GFX10-WGP-NEXT:    buffer_gl1_inv
7603; GFX10-WGP-NEXT:    s_endpgm
7604;
7605; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
7606; GFX10-CU:       ; %bb.0: ; %entry
7607; GFX10-CU-NEXT:    s_clause 0x1
7608; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7609; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7610; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7611; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
7612; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
7613; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7614; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7615; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7616; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7617; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7618; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7619; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7620; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7621; GFX10-CU-NEXT:    buffer_gl0_inv
7622; GFX10-CU-NEXT:    buffer_gl1_inv
7623; GFX10-CU-NEXT:    s_endpgm
7624;
7625; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
7626; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7627; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7628; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7629; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7630; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
7631; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
7632; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7633; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7634; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7635; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7636; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7637; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7638; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7639; SKIP-CACHE-INV-NEXT:    s_endpgm
7640;
7641; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
7642; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7643; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7644; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7645; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7646; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7647; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7648; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7649; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7650; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7651; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7652; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7653;
7654; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
7655; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7656; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7657; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7658; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7659; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7660; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7661; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7662; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7663; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7664; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7665; GFX90A-TGSPLIT-NEXT:    s_endpgm
7666    i32* %out, i32 %in, i32 %old) {
7667entry:
7668  %gep = getelementptr i32, i32* %out, i32 4
7669  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst seq_cst
7670  ret void
7671}
7672
7673define amdgpu_kernel void @flat_agent_one_as_monotonic_monotonic_ret_cmpxchg(
7674; GFX7-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
7675; GFX7:       ; %bb.0: ; %entry
7676; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7677; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7678; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7679; GFX7-NEXT:    s_add_u32 s4, s0, 16
7680; GFX7-NEXT:    s_addc_u32 s5, s1, 0
7681; GFX7-NEXT:    v_mov_b32_e32 v0, s4
7682; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7683; GFX7-NEXT:    v_mov_b32_e32 v1, s5
7684; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7685; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7686; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7687; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7688; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7689; GFX7-NEXT:    flat_store_dword v[0:1], v2
7690; GFX7-NEXT:    s_endpgm
7691;
7692; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
7693; GFX10-WGP:       ; %bb.0: ; %entry
7694; GFX10-WGP-NEXT:    s_clause 0x1
7695; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7696; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7697; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7698; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
7699; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
7700; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
7701; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7702; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
7703; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7704; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7705; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7706; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7707; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7708; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7709; GFX10-WGP-NEXT:    s_endpgm
7710;
7711; GFX10-CU-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
7712; GFX10-CU:       ; %bb.0: ; %entry
7713; GFX10-CU-NEXT:    s_clause 0x1
7714; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7715; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7716; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7717; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
7718; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
7719; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
7720; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7721; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
7722; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7723; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7724; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7725; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7726; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7727; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7728; GFX10-CU-NEXT:    s_endpgm
7729;
7730; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
7731; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7732; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7733; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7734; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7735; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
7736; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
7737; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7738; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7739; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7740; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7741; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7742; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7743; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7744; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7745; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7746; SKIP-CACHE-INV-NEXT:    s_endpgm
7747;
7748; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
7749; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7750; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7751; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7752; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7753; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7754; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7755; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7756; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7757; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7758; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7759;
7760; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
7761; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7762; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7763; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7764; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7765; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7766; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7767; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7768; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7769; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7770; GFX90A-TGSPLIT-NEXT:    s_endpgm
7771    i32* %out, i32 %in, i32 %old) {
7772entry:
7773  %gep = getelementptr i32, i32* %out, i32 4
7774  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") monotonic monotonic
7775  %val0 = extractvalue { i32, i1 } %val, 0
7776  store i32 %val0, i32* %out, align 4
7777  ret void
7778}
7779
7780define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_ret_cmpxchg(
7781; GFX7-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
7782; GFX7:       ; %bb.0: ; %entry
7783; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7784; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7785; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7786; GFX7-NEXT:    s_add_u32 s4, s0, 16
7787; GFX7-NEXT:    s_addc_u32 s5, s1, 0
7788; GFX7-NEXT:    v_mov_b32_e32 v0, s4
7789; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7790; GFX7-NEXT:    v_mov_b32_e32 v1, s5
7791; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7792; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7793; GFX7-NEXT:    s_waitcnt vmcnt(0)
7794; GFX7-NEXT:    buffer_wbinvl1_vol
7795; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7796; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7797; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7798; GFX7-NEXT:    flat_store_dword v[0:1], v2
7799; GFX7-NEXT:    s_endpgm
7800;
7801; GFX10-WGP-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
7802; GFX10-WGP:       ; %bb.0: ; %entry
7803; GFX10-WGP-NEXT:    s_clause 0x1
7804; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7805; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7806; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7807; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
7808; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
7809; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
7810; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7811; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
7812; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7813; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7814; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7815; GFX10-WGP-NEXT:    buffer_gl0_inv
7816; GFX10-WGP-NEXT:    buffer_gl1_inv
7817; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7818; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7819; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7820; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7821; GFX10-WGP-NEXT:    s_endpgm
7822;
7823; GFX10-CU-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
7824; GFX10-CU:       ; %bb.0: ; %entry
7825; GFX10-CU-NEXT:    s_clause 0x1
7826; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7827; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7828; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7829; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
7830; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
7831; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
7832; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7833; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
7834; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7835; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7836; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7837; GFX10-CU-NEXT:    buffer_gl0_inv
7838; GFX10-CU-NEXT:    buffer_gl1_inv
7839; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7840; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7841; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7842; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7843; GFX10-CU-NEXT:    s_endpgm
7844;
7845; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
7846; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7847; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7848; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7849; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7850; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
7851; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
7852; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7853; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7854; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7855; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7856; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7857; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7858; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7859; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7860; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7861; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7862; SKIP-CACHE-INV-NEXT:    s_endpgm
7863;
7864; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
7865; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7866; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7867; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7868; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7869; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7870; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7871; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7872; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7873; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7874; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7875; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7876; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7877;
7878; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
7879; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7880; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7881; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7882; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7883; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7884; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7885; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7886; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7887; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7888; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7889; GFX90A-TGSPLIT-NEXT:    s_endpgm
7890    i32* %out, i32 %in, i32 %old) {
7891entry:
7892  %gep = getelementptr i32, i32* %out, i32 4
7893  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire monotonic
7894  %val0 = extractvalue { i32, i1 } %val, 0
7895  store i32 %val0, i32* %out, align 4
7896  ret void
7897}
7898
7899define amdgpu_kernel void @flat_agent_one_as_release_monotonic_ret_cmpxchg(
7900; GFX7-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
7901; GFX7:       ; %bb.0: ; %entry
7902; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7903; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7904; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7905; GFX7-NEXT:    s_add_u32 s4, s0, 16
7906; GFX7-NEXT:    s_addc_u32 s5, s1, 0
7907; GFX7-NEXT:    v_mov_b32_e32 v0, s4
7908; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7909; GFX7-NEXT:    v_mov_b32_e32 v1, s5
7910; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7911; GFX7-NEXT:    s_waitcnt vmcnt(0)
7912; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7913; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7914; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7915; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7916; GFX7-NEXT:    flat_store_dword v[0:1], v2
7917; GFX7-NEXT:    s_endpgm
7918;
7919; GFX10-WGP-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
7920; GFX10-WGP:       ; %bb.0: ; %entry
7921; GFX10-WGP-NEXT:    s_clause 0x1
7922; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7923; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7924; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7925; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
7926; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
7927; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
7928; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7929; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
7930; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7931; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7932; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7933; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7934; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7935; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7936; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7937; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7938; GFX10-WGP-NEXT:    s_endpgm
7939;
7940; GFX10-CU-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
7941; GFX10-CU:       ; %bb.0: ; %entry
7942; GFX10-CU-NEXT:    s_clause 0x1
7943; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7944; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7945; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7946; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
7947; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
7948; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
7949; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7950; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
7951; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7952; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7953; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7954; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7955; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7956; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7957; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7958; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7959; GFX10-CU-NEXT:    s_endpgm
7960;
7961; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
7962; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7963; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7964; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7965; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7966; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
7967; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
7968; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7969; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7970; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7971; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7972; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7973; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7974; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7975; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7976; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7977; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7978; SKIP-CACHE-INV-NEXT:    s_endpgm
7979;
7980; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
7981; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7982; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7983; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7984; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7985; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7986; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7987; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7988; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7989; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7990; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7991; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7992;
7993; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
7994; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7995; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7996; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7997; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7998; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7999; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8000; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8001; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8002; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8003; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8004; GFX90A-TGSPLIT-NEXT:    s_endpgm
8005    i32* %out, i32 %in, i32 %old) {
8006entry:
8007  %gep = getelementptr i32, i32* %out, i32 4
8008  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release monotonic
8009  %val0 = extractvalue { i32, i1 } %val, 0
8010  store i32 %val0, i32* %out, align 4
8011  ret void
8012}
8013
8014define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg(
8015; GFX7-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
8016; GFX7:       ; %bb.0: ; %entry
8017; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8018; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8019; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8020; GFX7-NEXT:    s_add_u32 s4, s0, 16
8021; GFX7-NEXT:    s_addc_u32 s5, s1, 0
8022; GFX7-NEXT:    v_mov_b32_e32 v0, s4
8023; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8024; GFX7-NEXT:    v_mov_b32_e32 v1, s5
8025; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8026; GFX7-NEXT:    s_waitcnt vmcnt(0)
8027; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8028; GFX7-NEXT:    s_waitcnt vmcnt(0)
8029; GFX7-NEXT:    buffer_wbinvl1_vol
8030; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8031; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8032; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8033; GFX7-NEXT:    flat_store_dword v[0:1], v2
8034; GFX7-NEXT:    s_endpgm
8035;
8036; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
8037; GFX10-WGP:       ; %bb.0: ; %entry
8038; GFX10-WGP-NEXT:    s_clause 0x1
8039; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8040; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8041; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8042; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
8043; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
8044; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
8045; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8046; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
8047; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8048; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8049; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8050; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8051; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8052; GFX10-WGP-NEXT:    buffer_gl0_inv
8053; GFX10-WGP-NEXT:    buffer_gl1_inv
8054; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8055; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8056; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8057; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8058; GFX10-WGP-NEXT:    s_endpgm
8059;
8060; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
8061; GFX10-CU:       ; %bb.0: ; %entry
8062; GFX10-CU-NEXT:    s_clause 0x1
8063; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8064; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8065; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8066; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
8067; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
8068; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
8069; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8070; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
8071; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8072; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8073; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8074; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8075; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8076; GFX10-CU-NEXT:    buffer_gl0_inv
8077; GFX10-CU-NEXT:    buffer_gl1_inv
8078; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8079; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8080; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8081; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8082; GFX10-CU-NEXT:    s_endpgm
8083;
8084; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
8085; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8086; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8087; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8088; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8089; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
8090; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
8091; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8092; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8093; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8094; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8095; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8096; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8097; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8098; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8099; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8100; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8101; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8102; SKIP-CACHE-INV-NEXT:    s_endpgm
8103;
8104; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
8105; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8106; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8107; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8108; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8109; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8110; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8111; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8112; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8113; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8114; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
8115; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8116; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8117; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8118;
8119; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
8120; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8121; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8122; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8123; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8124; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8125; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8126; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8127; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8128; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8129; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8130; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8131; GFX90A-TGSPLIT-NEXT:    s_endpgm
8132    i32* %out, i32 %in, i32 %old) {
8133entry:
8134  %gep = getelementptr i32, i32* %out, i32 4
8135  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel monotonic
8136  %val0 = extractvalue { i32, i1 } %val, 0
8137  store i32 %val0, i32* %out, align 4
8138  ret void
8139}
8140
8141define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg(
8142; GFX7-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
8143; GFX7:       ; %bb.0: ; %entry
8144; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8145; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8146; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8147; GFX7-NEXT:    s_add_u32 s4, s0, 16
8148; GFX7-NEXT:    s_addc_u32 s5, s1, 0
8149; GFX7-NEXT:    v_mov_b32_e32 v0, s4
8150; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8151; GFX7-NEXT:    v_mov_b32_e32 v1, s5
8152; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8153; GFX7-NEXT:    s_waitcnt vmcnt(0)
8154; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8155; GFX7-NEXT:    s_waitcnt vmcnt(0)
8156; GFX7-NEXT:    buffer_wbinvl1_vol
8157; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8158; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8159; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8160; GFX7-NEXT:    flat_store_dword v[0:1], v2
8161; GFX7-NEXT:    s_endpgm
8162;
8163; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
8164; GFX10-WGP:       ; %bb.0: ; %entry
8165; GFX10-WGP-NEXT:    s_clause 0x1
8166; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8167; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8168; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8169; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
8170; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
8171; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
8172; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8173; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
8174; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8175; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8176; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8177; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8178; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8179; GFX10-WGP-NEXT:    buffer_gl0_inv
8180; GFX10-WGP-NEXT:    buffer_gl1_inv
8181; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8182; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8183; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8184; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8185; GFX10-WGP-NEXT:    s_endpgm
8186;
8187; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
8188; GFX10-CU:       ; %bb.0: ; %entry
8189; GFX10-CU-NEXT:    s_clause 0x1
8190; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8191; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8192; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8193; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
8194; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
8195; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
8196; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8197; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
8198; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8199; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8200; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8201; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8202; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8203; GFX10-CU-NEXT:    buffer_gl0_inv
8204; GFX10-CU-NEXT:    buffer_gl1_inv
8205; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8206; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8207; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8208; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8209; GFX10-CU-NEXT:    s_endpgm
8210;
8211; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
8212; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8213; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8214; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8215; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8216; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
8217; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
8218; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8219; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8220; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8221; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8222; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8223; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8224; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8225; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8226; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8227; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8228; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8229; SKIP-CACHE-INV-NEXT:    s_endpgm
8230;
8231; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
8232; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8233; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8234; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8235; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8236; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8237; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8238; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8239; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8240; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8241; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
8242; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8243; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8244; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8245;
8246; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
8247; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8248; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8249; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8250; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8251; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8252; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8253; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8254; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8255; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8256; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8257; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8258; GFX90A-TGSPLIT-NEXT:    s_endpgm
8259    i32* %out, i32 %in, i32 %old) {
8260entry:
8261  %gep = getelementptr i32, i32* %out, i32 4
8262  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst monotonic
8263  %val0 = extractvalue { i32, i1 } %val, 0
8264  store i32 %val0, i32* %out, align 4
8265  ret void
8266}
8267
8268define amdgpu_kernel void @flat_agent_one_as_monotonic_acquire_ret_cmpxchg(
8269; GFX7-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
8270; GFX7:       ; %bb.0: ; %entry
8271; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8272; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8273; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8274; GFX7-NEXT:    s_add_u32 s4, s0, 16
8275; GFX7-NEXT:    s_addc_u32 s5, s1, 0
8276; GFX7-NEXT:    v_mov_b32_e32 v0, s4
8277; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8278; GFX7-NEXT:    v_mov_b32_e32 v1, s5
8279; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8280; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8281; GFX7-NEXT:    s_waitcnt vmcnt(0)
8282; GFX7-NEXT:    buffer_wbinvl1_vol
8283; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8284; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8285; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8286; GFX7-NEXT:    flat_store_dword v[0:1], v2
8287; GFX7-NEXT:    s_endpgm
8288;
8289; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
8290; GFX10-WGP:       ; %bb.0: ; %entry
8291; GFX10-WGP-NEXT:    s_clause 0x1
8292; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8293; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8294; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8295; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
8296; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
8297; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
8298; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8299; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
8300; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8301; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8302; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8303; GFX10-WGP-NEXT:    buffer_gl0_inv
8304; GFX10-WGP-NEXT:    buffer_gl1_inv
8305; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8306; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8307; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8308; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8309; GFX10-WGP-NEXT:    s_endpgm
8310;
8311; GFX10-CU-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
8312; GFX10-CU:       ; %bb.0: ; %entry
8313; GFX10-CU-NEXT:    s_clause 0x1
8314; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8315; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8316; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8317; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
8318; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
8319; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
8320; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8321; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
8322; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8323; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8324; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8325; GFX10-CU-NEXT:    buffer_gl0_inv
8326; GFX10-CU-NEXT:    buffer_gl1_inv
8327; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8328; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8329; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8330; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8331; GFX10-CU-NEXT:    s_endpgm
8332;
8333; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
8334; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8335; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8336; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8337; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8338; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
8339; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
8340; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8341; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8342; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8343; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8344; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8345; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8346; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8347; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8348; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8349; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8350; SKIP-CACHE-INV-NEXT:    s_endpgm
8351;
8352; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
8353; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8354; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8355; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8356; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8357; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8358; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8359; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8360; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8361; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
8362; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8363; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8364; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8365;
8366; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
8367; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8368; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8369; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8370; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8371; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8372; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8373; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8374; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8375; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8376; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8377; GFX90A-TGSPLIT-NEXT:    s_endpgm
8378    i32* %out, i32 %in, i32 %old) {
8379entry:
8380  %gep = getelementptr i32, i32* %out, i32 4
8381  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") monotonic acquire
8382  %val0 = extractvalue { i32, i1 } %val, 0
8383  store i32 %val0, i32* %out, align 4
8384  ret void
8385}
8386
8387define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_ret_cmpxchg(
8388; GFX7-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
8389; GFX7:       ; %bb.0: ; %entry
8390; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8391; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8392; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8393; GFX7-NEXT:    s_add_u32 s4, s0, 16
8394; GFX7-NEXT:    s_addc_u32 s5, s1, 0
8395; GFX7-NEXT:    v_mov_b32_e32 v0, s4
8396; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8397; GFX7-NEXT:    v_mov_b32_e32 v1, s5
8398; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8399; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8400; GFX7-NEXT:    s_waitcnt vmcnt(0)
8401; GFX7-NEXT:    buffer_wbinvl1_vol
8402; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8403; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8404; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8405; GFX7-NEXT:    flat_store_dword v[0:1], v2
8406; GFX7-NEXT:    s_endpgm
8407;
8408; GFX10-WGP-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
8409; GFX10-WGP:       ; %bb.0: ; %entry
8410; GFX10-WGP-NEXT:    s_clause 0x1
8411; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8412; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8413; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8414; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
8415; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
8416; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
8417; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8418; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
8419; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8420; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8421; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8422; GFX10-WGP-NEXT:    buffer_gl0_inv
8423; GFX10-WGP-NEXT:    buffer_gl1_inv
8424; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8425; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8426; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8427; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8428; GFX10-WGP-NEXT:    s_endpgm
8429;
8430; GFX10-CU-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
8431; GFX10-CU:       ; %bb.0: ; %entry
8432; GFX10-CU-NEXT:    s_clause 0x1
8433; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8434; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8435; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8436; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
8437; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
8438; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
8439; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8440; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
8441; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8442; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8443; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8444; GFX10-CU-NEXT:    buffer_gl0_inv
8445; GFX10-CU-NEXT:    buffer_gl1_inv
8446; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8447; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8448; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8449; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8450; GFX10-CU-NEXT:    s_endpgm
8451;
8452; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
8453; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8454; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8455; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8456; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8457; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
8458; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
8459; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8460; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8461; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8462; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8463; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8464; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8465; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8466; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8467; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8468; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8469; SKIP-CACHE-INV-NEXT:    s_endpgm
8470;
8471; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
8472; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8473; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8474; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8475; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8476; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8477; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8478; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8479; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8480; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
8481; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8482; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8483; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8484;
8485; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
8486; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8487; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8488; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8489; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8490; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8491; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8492; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8493; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8494; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8495; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8496; GFX90A-TGSPLIT-NEXT:    s_endpgm
8497    i32* %out, i32 %in, i32 %old) {
8498entry:
8499  %gep = getelementptr i32, i32* %out, i32 4
8500  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire acquire
8501  %val0 = extractvalue { i32, i1 } %val, 0
8502  store i32 %val0, i32* %out, align 4
8503  ret void
8504}
8505
8506define amdgpu_kernel void @flat_agent_one_as_release_acquire_ret_cmpxchg(
8507; GFX7-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
8508; GFX7:       ; %bb.0: ; %entry
8509; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8510; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8511; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8512; GFX7-NEXT:    s_add_u32 s4, s0, 16
8513; GFX7-NEXT:    s_addc_u32 s5, s1, 0
8514; GFX7-NEXT:    v_mov_b32_e32 v0, s4
8515; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8516; GFX7-NEXT:    v_mov_b32_e32 v1, s5
8517; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8518; GFX7-NEXT:    s_waitcnt vmcnt(0)
8519; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8520; GFX7-NEXT:    s_waitcnt vmcnt(0)
8521; GFX7-NEXT:    buffer_wbinvl1_vol
8522; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8523; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8524; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8525; GFX7-NEXT:    flat_store_dword v[0:1], v2
8526; GFX7-NEXT:    s_endpgm
8527;
8528; GFX10-WGP-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
8529; GFX10-WGP:       ; %bb.0: ; %entry
8530; GFX10-WGP-NEXT:    s_clause 0x1
8531; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8532; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8533; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8534; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
8535; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
8536; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
8537; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8538; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
8539; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8540; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8541; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8542; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8543; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8544; GFX10-WGP-NEXT:    buffer_gl0_inv
8545; GFX10-WGP-NEXT:    buffer_gl1_inv
8546; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8547; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8548; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8549; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8550; GFX10-WGP-NEXT:    s_endpgm
8551;
8552; GFX10-CU-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
8553; GFX10-CU:       ; %bb.0: ; %entry
8554; GFX10-CU-NEXT:    s_clause 0x1
8555; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8556; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8557; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8558; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
8559; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
8560; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
8561; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8562; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
8563; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8564; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8565; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8566; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8567; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8568; GFX10-CU-NEXT:    buffer_gl0_inv
8569; GFX10-CU-NEXT:    buffer_gl1_inv
8570; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8571; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8572; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8573; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8574; GFX10-CU-NEXT:    s_endpgm
8575;
8576; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
8577; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8578; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8579; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8580; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8581; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
8582; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
8583; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8584; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8585; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8586; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8587; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8588; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8589; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8590; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8591; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8592; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8593; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8594; SKIP-CACHE-INV-NEXT:    s_endpgm
8595;
8596; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
8597; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8598; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8599; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8600; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8601; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8602; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8603; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8604; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8605; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8606; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
8607; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8608; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8609; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8610;
8611; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
8612; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8613; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8614; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8615; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8616; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8617; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8618; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8619; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8620; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8621; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8622; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8623; GFX90A-TGSPLIT-NEXT:    s_endpgm
8624    i32* %out, i32 %in, i32 %old) {
8625entry:
8626  %gep = getelementptr i32, i32* %out, i32 4
8627  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release acquire
8628  %val0 = extractvalue { i32, i1 } %val, 0
8629  store i32 %val0, i32* %out, align 4
8630  ret void
8631}
8632
8633define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_ret_cmpxchg(
8634; GFX7-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
8635; GFX7:       ; %bb.0: ; %entry
8636; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8637; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8638; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8639; GFX7-NEXT:    s_add_u32 s4, s0, 16
8640; GFX7-NEXT:    s_addc_u32 s5, s1, 0
8641; GFX7-NEXT:    v_mov_b32_e32 v0, s4
8642; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8643; GFX7-NEXT:    v_mov_b32_e32 v1, s5
8644; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8645; GFX7-NEXT:    s_waitcnt vmcnt(0)
8646; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8647; GFX7-NEXT:    s_waitcnt vmcnt(0)
8648; GFX7-NEXT:    buffer_wbinvl1_vol
8649; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8650; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8651; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8652; GFX7-NEXT:    flat_store_dword v[0:1], v2
8653; GFX7-NEXT:    s_endpgm
8654;
8655; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
8656; GFX10-WGP:       ; %bb.0: ; %entry
8657; GFX10-WGP-NEXT:    s_clause 0x1
8658; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8659; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8660; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8661; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
8662; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
8663; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
8664; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8665; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
8666; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8667; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8668; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8669; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8670; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8671; GFX10-WGP-NEXT:    buffer_gl0_inv
8672; GFX10-WGP-NEXT:    buffer_gl1_inv
8673; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8674; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8675; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8676; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8677; GFX10-WGP-NEXT:    s_endpgm
8678;
8679; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
8680; GFX10-CU:       ; %bb.0: ; %entry
8681; GFX10-CU-NEXT:    s_clause 0x1
8682; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8683; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8684; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8685; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
8686; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
8687; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
8688; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8689; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
8690; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8691; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8692; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8693; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8694; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8695; GFX10-CU-NEXT:    buffer_gl0_inv
8696; GFX10-CU-NEXT:    buffer_gl1_inv
8697; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8698; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8699; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8700; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8701; GFX10-CU-NEXT:    s_endpgm
8702;
8703; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
8704; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8705; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8706; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8707; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8708; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
8709; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
8710; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8711; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8712; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8713; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8714; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8715; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8716; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8717; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8718; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8719; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8720; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8721; SKIP-CACHE-INV-NEXT:    s_endpgm
8722;
8723; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
8724; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8725; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8726; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8727; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8728; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8729; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8730; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8731; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8732; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8733; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
8734; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8735; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8736; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8737;
8738; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
8739; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8740; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8741; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8742; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8743; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8744; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8745; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8746; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8747; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8748; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8749; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8750; GFX90A-TGSPLIT-NEXT:    s_endpgm
8751    i32* %out, i32 %in, i32 %old) {
8752entry:
8753  %gep = getelementptr i32, i32* %out, i32 4
8754  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel acquire
8755  %val0 = extractvalue { i32, i1 } %val, 0
8756  store i32 %val0, i32* %out, align 4
8757  ret void
8758}
8759
8760define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_ret_cmpxchg(
8761; GFX7-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
8762; GFX7:       ; %bb.0: ; %entry
8763; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8764; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8765; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8766; GFX7-NEXT:    s_add_u32 s4, s0, 16
8767; GFX7-NEXT:    s_addc_u32 s5, s1, 0
8768; GFX7-NEXT:    v_mov_b32_e32 v0, s4
8769; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8770; GFX7-NEXT:    v_mov_b32_e32 v1, s5
8771; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8772; GFX7-NEXT:    s_waitcnt vmcnt(0)
8773; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8774; GFX7-NEXT:    s_waitcnt vmcnt(0)
8775; GFX7-NEXT:    buffer_wbinvl1_vol
8776; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8777; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8778; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8779; GFX7-NEXT:    flat_store_dword v[0:1], v2
8780; GFX7-NEXT:    s_endpgm
8781;
8782; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
8783; GFX10-WGP:       ; %bb.0: ; %entry
8784; GFX10-WGP-NEXT:    s_clause 0x1
8785; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8786; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8787; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8788; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
8789; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
8790; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
8791; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8792; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
8793; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8794; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8795; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8796; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8797; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8798; GFX10-WGP-NEXT:    buffer_gl0_inv
8799; GFX10-WGP-NEXT:    buffer_gl1_inv
8800; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8801; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8802; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8803; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8804; GFX10-WGP-NEXT:    s_endpgm
8805;
8806; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
8807; GFX10-CU:       ; %bb.0: ; %entry
8808; GFX10-CU-NEXT:    s_clause 0x1
8809; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8810; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8811; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8812; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
8813; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
8814; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
8815; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8816; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
8817; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8818; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8819; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8820; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8821; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8822; GFX10-CU-NEXT:    buffer_gl0_inv
8823; GFX10-CU-NEXT:    buffer_gl1_inv
8824; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8825; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8826; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8827; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8828; GFX10-CU-NEXT:    s_endpgm
8829;
8830; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
8831; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8832; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8833; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8834; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8835; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
8836; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
8837; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8838; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8839; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8840; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8841; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8842; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8843; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8844; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8845; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8846; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8847; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8848; SKIP-CACHE-INV-NEXT:    s_endpgm
8849;
8850; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
8851; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8852; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8853; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8854; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8855; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8856; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8857; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8858; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8859; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8860; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
8861; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8862; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8863; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8864;
8865; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
8866; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8867; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8868; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8869; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8870; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8871; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8872; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8873; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8874; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8875; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8876; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8877; GFX90A-TGSPLIT-NEXT:    s_endpgm
8878    i32* %out, i32 %in, i32 %old) {
8879entry:
8880  %gep = getelementptr i32, i32* %out, i32 4
8881  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst acquire
8882  %val0 = extractvalue { i32, i1 } %val, 0
8883  store i32 %val0, i32* %out, align 4
8884  ret void
8885}
8886
8887define amdgpu_kernel void @flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg(
8888; GFX7-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
8889; GFX7:       ; %bb.0: ; %entry
8890; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8891; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8892; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8893; GFX7-NEXT:    s_add_u32 s4, s0, 16
8894; GFX7-NEXT:    s_addc_u32 s5, s1, 0
8895; GFX7-NEXT:    v_mov_b32_e32 v0, s4
8896; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8897; GFX7-NEXT:    v_mov_b32_e32 v1, s5
8898; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8899; GFX7-NEXT:    s_waitcnt vmcnt(0)
8900; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8901; GFX7-NEXT:    s_waitcnt vmcnt(0)
8902; GFX7-NEXT:    buffer_wbinvl1_vol
8903; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8904; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8905; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8906; GFX7-NEXT:    flat_store_dword v[0:1], v2
8907; GFX7-NEXT:    s_endpgm
8908;
8909; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
8910; GFX10-WGP:       ; %bb.0: ; %entry
8911; GFX10-WGP-NEXT:    s_clause 0x1
8912; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8913; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8914; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8915; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
8916; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
8917; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
8918; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8919; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
8920; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8921; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8922; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8923; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8924; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8925; GFX10-WGP-NEXT:    buffer_gl0_inv
8926; GFX10-WGP-NEXT:    buffer_gl1_inv
8927; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8928; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8929; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8930; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
8931; GFX10-WGP-NEXT:    s_endpgm
8932;
8933; GFX10-CU-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
8934; GFX10-CU:       ; %bb.0: ; %entry
8935; GFX10-CU-NEXT:    s_clause 0x1
8936; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8937; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8938; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8939; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
8940; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
8941; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
8942; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8943; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
8944; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8945; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8946; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8947; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8948; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8949; GFX10-CU-NEXT:    buffer_gl0_inv
8950; GFX10-CU-NEXT:    buffer_gl1_inv
8951; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8952; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8953; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8954; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
8955; GFX10-CU-NEXT:    s_endpgm
8956;
8957; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
8958; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8959; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8960; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8961; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8962; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
8963; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
8964; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
8965; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8966; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
8967; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8968; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8969; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8970; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8971; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8972; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8973; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8974; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
8975; SKIP-CACHE-INV-NEXT:    s_endpgm
8976;
8977; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
8978; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8979; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8980; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8981; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8982; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8983; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8984; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8985; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8986; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8987; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
8988; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8989; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
8990; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8991;
8992; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
8993; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8994; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8995; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8996; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8997; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8998; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8999; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9000; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9001; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9002; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9003; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9004; GFX90A-TGSPLIT-NEXT:    s_endpgm
9005    i32* %out, i32 %in, i32 %old) {
9006entry:
9007  %gep = getelementptr i32, i32* %out, i32 4
9008  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") monotonic seq_cst
9009  %val0 = extractvalue { i32, i1 } %val, 0
9010  store i32 %val0, i32* %out, align 4
9011  ret void
9012}
9013
9014define amdgpu_kernel void @flat_agent_one_as_acquire_seq_cst_ret_cmpxchg(
9015; GFX7-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
9016; GFX7:       ; %bb.0: ; %entry
9017; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9018; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9019; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9020; GFX7-NEXT:    s_add_u32 s4, s0, 16
9021; GFX7-NEXT:    s_addc_u32 s5, s1, 0
9022; GFX7-NEXT:    v_mov_b32_e32 v0, s4
9023; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9024; GFX7-NEXT:    v_mov_b32_e32 v1, s5
9025; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9026; GFX7-NEXT:    s_waitcnt vmcnt(0)
9027; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9028; GFX7-NEXT:    s_waitcnt vmcnt(0)
9029; GFX7-NEXT:    buffer_wbinvl1_vol
9030; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9031; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9032; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9033; GFX7-NEXT:    flat_store_dword v[0:1], v2
9034; GFX7-NEXT:    s_endpgm
9035;
9036; GFX10-WGP-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
9037; GFX10-WGP:       ; %bb.0: ; %entry
9038; GFX10-WGP-NEXT:    s_clause 0x1
9039; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9040; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9041; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9042; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
9043; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
9044; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
9045; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9046; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
9047; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9048; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9049; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9050; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9051; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9052; GFX10-WGP-NEXT:    buffer_gl0_inv
9053; GFX10-WGP-NEXT:    buffer_gl1_inv
9054; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9055; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9056; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9057; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
9058; GFX10-WGP-NEXT:    s_endpgm
9059;
9060; GFX10-CU-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
9061; GFX10-CU:       ; %bb.0: ; %entry
9062; GFX10-CU-NEXT:    s_clause 0x1
9063; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9064; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9065; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9066; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
9067; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
9068; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
9069; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9070; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
9071; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9072; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9073; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9074; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9075; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9076; GFX10-CU-NEXT:    buffer_gl0_inv
9077; GFX10-CU-NEXT:    buffer_gl1_inv
9078; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9079; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9080; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9081; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
9082; GFX10-CU-NEXT:    s_endpgm
9083;
9084; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
9085; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9086; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9087; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9088; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9089; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
9090; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
9091; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
9092; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9093; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
9094; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9095; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9096; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9097; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9098; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9099; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9100; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9101; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
9102; SKIP-CACHE-INV-NEXT:    s_endpgm
9103;
9104; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
9105; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9106; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9107; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9108; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9109; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9110; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9111; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9112; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9113; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9114; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9115; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9116; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9117; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9118;
9119; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
9120; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9121; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9122; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9123; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9124; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9125; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9126; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9127; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9128; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9129; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9130; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9131; GFX90A-TGSPLIT-NEXT:    s_endpgm
9132    i32* %out, i32 %in, i32 %old) {
9133entry:
9134  %gep = getelementptr i32, i32* %out, i32 4
9135  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire seq_cst
9136  %val0 = extractvalue { i32, i1 } %val, 0
9137  store i32 %val0, i32* %out, align 4
9138  ret void
9139}
9140
9141define amdgpu_kernel void @flat_agent_one_as_release_seq_cst_ret_cmpxchg(
9142; GFX7-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
9143; GFX7:       ; %bb.0: ; %entry
9144; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9145; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9146; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9147; GFX7-NEXT:    s_add_u32 s4, s0, 16
9148; GFX7-NEXT:    s_addc_u32 s5, s1, 0
9149; GFX7-NEXT:    v_mov_b32_e32 v0, s4
9150; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9151; GFX7-NEXT:    v_mov_b32_e32 v1, s5
9152; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9153; GFX7-NEXT:    s_waitcnt vmcnt(0)
9154; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9155; GFX7-NEXT:    s_waitcnt vmcnt(0)
9156; GFX7-NEXT:    buffer_wbinvl1_vol
9157; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9158; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9159; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9160; GFX7-NEXT:    flat_store_dword v[0:1], v2
9161; GFX7-NEXT:    s_endpgm
9162;
9163; GFX10-WGP-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
9164; GFX10-WGP:       ; %bb.0: ; %entry
9165; GFX10-WGP-NEXT:    s_clause 0x1
9166; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9167; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9168; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9169; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
9170; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
9171; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
9172; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9173; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
9174; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9175; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9176; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9177; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9178; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9179; GFX10-WGP-NEXT:    buffer_gl0_inv
9180; GFX10-WGP-NEXT:    buffer_gl1_inv
9181; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9182; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9183; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9184; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
9185; GFX10-WGP-NEXT:    s_endpgm
9186;
9187; GFX10-CU-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
9188; GFX10-CU:       ; %bb.0: ; %entry
9189; GFX10-CU-NEXT:    s_clause 0x1
9190; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9191; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9192; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9193; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
9194; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
9195; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
9196; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9197; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
9198; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9199; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9200; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9201; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9202; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9203; GFX10-CU-NEXT:    buffer_gl0_inv
9204; GFX10-CU-NEXT:    buffer_gl1_inv
9205; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9206; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9207; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9208; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
9209; GFX10-CU-NEXT:    s_endpgm
9210;
9211; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
9212; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9213; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9214; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9215; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9216; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
9217; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
9218; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
9219; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9220; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
9221; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9222; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9223; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9224; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9225; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9226; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9227; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9228; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
9229; SKIP-CACHE-INV-NEXT:    s_endpgm
9230;
9231; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
9232; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9233; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9234; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9235; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9236; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9237; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9238; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9239; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9240; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9241; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9242; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9243; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9244; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9245;
9246; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
9247; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9248; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9249; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9250; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9251; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9252; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9253; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9254; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9255; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9256; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9257; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9258; GFX90A-TGSPLIT-NEXT:    s_endpgm
9259    i32* %out, i32 %in, i32 %old) {
9260entry:
9261  %gep = getelementptr i32, i32* %out, i32 4
9262  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release seq_cst
9263  %val0 = extractvalue { i32, i1 } %val, 0
9264  store i32 %val0, i32* %out, align 4
9265  ret void
9266}
9267
9268define amdgpu_kernel void @flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg(
9269; GFX7-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
9270; GFX7:       ; %bb.0: ; %entry
9271; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9272; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9273; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9274; GFX7-NEXT:    s_add_u32 s4, s0, 16
9275; GFX7-NEXT:    s_addc_u32 s5, s1, 0
9276; GFX7-NEXT:    v_mov_b32_e32 v0, s4
9277; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9278; GFX7-NEXT:    v_mov_b32_e32 v1, s5
9279; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9280; GFX7-NEXT:    s_waitcnt vmcnt(0)
9281; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9282; GFX7-NEXT:    s_waitcnt vmcnt(0)
9283; GFX7-NEXT:    buffer_wbinvl1_vol
9284; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9285; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9286; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9287; GFX7-NEXT:    flat_store_dword v[0:1], v2
9288; GFX7-NEXT:    s_endpgm
9289;
9290; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
9291; GFX10-WGP:       ; %bb.0: ; %entry
9292; GFX10-WGP-NEXT:    s_clause 0x1
9293; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9294; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9295; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9296; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
9297; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
9298; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
9299; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9300; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
9301; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9302; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9303; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9304; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9305; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9306; GFX10-WGP-NEXT:    buffer_gl0_inv
9307; GFX10-WGP-NEXT:    buffer_gl1_inv
9308; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9309; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9310; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9311; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
9312; GFX10-WGP-NEXT:    s_endpgm
9313;
9314; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
9315; GFX10-CU:       ; %bb.0: ; %entry
9316; GFX10-CU-NEXT:    s_clause 0x1
9317; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9318; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9319; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9320; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
9321; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
9322; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
9323; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9324; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
9325; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9326; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9327; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9328; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9329; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9330; GFX10-CU-NEXT:    buffer_gl0_inv
9331; GFX10-CU-NEXT:    buffer_gl1_inv
9332; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9333; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9334; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9335; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
9336; GFX10-CU-NEXT:    s_endpgm
9337;
9338; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
9339; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9340; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9341; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9342; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9343; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
9344; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
9345; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
9346; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9347; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
9348; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9349; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9350; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9351; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9352; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9353; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9354; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9355; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
9356; SKIP-CACHE-INV-NEXT:    s_endpgm
9357;
9358; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
9359; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9360; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9361; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9362; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9363; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9364; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9365; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9366; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9367; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9368; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9369; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9370; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9371; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9372;
9373; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
9374; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9375; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9376; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9377; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9378; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9379; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9380; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9381; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9382; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9383; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9384; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9385; GFX90A-TGSPLIT-NEXT:    s_endpgm
9386    i32* %out, i32 %in, i32 %old) {
9387entry:
9388  %gep = getelementptr i32, i32* %out, i32 4
9389  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel seq_cst
9390  %val0 = extractvalue { i32, i1 } %val, 0
9391  store i32 %val0, i32* %out, align 4
9392  ret void
9393}
9394
9395define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg(
9396; GFX7-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
9397; GFX7:       ; %bb.0: ; %entry
9398; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9399; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9400; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9401; GFX7-NEXT:    s_add_u32 s4, s0, 16
9402; GFX7-NEXT:    s_addc_u32 s5, s1, 0
9403; GFX7-NEXT:    v_mov_b32_e32 v0, s4
9404; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9405; GFX7-NEXT:    v_mov_b32_e32 v1, s5
9406; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9407; GFX7-NEXT:    s_waitcnt vmcnt(0)
9408; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9409; GFX7-NEXT:    s_waitcnt vmcnt(0)
9410; GFX7-NEXT:    buffer_wbinvl1_vol
9411; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9412; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9413; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9414; GFX7-NEXT:    flat_store_dword v[0:1], v2
9415; GFX7-NEXT:    s_endpgm
9416;
9417; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
9418; GFX10-WGP:       ; %bb.0: ; %entry
9419; GFX10-WGP-NEXT:    s_clause 0x1
9420; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9421; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9422; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9423; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
9424; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
9425; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
9426; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9427; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
9428; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9429; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9430; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9431; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9432; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9433; GFX10-WGP-NEXT:    buffer_gl0_inv
9434; GFX10-WGP-NEXT:    buffer_gl1_inv
9435; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9436; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9437; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9438; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
9439; GFX10-WGP-NEXT:    s_endpgm
9440;
9441; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
9442; GFX10-CU:       ; %bb.0: ; %entry
9443; GFX10-CU-NEXT:    s_clause 0x1
9444; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9445; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9446; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9447; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
9448; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
9449; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
9450; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9451; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
9452; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9453; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9454; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9455; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9456; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9457; GFX10-CU-NEXT:    buffer_gl0_inv
9458; GFX10-CU-NEXT:    buffer_gl1_inv
9459; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9460; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9461; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9462; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
9463; GFX10-CU-NEXT:    s_endpgm
9464;
9465; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
9466; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9467; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9468; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9469; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9470; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
9471; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
9472; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
9473; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9474; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
9475; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9476; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9477; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9478; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9479; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9480; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9481; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9482; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
9483; SKIP-CACHE-INV-NEXT:    s_endpgm
9484;
9485; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
9486; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9487; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9488; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9489; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9490; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9491; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9492; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9493; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9494; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9495; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9496; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9497; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9498; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9499;
9500; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
9501; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9502; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9503; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9504; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9505; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9506; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9507; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9508; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9509; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9510; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9511; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9512; GFX90A-TGSPLIT-NEXT:    s_endpgm
9513    i32* %out, i32 %in, i32 %old) {
9514entry:
9515  %gep = getelementptr i32, i32* %out, i32 4
9516  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst seq_cst
9517  %val0 = extractvalue { i32, i1 } %val, 0
9518  store i32 %val0, i32* %out, align 4
9519  ret void
9520}
9521
9522