1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
3; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
4; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
5; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
6; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-NOTTGSPLIT %s
7; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-TGSPLIT %s
8
9define amdgpu_kernel void @flat_agent_unordered_load(
10; GFX7-LABEL: flat_agent_unordered_load:
11; GFX7:       ; %bb.0: ; %entry
12; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
13; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14; GFX7-NEXT:    v_mov_b32_e32 v0, s0
15; GFX7-NEXT:    v_mov_b32_e32 v1, s1
16; GFX7-NEXT:    flat_load_dword v0, v[0:1]
17; GFX7-NEXT:    v_mov_b32_e32 v2, s2
18; GFX7-NEXT:    v_mov_b32_e32 v3, s3
19; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
20; GFX7-NEXT:    flat_store_dword v[2:3], v0
21; GFX7-NEXT:    s_endpgm
22;
23; GFX10-WGP-LABEL: flat_agent_unordered_load:
24; GFX10-WGP:       ; %bb.0: ; %entry
25; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
26; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
27; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
28; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
29; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
30; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
31; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
32; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
33; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
34; GFX10-WGP-NEXT:    s_endpgm
35;
36; GFX10-CU-LABEL: flat_agent_unordered_load:
37; GFX10-CU:       ; %bb.0: ; %entry
38; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
39; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
40; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
41; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
42; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
43; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
44; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
45; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
46; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
47; GFX10-CU-NEXT:    s_endpgm
48;
49; SKIP-CACHE-INV-LABEL: flat_agent_unordered_load:
50; SKIP-CACHE-INV:       ; %bb.0: ; %entry
51; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
52; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
53; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
54; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
55; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
56; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
57; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
58; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
59; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
60; SKIP-CACHE-INV-NEXT:    s_endpgm
61;
62; GFX90A-NOTTGSPLIT-LABEL: flat_agent_unordered_load:
63; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
64; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
65; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
66; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
67; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
68; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v0, v[0:1]
69; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
70; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
71; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
72; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[2:3], v0
73; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
74;
75; GFX90A-TGSPLIT-LABEL: flat_agent_unordered_load:
76; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
77; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
78; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
79; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
80; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
81; GFX90A-TGSPLIT-NEXT:    flat_load_dword v0, v[0:1]
82; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
83; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
84; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
85; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[2:3], v0
86; GFX90A-TGSPLIT-NEXT:    s_endpgm
87    i32* %in, i32* %out) {
88entry:
89  %val = load atomic i32, i32* %in syncscope("agent") unordered, align 4
90  store i32 %val, i32* %out
91  ret void
92}
93
94define amdgpu_kernel void @flat_agent_monotonic_load(
95; GFX7-LABEL: flat_agent_monotonic_load:
96; GFX7:       ; %bb.0: ; %entry
97; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
98; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
99; GFX7-NEXT:    v_mov_b32_e32 v0, s0
100; GFX7-NEXT:    v_mov_b32_e32 v1, s1
101; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
102; GFX7-NEXT:    v_mov_b32_e32 v2, s2
103; GFX7-NEXT:    v_mov_b32_e32 v3, s3
104; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
105; GFX7-NEXT:    flat_store_dword v[2:3], v0
106; GFX7-NEXT:    s_endpgm
107;
108; GFX10-WGP-LABEL: flat_agent_monotonic_load:
109; GFX10-WGP:       ; %bb.0: ; %entry
110; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
111; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
112; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
113; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
114; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
115; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
116; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
117; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
118; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
119; GFX10-WGP-NEXT:    s_endpgm
120;
121; GFX10-CU-LABEL: flat_agent_monotonic_load:
122; GFX10-CU:       ; %bb.0: ; %entry
123; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
124; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
125; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
126; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
127; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
128; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
129; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
130; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
131; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
132; GFX10-CU-NEXT:    s_endpgm
133;
134; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_load:
135; SKIP-CACHE-INV:       ; %bb.0: ; %entry
136; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
137; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
138; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
139; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
140; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1] glc
141; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
142; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
143; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
144; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
145; SKIP-CACHE-INV-NEXT:    s_endpgm
146;
147; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_load:
148; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
149; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
150; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
151; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
152; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
153; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v0, v[0:1] glc
154; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
155; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
156; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
157; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[2:3], v0
158; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
159;
160; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_load:
161; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
162; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
163; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
164; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
165; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
166; GFX90A-TGSPLIT-NEXT:    flat_load_dword v0, v[0:1] glc
167; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
168; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
169; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
170; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[2:3], v0
171; GFX90A-TGSPLIT-NEXT:    s_endpgm
172    i32* %in, i32* %out) {
173entry:
174  %val = load atomic i32, i32* %in syncscope("agent") monotonic, align 4
175  store i32 %val, i32* %out
176  ret void
177}
178
179define amdgpu_kernel void @flat_agent_acquire_load(
180; GFX7-LABEL: flat_agent_acquire_load:
181; GFX7:       ; %bb.0: ; %entry
182; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
183; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
184; GFX7-NEXT:    v_mov_b32_e32 v0, s0
185; GFX7-NEXT:    v_mov_b32_e32 v1, s1
186; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
187; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
188; GFX7-NEXT:    buffer_wbinvl1_vol
189; GFX7-NEXT:    v_mov_b32_e32 v2, s2
190; GFX7-NEXT:    v_mov_b32_e32 v3, s3
191; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
192; GFX7-NEXT:    flat_store_dword v[2:3], v0
193; GFX7-NEXT:    s_endpgm
194;
195; GFX10-WGP-LABEL: flat_agent_acquire_load:
196; GFX10-WGP:       ; %bb.0: ; %entry
197; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
198; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
199; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
200; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
201; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
202; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
203; GFX10-WGP-NEXT:    buffer_gl0_inv
204; GFX10-WGP-NEXT:    buffer_gl1_inv
205; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
206; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
207; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
208; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
209; GFX10-WGP-NEXT:    s_endpgm
210;
211; GFX10-CU-LABEL: flat_agent_acquire_load:
212; GFX10-CU:       ; %bb.0: ; %entry
213; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
214; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
215; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
216; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
217; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
218; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
219; GFX10-CU-NEXT:    buffer_gl0_inv
220; GFX10-CU-NEXT:    buffer_gl1_inv
221; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
222; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
223; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
224; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
225; GFX10-CU-NEXT:    s_endpgm
226;
227; SKIP-CACHE-INV-LABEL: flat_agent_acquire_load:
228; SKIP-CACHE-INV:       ; %bb.0: ; %entry
229; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
230; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
231; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
232; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
233; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1] glc
234; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
235; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
236; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
237; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
238; SKIP-CACHE-INV-NEXT:    s_endpgm
239;
240; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_load:
241; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
242; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
243; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
244; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
245; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
246; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v0, v[0:1] glc
247; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
248; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
249; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
250; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
251; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
252; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[2:3], v0
253; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
254;
255; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_load:
256; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
257; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
258; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
259; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
260; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
261; GFX90A-TGSPLIT-NEXT:    flat_load_dword v0, v[0:1] glc
262; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
263; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
264; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
265; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
266; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[2:3], v0
267; GFX90A-TGSPLIT-NEXT:    s_endpgm
268    i32* %in, i32* %out) {
269entry:
270  %val = load atomic i32, i32* %in syncscope("agent") acquire, align 4
271  store i32 %val, i32* %out
272  ret void
273}
274
275define amdgpu_kernel void @flat_agent_seq_cst_load(
276; GFX7-LABEL: flat_agent_seq_cst_load:
277; GFX7:       ; %bb.0: ; %entry
278; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
279; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
280; GFX7-NEXT:    v_mov_b32_e32 v0, s0
281; GFX7-NEXT:    v_mov_b32_e32 v1, s1
282; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
283; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
284; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
285; GFX7-NEXT:    buffer_wbinvl1_vol
286; GFX7-NEXT:    v_mov_b32_e32 v2, s2
287; GFX7-NEXT:    v_mov_b32_e32 v3, s3
288; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
289; GFX7-NEXT:    flat_store_dword v[2:3], v0
290; GFX7-NEXT:    s_endpgm
291;
292; GFX10-WGP-LABEL: flat_agent_seq_cst_load:
293; GFX10-WGP:       ; %bb.0: ; %entry
294; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
295; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
296; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
297; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
298; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
299; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
300; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
301; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
302; GFX10-WGP-NEXT:    buffer_gl0_inv
303; GFX10-WGP-NEXT:    buffer_gl1_inv
304; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
305; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
306; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
307; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
308; GFX10-WGP-NEXT:    s_endpgm
309;
310; GFX10-CU-LABEL: flat_agent_seq_cst_load:
311; GFX10-CU:       ; %bb.0: ; %entry
312; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
313; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
314; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
315; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
316; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
317; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
318; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
319; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
320; GFX10-CU-NEXT:    buffer_gl0_inv
321; GFX10-CU-NEXT:    buffer_gl1_inv
322; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
323; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
324; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
325; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
326; GFX10-CU-NEXT:    s_endpgm
327;
328; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_load:
329; SKIP-CACHE-INV:       ; %bb.0: ; %entry
330; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
331; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
332; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
333; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
334; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
335; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1] glc
336; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
337; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
338; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
339; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
340; SKIP-CACHE-INV-NEXT:    s_endpgm
341;
342; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_load:
343; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
344; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
345; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
346; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
347; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
348; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
349; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v0, v[0:1] glc
350; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
351; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
352; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
353; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
354; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
355; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[2:3], v0
356; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
357;
358; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_load:
359; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
360; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
361; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
362; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
363; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
364; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
365; GFX90A-TGSPLIT-NEXT:    flat_load_dword v0, v[0:1] glc
366; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
367; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
368; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
369; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
370; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[2:3], v0
371; GFX90A-TGSPLIT-NEXT:    s_endpgm
372    i32* %in, i32* %out) {
373entry:
374  %val = load atomic i32, i32* %in syncscope("agent") seq_cst, align 4
375  store i32 %val, i32* %out
376  ret void
377}
378
379define amdgpu_kernel void @flat_agent_unordered_store(
380; GFX7-LABEL: flat_agent_unordered_store:
381; GFX7:       ; %bb.0: ; %entry
382; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
383; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
384; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
385; GFX7-NEXT:    v_mov_b32_e32 v2, s2
386; GFX7-NEXT:    v_mov_b32_e32 v0, s0
387; GFX7-NEXT:    v_mov_b32_e32 v1, s1
388; GFX7-NEXT:    flat_store_dword v[0:1], v2
389; GFX7-NEXT:    s_endpgm
390;
391; GFX10-WGP-LABEL: flat_agent_unordered_store:
392; GFX10-WGP:       ; %bb.0: ; %entry
393; GFX10-WGP-NEXT:    s_clause 0x1
394; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
395; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
396; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
397; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
398; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
399; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
400; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
401; GFX10-WGP-NEXT:    s_endpgm
402;
403; GFX10-CU-LABEL: flat_agent_unordered_store:
404; GFX10-CU:       ; %bb.0: ; %entry
405; GFX10-CU-NEXT:    s_clause 0x1
406; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
407; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
408; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
409; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
410; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
411; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
412; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
413; GFX10-CU-NEXT:    s_endpgm
414;
415; SKIP-CACHE-INV-LABEL: flat_agent_unordered_store:
416; SKIP-CACHE-INV:       ; %bb.0: ; %entry
417; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
418; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
419; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
420; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
421; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
422; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
423; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
424; SKIP-CACHE-INV-NEXT:    s_endpgm
425;
426; GFX90A-NOTTGSPLIT-LABEL: flat_agent_unordered_store:
427; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
428; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
429; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
430; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
431; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
432; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
433; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
434; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
435;
436; GFX90A-TGSPLIT-LABEL: flat_agent_unordered_store:
437; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
438; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
439; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
440; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
441; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
442; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
443; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
444; GFX90A-TGSPLIT-NEXT:    s_endpgm
445    i32 %in, i32* %out) {
446entry:
447  store atomic i32 %in, i32* %out syncscope("agent") unordered, align 4
448  ret void
449}
450
451define amdgpu_kernel void @flat_agent_monotonic_store(
452; GFX7-LABEL: flat_agent_monotonic_store:
453; GFX7:       ; %bb.0: ; %entry
454; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
455; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
456; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
457; GFX7-NEXT:    v_mov_b32_e32 v2, s2
458; GFX7-NEXT:    v_mov_b32_e32 v0, s0
459; GFX7-NEXT:    v_mov_b32_e32 v1, s1
460; GFX7-NEXT:    flat_store_dword v[0:1], v2
461; GFX7-NEXT:    s_endpgm
462;
463; GFX10-WGP-LABEL: flat_agent_monotonic_store:
464; GFX10-WGP:       ; %bb.0: ; %entry
465; GFX10-WGP-NEXT:    s_clause 0x1
466; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
467; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
468; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
469; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
470; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
471; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
472; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
473; GFX10-WGP-NEXT:    s_endpgm
474;
475; GFX10-CU-LABEL: flat_agent_monotonic_store:
476; GFX10-CU:       ; %bb.0: ; %entry
477; GFX10-CU-NEXT:    s_clause 0x1
478; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
479; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
480; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
481; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
482; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
483; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
484; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
485; GFX10-CU-NEXT:    s_endpgm
486;
487; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_store:
488; SKIP-CACHE-INV:       ; %bb.0: ; %entry
489; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
490; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
491; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
492; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
493; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
494; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
495; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
496; SKIP-CACHE-INV-NEXT:    s_endpgm
497;
498; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_store:
499; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
500; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
501; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
502; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
503; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
504; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
505; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
506; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
507;
508; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_store:
509; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
510; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
511; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
512; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
513; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
514; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
515; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
516; GFX90A-TGSPLIT-NEXT:    s_endpgm
517    i32 %in, i32* %out) {
518entry:
519  store atomic i32 %in, i32* %out syncscope("agent") monotonic, align 4
520  ret void
521}
522
523define amdgpu_kernel void @flat_agent_release_store(
524; GFX7-LABEL: flat_agent_release_store:
525; GFX7:       ; %bb.0: ; %entry
526; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
527; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
528; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
529; GFX7-NEXT:    v_mov_b32_e32 v2, s2
530; GFX7-NEXT:    v_mov_b32_e32 v0, s0
531; GFX7-NEXT:    v_mov_b32_e32 v1, s1
532; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
533; GFX7-NEXT:    flat_store_dword v[0:1], v2
534; GFX7-NEXT:    s_endpgm
535;
536; GFX10-WGP-LABEL: flat_agent_release_store:
537; GFX10-WGP:       ; %bb.0: ; %entry
538; GFX10-WGP-NEXT:    s_clause 0x1
539; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
540; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
541; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
542; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
543; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
544; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
545; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
546; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
547; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
548; GFX10-WGP-NEXT:    s_endpgm
549;
550; GFX10-CU-LABEL: flat_agent_release_store:
551; GFX10-CU:       ; %bb.0: ; %entry
552; GFX10-CU-NEXT:    s_clause 0x1
553; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
554; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
555; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
556; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
557; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
558; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
559; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
560; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
561; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
562; GFX10-CU-NEXT:    s_endpgm
563;
564; SKIP-CACHE-INV-LABEL: flat_agent_release_store:
565; SKIP-CACHE-INV:       ; %bb.0: ; %entry
566; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
567; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
568; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
569; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
570; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
571; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
572; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
573; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
574; SKIP-CACHE-INV-NEXT:    s_endpgm
575;
576; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_store:
577; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
578; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
579; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
580; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
581; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
582; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
583; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
584; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
585; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
586;
587; GFX90A-TGSPLIT-LABEL: flat_agent_release_store:
588; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
589; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
590; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
591; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
592; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
593; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
594; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
595; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
596; GFX90A-TGSPLIT-NEXT:    s_endpgm
597    i32 %in, i32* %out) {
598entry:
599  store atomic i32 %in, i32* %out syncscope("agent") release, align 4
600  ret void
601}
602
603define amdgpu_kernel void @flat_agent_seq_cst_store(
604; GFX7-LABEL: flat_agent_seq_cst_store:
605; GFX7:       ; %bb.0: ; %entry
606; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
607; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
608; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
609; GFX7-NEXT:    v_mov_b32_e32 v2, s2
610; GFX7-NEXT:    v_mov_b32_e32 v0, s0
611; GFX7-NEXT:    v_mov_b32_e32 v1, s1
612; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
613; GFX7-NEXT:    flat_store_dword v[0:1], v2
614; GFX7-NEXT:    s_endpgm
615;
616; GFX10-WGP-LABEL: flat_agent_seq_cst_store:
617; GFX10-WGP:       ; %bb.0: ; %entry
618; GFX10-WGP-NEXT:    s_clause 0x1
619; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
620; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
621; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
622; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
623; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
624; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
625; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
626; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
627; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
628; GFX10-WGP-NEXT:    s_endpgm
629;
630; GFX10-CU-LABEL: flat_agent_seq_cst_store:
631; GFX10-CU:       ; %bb.0: ; %entry
632; GFX10-CU-NEXT:    s_clause 0x1
633; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
634; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
635; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
636; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
637; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
638; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
639; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
640; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
641; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
642; GFX10-CU-NEXT:    s_endpgm
643;
644; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_store:
645; SKIP-CACHE-INV:       ; %bb.0: ; %entry
646; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
647; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
648; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
649; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
650; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
651; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
652; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
653; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
654; SKIP-CACHE-INV-NEXT:    s_endpgm
655;
656; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_store:
657; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
658; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
659; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
660; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
661; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
662; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
663; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
664; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
665; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
666;
667; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_store:
668; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
669; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
670; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
671; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
672; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
673; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
674; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
675; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
676; GFX90A-TGSPLIT-NEXT:    s_endpgm
677    i32 %in, i32* %out) {
678entry:
679  store atomic i32 %in, i32* %out syncscope("agent") seq_cst, align 4
680  ret void
681}
682
683define amdgpu_kernel void @flat_agent_monotonic_atomicrmw(
684; GFX7-LABEL: flat_agent_monotonic_atomicrmw:
685; GFX7:       ; %bb.0: ; %entry
686; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
687; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
688; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
689; GFX7-NEXT:    v_mov_b32_e32 v0, s0
690; GFX7-NEXT:    v_mov_b32_e32 v1, s1
691; GFX7-NEXT:    v_mov_b32_e32 v2, s2
692; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
693; GFX7-NEXT:    s_endpgm
694;
695; GFX10-WGP-LABEL: flat_agent_monotonic_atomicrmw:
696; GFX10-WGP:       ; %bb.0: ; %entry
697; GFX10-WGP-NEXT:    s_clause 0x1
698; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
699; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
700; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
701; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
702; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
703; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
704; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
705; GFX10-WGP-NEXT:    s_endpgm
706;
707; GFX10-CU-LABEL: flat_agent_monotonic_atomicrmw:
708; GFX10-CU:       ; %bb.0: ; %entry
709; GFX10-CU-NEXT:    s_clause 0x1
710; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
711; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
712; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
713; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
714; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
715; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
716; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
717; GFX10-CU-NEXT:    s_endpgm
718;
719; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_atomicrmw:
720; SKIP-CACHE-INV:       ; %bb.0: ; %entry
721; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
722; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
723; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
724; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
725; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
726; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
727; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
728; SKIP-CACHE-INV-NEXT:    s_endpgm
729;
730; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_atomicrmw:
731; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
732; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
733; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
734; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
735; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
736; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
737; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
738; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
739;
740; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_atomicrmw:
741; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
742; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
743; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
744; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
745; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
746; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
747; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
748; GFX90A-TGSPLIT-NEXT:    s_endpgm
749    i32* %out, i32 %in) {
750entry:
751  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") monotonic
752  ret void
753}
754
755define amdgpu_kernel void @flat_agent_acquire_atomicrmw(
756; GFX7-LABEL: flat_agent_acquire_atomicrmw:
757; GFX7:       ; %bb.0: ; %entry
758; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
759; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
760; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
761; GFX7-NEXT:    v_mov_b32_e32 v0, s0
762; GFX7-NEXT:    v_mov_b32_e32 v1, s1
763; GFX7-NEXT:    v_mov_b32_e32 v2, s2
764; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
765; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
766; GFX7-NEXT:    buffer_wbinvl1_vol
767; GFX7-NEXT:    s_endpgm
768;
769; GFX10-WGP-LABEL: flat_agent_acquire_atomicrmw:
770; GFX10-WGP:       ; %bb.0: ; %entry
771; GFX10-WGP-NEXT:    s_clause 0x1
772; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
773; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
774; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
775; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
776; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
777; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
778; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
779; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
780; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
781; GFX10-WGP-NEXT:    buffer_gl0_inv
782; GFX10-WGP-NEXT:    buffer_gl1_inv
783; GFX10-WGP-NEXT:    s_endpgm
784;
785; GFX10-CU-LABEL: flat_agent_acquire_atomicrmw:
786; GFX10-CU:       ; %bb.0: ; %entry
787; GFX10-CU-NEXT:    s_clause 0x1
788; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
789; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
790; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
791; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
792; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
793; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
794; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
795; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
796; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
797; GFX10-CU-NEXT:    buffer_gl0_inv
798; GFX10-CU-NEXT:    buffer_gl1_inv
799; GFX10-CU-NEXT:    s_endpgm
800;
801; SKIP-CACHE-INV-LABEL: flat_agent_acquire_atomicrmw:
802; SKIP-CACHE-INV:       ; %bb.0: ; %entry
803; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
804; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
805; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
806; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
807; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
808; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
809; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
810; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
811; SKIP-CACHE-INV-NEXT:    s_endpgm
812;
813; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_atomicrmw:
814; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
815; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
816; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
817; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
818; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
819; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
820; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
821; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
822; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
823; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
824;
825; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_atomicrmw:
826; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
827; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
828; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
829; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
830; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
831; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
832; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
833; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
834; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
835; GFX90A-TGSPLIT-NEXT:    s_endpgm
836    i32* %out, i32 %in) {
837entry:
838  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") acquire
839  ret void
840}
841
842define amdgpu_kernel void @flat_agent_release_atomicrmw(
843; GFX7-LABEL: flat_agent_release_atomicrmw:
844; GFX7:       ; %bb.0: ; %entry
845; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
846; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
847; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
848; GFX7-NEXT:    v_mov_b32_e32 v0, s0
849; GFX7-NEXT:    v_mov_b32_e32 v1, s1
850; GFX7-NEXT:    v_mov_b32_e32 v2, s2
851; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
852; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
853; GFX7-NEXT:    s_endpgm
854;
855; GFX10-WGP-LABEL: flat_agent_release_atomicrmw:
856; GFX10-WGP:       ; %bb.0: ; %entry
857; GFX10-WGP-NEXT:    s_clause 0x1
858; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
859; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
860; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
861; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
862; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
863; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
864; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
865; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
866; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
867; GFX10-WGP-NEXT:    s_endpgm
868;
869; GFX10-CU-LABEL: flat_agent_release_atomicrmw:
870; GFX10-CU:       ; %bb.0: ; %entry
871; GFX10-CU-NEXT:    s_clause 0x1
872; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
873; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
874; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
875; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
876; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
877; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
878; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
879; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
880; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
881; GFX10-CU-NEXT:    s_endpgm
882;
883; SKIP-CACHE-INV-LABEL: flat_agent_release_atomicrmw:
884; SKIP-CACHE-INV:       ; %bb.0: ; %entry
885; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
886; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
887; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
888; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
889; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
890; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
891; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
892; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
893; SKIP-CACHE-INV-NEXT:    s_endpgm
894;
895; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_atomicrmw:
896; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
897; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
898; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
899; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
900; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
901; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
902; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
903; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
904; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
905;
906; GFX90A-TGSPLIT-LABEL: flat_agent_release_atomicrmw:
907; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
908; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
909; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
910; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
911; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
912; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
913; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
914; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
915; GFX90A-TGSPLIT-NEXT:    s_endpgm
916    i32* %out, i32 %in) {
917entry:
918  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") release
919  ret void
920}
921
922define amdgpu_kernel void @flat_agent_acq_rel_atomicrmw(
923; GFX7-LABEL: flat_agent_acq_rel_atomicrmw:
924; GFX7:       ; %bb.0: ; %entry
925; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
926; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
927; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
928; GFX7-NEXT:    v_mov_b32_e32 v0, s0
929; GFX7-NEXT:    v_mov_b32_e32 v1, s1
930; GFX7-NEXT:    v_mov_b32_e32 v2, s2
931; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
932; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
933; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
934; GFX7-NEXT:    buffer_wbinvl1_vol
935; GFX7-NEXT:    s_endpgm
936;
937; GFX10-WGP-LABEL: flat_agent_acq_rel_atomicrmw:
938; GFX10-WGP:       ; %bb.0: ; %entry
939; GFX10-WGP-NEXT:    s_clause 0x1
940; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
941; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
942; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
943; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
944; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
945; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
946; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
947; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
948; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
949; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
950; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
951; GFX10-WGP-NEXT:    buffer_gl0_inv
952; GFX10-WGP-NEXT:    buffer_gl1_inv
953; GFX10-WGP-NEXT:    s_endpgm
954;
955; GFX10-CU-LABEL: flat_agent_acq_rel_atomicrmw:
956; GFX10-CU:       ; %bb.0: ; %entry
957; GFX10-CU-NEXT:    s_clause 0x1
958; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
959; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
960; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
961; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
962; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
963; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
964; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
965; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
966; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
967; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
968; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
969; GFX10-CU-NEXT:    buffer_gl0_inv
970; GFX10-CU-NEXT:    buffer_gl1_inv
971; GFX10-CU-NEXT:    s_endpgm
972;
973; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_atomicrmw:
974; SKIP-CACHE-INV:       ; %bb.0: ; %entry
975; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
976; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
977; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
978; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
979; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
980; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
981; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
982; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
983; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
984; SKIP-CACHE-INV-NEXT:    s_endpgm
985;
986; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_atomicrmw:
987; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
988; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
989; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
990; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
991; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
992; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
993; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
994; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
995; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
996; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
997; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
998;
999; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_atomicrmw:
1000; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1001; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1002; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1003; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1004; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1005; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1006; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1007; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1008; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1009; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1010; GFX90A-TGSPLIT-NEXT:    s_endpgm
1011    i32* %out, i32 %in) {
1012entry:
1013  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") acq_rel
1014  ret void
1015}
1016
1017define amdgpu_kernel void @flat_agent_seq_cst_atomicrmw(
1018; GFX7-LABEL: flat_agent_seq_cst_atomicrmw:
1019; GFX7:       ; %bb.0: ; %entry
1020; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1021; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1022; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1023; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1024; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1025; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1026; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1027; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1028; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1029; GFX7-NEXT:    buffer_wbinvl1_vol
1030; GFX7-NEXT:    s_endpgm
1031;
1032; GFX10-WGP-LABEL: flat_agent_seq_cst_atomicrmw:
1033; GFX10-WGP:       ; %bb.0: ; %entry
1034; GFX10-WGP-NEXT:    s_clause 0x1
1035; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1036; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1037; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1038; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1039; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1040; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1041; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1042; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1043; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1044; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1045; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1046; GFX10-WGP-NEXT:    buffer_gl0_inv
1047; GFX10-WGP-NEXT:    buffer_gl1_inv
1048; GFX10-WGP-NEXT:    s_endpgm
1049;
1050; GFX10-CU-LABEL: flat_agent_seq_cst_atomicrmw:
1051; GFX10-CU:       ; %bb.0: ; %entry
1052; GFX10-CU-NEXT:    s_clause 0x1
1053; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1054; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1055; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1056; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1057; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1058; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1059; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1060; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1061; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1062; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1063; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1064; GFX10-CU-NEXT:    buffer_gl0_inv
1065; GFX10-CU-NEXT:    buffer_gl1_inv
1066; GFX10-CU-NEXT:    s_endpgm
1067;
1068; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_atomicrmw:
1069; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1070; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
1071; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
1072; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1073; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1074; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1075; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1076; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1077; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1078; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1079; SKIP-CACHE-INV-NEXT:    s_endpgm
1080;
1081; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_atomicrmw:
1082; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1083; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1084; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1085; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1086; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1087; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1088; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1089; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1090; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1091; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1092; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1093;
1094; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_atomicrmw:
1095; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1096; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1097; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1098; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1099; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1100; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1101; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1102; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1103; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1104; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1105; GFX90A-TGSPLIT-NEXT:    s_endpgm
1106    i32* %out, i32 %in) {
1107entry:
1108  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") seq_cst
1109  ret void
1110}
1111
1112define amdgpu_kernel void @flat_agent_acquire_ret_atomicrmw(
1113; GFX7-LABEL: flat_agent_acquire_ret_atomicrmw:
1114; GFX7:       ; %bb.0: ; %entry
1115; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1116; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1117; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1118; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1119; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1120; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1121; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1122; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1123; GFX7-NEXT:    buffer_wbinvl1_vol
1124; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1125; GFX7-NEXT:    flat_store_dword v[0:1], v2
1126; GFX7-NEXT:    s_endpgm
1127;
1128; GFX10-WGP-LABEL: flat_agent_acquire_ret_atomicrmw:
1129; GFX10-WGP:       ; %bb.0: ; %entry
1130; GFX10-WGP-NEXT:    s_clause 0x1
1131; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1132; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1133; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1134; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1135; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1136; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1137; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1138; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1139; GFX10-WGP-NEXT:    buffer_gl0_inv
1140; GFX10-WGP-NEXT:    buffer_gl1_inv
1141; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1142; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1143; GFX10-WGP-NEXT:    s_endpgm
1144;
1145; GFX10-CU-LABEL: flat_agent_acquire_ret_atomicrmw:
1146; GFX10-CU:       ; %bb.0: ; %entry
1147; GFX10-CU-NEXT:    s_clause 0x1
1148; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1149; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1150; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1151; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1152; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1153; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1154; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1155; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1156; GFX10-CU-NEXT:    buffer_gl0_inv
1157; GFX10-CU-NEXT:    buffer_gl1_inv
1158; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1159; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1160; GFX10-CU-NEXT:    s_endpgm
1161;
1162; SKIP-CACHE-INV-LABEL: flat_agent_acquire_ret_atomicrmw:
1163; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1164; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
1165; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
1166; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1167; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1168; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1169; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1170; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1171; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1172; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1173; SKIP-CACHE-INV-NEXT:    s_endpgm
1174;
1175; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_ret_atomicrmw:
1176; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1177; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1178; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1179; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1180; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1181; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1182; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1183; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1184; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1185; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1186; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1187; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1188;
1189; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_ret_atomicrmw:
1190; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1191; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1192; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1193; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1194; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1195; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1196; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1197; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1198; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1199; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1200; GFX90A-TGSPLIT-NEXT:    s_endpgm
1201    i32* %out, i32 %in) {
1202entry:
1203  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") acquire
1204  store i32 %val, i32* %out, align 4
1205  ret void
1206}
1207
1208define amdgpu_kernel void @flat_agent_acq_rel_ret_atomicrmw(
1209; GFX7-LABEL: flat_agent_acq_rel_ret_atomicrmw:
1210; GFX7:       ; %bb.0: ; %entry
1211; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1212; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1213; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1214; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1215; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1216; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1217; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1218; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1219; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1220; GFX7-NEXT:    buffer_wbinvl1_vol
1221; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1222; GFX7-NEXT:    flat_store_dword v[0:1], v2
1223; GFX7-NEXT:    s_endpgm
1224;
1225; GFX10-WGP-LABEL: flat_agent_acq_rel_ret_atomicrmw:
1226; GFX10-WGP:       ; %bb.0: ; %entry
1227; GFX10-WGP-NEXT:    s_clause 0x1
1228; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1229; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1230; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1231; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1232; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1233; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1234; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1235; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1236; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1237; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1238; GFX10-WGP-NEXT:    buffer_gl0_inv
1239; GFX10-WGP-NEXT:    buffer_gl1_inv
1240; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1241; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1242; GFX10-WGP-NEXT:    s_endpgm
1243;
1244; GFX10-CU-LABEL: flat_agent_acq_rel_ret_atomicrmw:
1245; GFX10-CU:       ; %bb.0: ; %entry
1246; GFX10-CU-NEXT:    s_clause 0x1
1247; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1248; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1249; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1250; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1251; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1252; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1253; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1254; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1255; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1256; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1257; GFX10-CU-NEXT:    buffer_gl0_inv
1258; GFX10-CU-NEXT:    buffer_gl1_inv
1259; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1260; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1261; GFX10-CU-NEXT:    s_endpgm
1262;
1263; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_ret_atomicrmw:
1264; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1265; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
1266; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
1267; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1268; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1269; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1270; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1271; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1272; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1273; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1274; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1275; SKIP-CACHE-INV-NEXT:    s_endpgm
1276;
1277; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_ret_atomicrmw:
1278; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1279; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1280; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1281; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1282; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1283; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1284; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1285; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1286; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1287; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1288; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1289; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1290; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1291;
1292; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_ret_atomicrmw:
1293; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1294; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1295; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1296; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1297; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1298; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1299; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1300; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1301; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1302; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1303; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1304; GFX90A-TGSPLIT-NEXT:    s_endpgm
1305    i32* %out, i32 %in) {
1306entry:
1307  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") acq_rel
1308  store i32 %val, i32* %out, align 4
1309  ret void
1310}
1311
1312define amdgpu_kernel void @flat_agent_seq_cst_ret_atomicrmw(
1313; GFX7-LABEL: flat_agent_seq_cst_ret_atomicrmw:
1314; GFX7:       ; %bb.0: ; %entry
1315; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1316; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1317; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1318; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1319; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1320; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1321; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1322; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1323; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1324; GFX7-NEXT:    buffer_wbinvl1_vol
1325; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1326; GFX7-NEXT:    flat_store_dword v[0:1], v2
1327; GFX7-NEXT:    s_endpgm
1328;
1329; GFX10-WGP-LABEL: flat_agent_seq_cst_ret_atomicrmw:
1330; GFX10-WGP:       ; %bb.0: ; %entry
1331; GFX10-WGP-NEXT:    s_clause 0x1
1332; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1333; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1334; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1335; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1336; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1337; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1338; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1339; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1340; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1341; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1342; GFX10-WGP-NEXT:    buffer_gl0_inv
1343; GFX10-WGP-NEXT:    buffer_gl1_inv
1344; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1345; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1346; GFX10-WGP-NEXT:    s_endpgm
1347;
1348; GFX10-CU-LABEL: flat_agent_seq_cst_ret_atomicrmw:
1349; GFX10-CU:       ; %bb.0: ; %entry
1350; GFX10-CU-NEXT:    s_clause 0x1
1351; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1352; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1353; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1354; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1355; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1356; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1357; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1358; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1359; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1360; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1361; GFX10-CU-NEXT:    buffer_gl0_inv
1362; GFX10-CU-NEXT:    buffer_gl1_inv
1363; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1364; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1365; GFX10-CU-NEXT:    s_endpgm
1366;
1367; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_ret_atomicrmw:
1368; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1369; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
1370; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
1371; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1372; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1373; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1374; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1375; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1376; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1377; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1378; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1379; SKIP-CACHE-INV-NEXT:    s_endpgm
1380;
1381; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_ret_atomicrmw:
1382; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1383; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1384; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1385; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1386; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1387; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1388; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1389; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1390; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1391; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1392; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1393; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1394; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1395;
1396; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_ret_atomicrmw:
1397; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1398; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1399; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1400; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1401; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1402; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1403; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1404; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1405; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1406; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1407; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1408; GFX90A-TGSPLIT-NEXT:    s_endpgm
1409    i32* %out, i32 %in) {
1410entry:
1411  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") seq_cst
1412  store i32 %val, i32* %out, align 4
1413  ret void
1414}
1415
1416define amdgpu_kernel void @flat_agent_monotonic_monotonic_cmpxchg(
1417; GFX7-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
1418; GFX7:       ; %bb.0: ; %entry
1419; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1420; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1421; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1422; GFX7-NEXT:    s_add_u32 s0, s0, 16
1423; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1424; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1425; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1426; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1427; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1428; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1429; GFX7-NEXT:    s_endpgm
1430;
1431; GFX10-WGP-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
1432; GFX10-WGP:       ; %bb.0: ; %entry
1433; GFX10-WGP-NEXT:    s_clause 0x1
1434; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1435; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1436; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1437; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1438; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1439; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1440; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1441; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1442; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1443; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1444; GFX10-WGP-NEXT:    s_endpgm
1445;
1446; GFX10-CU-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
1447; GFX10-CU:       ; %bb.0: ; %entry
1448; GFX10-CU-NEXT:    s_clause 0x1
1449; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1450; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1451; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1452; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1453; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1454; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1455; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1456; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1457; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1458; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1459; GFX10-CU-NEXT:    s_endpgm
1460;
1461; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
1462; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1463; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
1464; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
1465; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1466; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1467; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1468; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1469; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1470; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1471; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1472; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1473; SKIP-CACHE-INV-NEXT:    s_endpgm
1474;
1475; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
1476; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1477; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1478; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1479; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1480; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1481; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1482; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1483; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1484;
1485; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
1486; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1487; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1488; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1489; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1490; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1491; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1492; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1493; GFX90A-TGSPLIT-NEXT:    s_endpgm
1494    i32* %out, i32 %in, i32 %old) {
1495entry:
1496  %gep = getelementptr i32, i32* %out, i32 4
1497  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") monotonic monotonic
1498  ret void
1499}
1500
1501define amdgpu_kernel void @flat_agent_acquire_monotonic_cmpxchg(
1502; GFX7-LABEL: flat_agent_acquire_monotonic_cmpxchg:
1503; GFX7:       ; %bb.0: ; %entry
1504; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1505; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1506; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1507; GFX7-NEXT:    s_add_u32 s0, s0, 16
1508; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1509; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1510; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1511; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1512; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1513; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1514; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1515; GFX7-NEXT:    buffer_wbinvl1_vol
1516; GFX7-NEXT:    s_endpgm
1517;
1518; GFX10-WGP-LABEL: flat_agent_acquire_monotonic_cmpxchg:
1519; GFX10-WGP:       ; %bb.0: ; %entry
1520; GFX10-WGP-NEXT:    s_clause 0x1
1521; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1522; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1523; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1524; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1525; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1526; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1527; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1528; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1529; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1530; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1531; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1532; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1533; GFX10-WGP-NEXT:    buffer_gl0_inv
1534; GFX10-WGP-NEXT:    buffer_gl1_inv
1535; GFX10-WGP-NEXT:    s_endpgm
1536;
1537; GFX10-CU-LABEL: flat_agent_acquire_monotonic_cmpxchg:
1538; GFX10-CU:       ; %bb.0: ; %entry
1539; GFX10-CU-NEXT:    s_clause 0x1
1540; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1541; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1542; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1543; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1544; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1545; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1546; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1547; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1548; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1549; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1550; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1551; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1552; GFX10-CU-NEXT:    buffer_gl0_inv
1553; GFX10-CU-NEXT:    buffer_gl1_inv
1554; GFX10-CU-NEXT:    s_endpgm
1555;
1556; SKIP-CACHE-INV-LABEL: flat_agent_acquire_monotonic_cmpxchg:
1557; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1558; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
1559; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
1560; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1561; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1562; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1563; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1564; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1565; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1566; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1567; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1568; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1569; SKIP-CACHE-INV-NEXT:    s_endpgm
1570;
1571; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_monotonic_cmpxchg:
1572; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1573; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1574; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1575; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1576; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1577; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1578; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1579; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1580; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1581; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1582;
1583; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_monotonic_cmpxchg:
1584; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1585; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1586; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1587; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1588; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1589; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1590; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1591; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1592; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1593; GFX90A-TGSPLIT-NEXT:    s_endpgm
1594    i32* %out, i32 %in, i32 %old) {
1595entry:
1596  %gep = getelementptr i32, i32* %out, i32 4
1597  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire monotonic
1598  ret void
1599}
1600
1601define amdgpu_kernel void @flat_agent_release_monotonic_cmpxchg(
1602; GFX7-LABEL: flat_agent_release_monotonic_cmpxchg:
1603; GFX7:       ; %bb.0: ; %entry
1604; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1605; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1606; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1607; GFX7-NEXT:    s_add_u32 s0, s0, 16
1608; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1609; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1610; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1611; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1612; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1613; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1614; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1615; GFX7-NEXT:    s_endpgm
1616;
1617; GFX10-WGP-LABEL: flat_agent_release_monotonic_cmpxchg:
1618; GFX10-WGP:       ; %bb.0: ; %entry
1619; GFX10-WGP-NEXT:    s_clause 0x1
1620; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1621; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1622; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1623; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1624; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1625; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1626; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1627; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1628; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1629; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1630; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1631; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1632; GFX10-WGP-NEXT:    s_endpgm
1633;
1634; GFX10-CU-LABEL: flat_agent_release_monotonic_cmpxchg:
1635; GFX10-CU:       ; %bb.0: ; %entry
1636; GFX10-CU-NEXT:    s_clause 0x1
1637; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1638; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1639; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1640; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1641; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1642; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1643; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1644; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1645; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1646; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1647; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1648; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1649; GFX10-CU-NEXT:    s_endpgm
1650;
1651; SKIP-CACHE-INV-LABEL: flat_agent_release_monotonic_cmpxchg:
1652; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1653; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
1654; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
1655; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1656; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1657; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1658; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1659; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1660; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1661; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1662; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1663; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1664; SKIP-CACHE-INV-NEXT:    s_endpgm
1665;
1666; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_monotonic_cmpxchg:
1667; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1668; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1669; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1670; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1671; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1672; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1673; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1674; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1675; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1676;
1677; GFX90A-TGSPLIT-LABEL: flat_agent_release_monotonic_cmpxchg:
1678; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1679; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1680; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1681; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1682; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1683; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1684; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1685; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1686; GFX90A-TGSPLIT-NEXT:    s_endpgm
1687    i32* %out, i32 %in, i32 %old) {
1688entry:
1689  %gep = getelementptr i32, i32* %out, i32 4
1690  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release monotonic
1691  ret void
1692}
1693
1694define amdgpu_kernel void @flat_agent_acq_rel_monotonic_cmpxchg(
1695; GFX7-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
1696; GFX7:       ; %bb.0: ; %entry
1697; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1698; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1699; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1700; GFX7-NEXT:    s_add_u32 s0, s0, 16
1701; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1702; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1703; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1704; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1705; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1706; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1707; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1708; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1709; GFX7-NEXT:    buffer_wbinvl1_vol
1710; GFX7-NEXT:    s_endpgm
1711;
1712; GFX10-WGP-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
1713; GFX10-WGP:       ; %bb.0: ; %entry
1714; GFX10-WGP-NEXT:    s_clause 0x1
1715; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1716; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1717; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1718; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1719; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1720; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1721; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1722; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1723; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1724; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1725; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1726; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1727; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1728; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1729; GFX10-WGP-NEXT:    buffer_gl0_inv
1730; GFX10-WGP-NEXT:    buffer_gl1_inv
1731; GFX10-WGP-NEXT:    s_endpgm
1732;
1733; GFX10-CU-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
1734; GFX10-CU:       ; %bb.0: ; %entry
1735; GFX10-CU-NEXT:    s_clause 0x1
1736; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1737; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1738; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1739; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1740; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1741; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1742; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1743; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1744; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1745; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1746; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1747; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1748; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1749; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1750; GFX10-CU-NEXT:    buffer_gl0_inv
1751; GFX10-CU-NEXT:    buffer_gl1_inv
1752; GFX10-CU-NEXT:    s_endpgm
1753;
1754; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
1755; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1756; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
1757; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
1758; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1759; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1760; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1761; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1762; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1763; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1764; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1765; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1766; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1767; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1768; SKIP-CACHE-INV-NEXT:    s_endpgm
1769;
1770; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
1771; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1772; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1773; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1774; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1775; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1776; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1777; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1778; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1779; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1780; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1781; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1782;
1783; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
1784; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1785; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1786; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1787; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1788; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1789; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1790; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1791; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1792; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1793; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1794; GFX90A-TGSPLIT-NEXT:    s_endpgm
1795    i32* %out, i32 %in, i32 %old) {
1796entry:
1797  %gep = getelementptr i32, i32* %out, i32 4
1798  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel monotonic
1799  ret void
1800}
1801
1802define amdgpu_kernel void @flat_agent_seq_cst_monotonic_cmpxchg(
1803; GFX7-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
1804; GFX7:       ; %bb.0: ; %entry
1805; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1806; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1807; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1808; GFX7-NEXT:    s_add_u32 s0, s0, 16
1809; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1810; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1811; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1812; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1813; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1814; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1815; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1816; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1817; GFX7-NEXT:    buffer_wbinvl1_vol
1818; GFX7-NEXT:    s_endpgm
1819;
1820; GFX10-WGP-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
1821; GFX10-WGP:       ; %bb.0: ; %entry
1822; GFX10-WGP-NEXT:    s_clause 0x1
1823; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1824; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1825; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1826; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1827; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1828; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1829; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1830; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1831; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1832; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1833; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1834; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1835; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1836; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1837; GFX10-WGP-NEXT:    buffer_gl0_inv
1838; GFX10-WGP-NEXT:    buffer_gl1_inv
1839; GFX10-WGP-NEXT:    s_endpgm
1840;
1841; GFX10-CU-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
1842; GFX10-CU:       ; %bb.0: ; %entry
1843; GFX10-CU-NEXT:    s_clause 0x1
1844; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1845; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1846; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1847; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1848; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1849; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1850; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1851; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1852; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1853; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1854; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1855; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1856; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1857; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1858; GFX10-CU-NEXT:    buffer_gl0_inv
1859; GFX10-CU-NEXT:    buffer_gl1_inv
1860; GFX10-CU-NEXT:    s_endpgm
1861;
1862; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
1863; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1864; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
1865; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
1866; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1867; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1868; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1869; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1870; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1871; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1872; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1873; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1874; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1875; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1876; SKIP-CACHE-INV-NEXT:    s_endpgm
1877;
1878; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
1879; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1880; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1881; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1882; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1883; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1884; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1885; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1886; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1887; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1888; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1889; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1890;
1891; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
1892; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1893; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1894; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1895; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1896; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1897; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1898; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1899; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1900; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1901; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1902; GFX90A-TGSPLIT-NEXT:    s_endpgm
1903    i32* %out, i32 %in, i32 %old) {
1904entry:
1905  %gep = getelementptr i32, i32* %out, i32 4
1906  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst monotonic
1907  ret void
1908}
1909
1910define amdgpu_kernel void @flat_agent_acquire_acquire_cmpxchg(
1911; GFX7-LABEL: flat_agent_acquire_acquire_cmpxchg:
1912; GFX7:       ; %bb.0: ; %entry
1913; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1914; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1915; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1916; GFX7-NEXT:    s_add_u32 s0, s0, 16
1917; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1918; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1919; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1920; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1921; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1922; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1923; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1924; GFX7-NEXT:    buffer_wbinvl1_vol
1925; GFX7-NEXT:    s_endpgm
1926;
1927; GFX10-WGP-LABEL: flat_agent_acquire_acquire_cmpxchg:
1928; GFX10-WGP:       ; %bb.0: ; %entry
1929; GFX10-WGP-NEXT:    s_clause 0x1
1930; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1931; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1932; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1933; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1934; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1935; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1936; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1937; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1938; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1939; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1940; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1941; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1942; GFX10-WGP-NEXT:    buffer_gl0_inv
1943; GFX10-WGP-NEXT:    buffer_gl1_inv
1944; GFX10-WGP-NEXT:    s_endpgm
1945;
1946; GFX10-CU-LABEL: flat_agent_acquire_acquire_cmpxchg:
1947; GFX10-CU:       ; %bb.0: ; %entry
1948; GFX10-CU-NEXT:    s_clause 0x1
1949; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1950; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1951; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1952; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1953; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1954; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1955; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1956; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1957; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1958; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1959; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1960; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1961; GFX10-CU-NEXT:    buffer_gl0_inv
1962; GFX10-CU-NEXT:    buffer_gl1_inv
1963; GFX10-CU-NEXT:    s_endpgm
1964;
1965; SKIP-CACHE-INV-LABEL: flat_agent_acquire_acquire_cmpxchg:
1966; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1967; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
1968; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
1969; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1970; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1971; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1972; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1973; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1974; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1975; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1976; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1977; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1978; SKIP-CACHE-INV-NEXT:    s_endpgm
1979;
1980; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_acquire_cmpxchg:
1981; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1982; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1983; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1984; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1985; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1986; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1987; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1988; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1989; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1990; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1991;
1992; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_acquire_cmpxchg:
1993; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1994; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1995; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1996; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1997; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1998; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1999; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2000; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2001; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2002; GFX90A-TGSPLIT-NEXT:    s_endpgm
2003    i32* %out, i32 %in, i32 %old) {
2004entry:
2005  %gep = getelementptr i32, i32* %out, i32 4
2006  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire acquire
2007  ret void
2008}
2009
2010define amdgpu_kernel void @flat_agent_release_acquire_cmpxchg(
2011; GFX7-LABEL: flat_agent_release_acquire_cmpxchg:
2012; GFX7:       ; %bb.0: ; %entry
2013; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2014; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2015; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2016; GFX7-NEXT:    s_add_u32 s0, s0, 16
2017; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2018; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2019; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2020; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2021; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2022; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2023; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2024; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2025; GFX7-NEXT:    buffer_wbinvl1_vol
2026; GFX7-NEXT:    s_endpgm
2027;
2028; GFX10-WGP-LABEL: flat_agent_release_acquire_cmpxchg:
2029; GFX10-WGP:       ; %bb.0: ; %entry
2030; GFX10-WGP-NEXT:    s_clause 0x1
2031; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2032; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2033; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2034; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2035; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2036; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2037; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2038; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2039; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2040; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2041; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2042; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2043; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2044; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2045; GFX10-WGP-NEXT:    buffer_gl0_inv
2046; GFX10-WGP-NEXT:    buffer_gl1_inv
2047; GFX10-WGP-NEXT:    s_endpgm
2048;
2049; GFX10-CU-LABEL: flat_agent_release_acquire_cmpxchg:
2050; GFX10-CU:       ; %bb.0: ; %entry
2051; GFX10-CU-NEXT:    s_clause 0x1
2052; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2053; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2054; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2055; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2056; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2057; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2058; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2059; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2060; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2061; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2062; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2063; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2064; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2065; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2066; GFX10-CU-NEXT:    buffer_gl0_inv
2067; GFX10-CU-NEXT:    buffer_gl1_inv
2068; GFX10-CU-NEXT:    s_endpgm
2069;
2070; SKIP-CACHE-INV-LABEL: flat_agent_release_acquire_cmpxchg:
2071; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2072; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
2073; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2074; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2075; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2076; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2077; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2078; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2079; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2080; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2081; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2082; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2083; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2084; SKIP-CACHE-INV-NEXT:    s_endpgm
2085;
2086; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_acquire_cmpxchg:
2087; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2088; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2089; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2090; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2091; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2092; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2093; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2094; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2095; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2096; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2097; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2098;
2099; GFX90A-TGSPLIT-LABEL: flat_agent_release_acquire_cmpxchg:
2100; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2101; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2102; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2103; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2104; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2105; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2106; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2107; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2108; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2109; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2110; GFX90A-TGSPLIT-NEXT:    s_endpgm
2111    i32* %out, i32 %in, i32 %old) {
2112entry:
2113  %gep = getelementptr i32, i32* %out, i32 4
2114  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release acquire
2115  ret void
2116}
2117
2118define amdgpu_kernel void @flat_agent_acq_rel_acquire_cmpxchg(
2119; GFX7-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
2120; GFX7:       ; %bb.0: ; %entry
2121; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2122; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2123; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2124; GFX7-NEXT:    s_add_u32 s0, s0, 16
2125; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2126; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2127; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2128; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2129; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2130; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2131; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2132; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2133; GFX7-NEXT:    buffer_wbinvl1_vol
2134; GFX7-NEXT:    s_endpgm
2135;
2136; GFX10-WGP-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
2137; GFX10-WGP:       ; %bb.0: ; %entry
2138; GFX10-WGP-NEXT:    s_clause 0x1
2139; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2140; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2141; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2142; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2143; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2144; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2145; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2146; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2147; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2148; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2149; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2150; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2151; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2152; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2153; GFX10-WGP-NEXT:    buffer_gl0_inv
2154; GFX10-WGP-NEXT:    buffer_gl1_inv
2155; GFX10-WGP-NEXT:    s_endpgm
2156;
2157; GFX10-CU-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
2158; GFX10-CU:       ; %bb.0: ; %entry
2159; GFX10-CU-NEXT:    s_clause 0x1
2160; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2161; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2162; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2163; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2164; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2165; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2166; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2167; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2168; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2169; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2170; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2171; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2172; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2173; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2174; GFX10-CU-NEXT:    buffer_gl0_inv
2175; GFX10-CU-NEXT:    buffer_gl1_inv
2176; GFX10-CU-NEXT:    s_endpgm
2177;
2178; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
2179; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2180; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
2181; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2182; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2183; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2184; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2185; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2186; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2187; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2188; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2189; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2190; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2191; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2192; SKIP-CACHE-INV-NEXT:    s_endpgm
2193;
2194; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
2195; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2196; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2197; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2198; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2199; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2200; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2201; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2202; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2203; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2204; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2205; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2206;
2207; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
2208; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2209; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2210; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2211; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2212; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2213; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2214; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2215; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2216; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2217; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2218; GFX90A-TGSPLIT-NEXT:    s_endpgm
2219    i32* %out, i32 %in, i32 %old) {
2220entry:
2221  %gep = getelementptr i32, i32* %out, i32 4
2222  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel acquire
2223  ret void
2224}
2225
2226define amdgpu_kernel void @flat_agent_seq_cst_acquire_cmpxchg(
2227; GFX7-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
2228; GFX7:       ; %bb.0: ; %entry
2229; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2230; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2231; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2232; GFX7-NEXT:    s_add_u32 s0, s0, 16
2233; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2234; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2235; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2236; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2237; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2238; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2239; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2240; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2241; GFX7-NEXT:    buffer_wbinvl1_vol
2242; GFX7-NEXT:    s_endpgm
2243;
2244; GFX10-WGP-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
2245; GFX10-WGP:       ; %bb.0: ; %entry
2246; GFX10-WGP-NEXT:    s_clause 0x1
2247; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2248; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2249; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2250; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2251; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2252; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2253; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2254; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2255; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2256; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2257; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2258; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2259; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2260; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2261; GFX10-WGP-NEXT:    buffer_gl0_inv
2262; GFX10-WGP-NEXT:    buffer_gl1_inv
2263; GFX10-WGP-NEXT:    s_endpgm
2264;
2265; GFX10-CU-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
2266; GFX10-CU:       ; %bb.0: ; %entry
2267; GFX10-CU-NEXT:    s_clause 0x1
2268; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2269; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2270; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2271; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2272; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2273; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2274; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2275; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2276; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2277; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2278; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2279; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2280; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2281; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2282; GFX10-CU-NEXT:    buffer_gl0_inv
2283; GFX10-CU-NEXT:    buffer_gl1_inv
2284; GFX10-CU-NEXT:    s_endpgm
2285;
2286; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
2287; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2288; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
2289; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2290; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2291; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2292; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2293; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2294; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2295; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2296; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2297; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2298; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2299; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2300; SKIP-CACHE-INV-NEXT:    s_endpgm
2301;
2302; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
2303; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2304; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2305; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2306; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2307; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2308; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2309; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2310; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2311; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2312; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2313; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2314;
2315; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
2316; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2317; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2318; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2319; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2320; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2321; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2322; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2323; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2324; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2325; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2326; GFX90A-TGSPLIT-NEXT:    s_endpgm
2327    i32* %out, i32 %in, i32 %old) {
2328entry:
2329  %gep = getelementptr i32, i32* %out, i32 4
2330  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst acquire
2331  ret void
2332}
2333
2334define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_cmpxchg(
2335; GFX7-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
2336; GFX7:       ; %bb.0: ; %entry
2337; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2338; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2339; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2340; GFX7-NEXT:    s_add_u32 s0, s0, 16
2341; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2342; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2343; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2344; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2345; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2346; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2347; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2348; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2349; GFX7-NEXT:    buffer_wbinvl1_vol
2350; GFX7-NEXT:    s_endpgm
2351;
2352; GFX10-WGP-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
2353; GFX10-WGP:       ; %bb.0: ; %entry
2354; GFX10-WGP-NEXT:    s_clause 0x1
2355; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2356; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2357; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2358; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2359; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2360; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2361; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2362; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2363; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2364; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2365; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2366; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2367; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2368; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2369; GFX10-WGP-NEXT:    buffer_gl0_inv
2370; GFX10-WGP-NEXT:    buffer_gl1_inv
2371; GFX10-WGP-NEXT:    s_endpgm
2372;
2373; GFX10-CU-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
2374; GFX10-CU:       ; %bb.0: ; %entry
2375; GFX10-CU-NEXT:    s_clause 0x1
2376; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2377; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2378; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2379; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2380; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2381; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2382; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2383; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2384; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2385; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2386; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2387; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2388; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2389; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2390; GFX10-CU-NEXT:    buffer_gl0_inv
2391; GFX10-CU-NEXT:    buffer_gl1_inv
2392; GFX10-CU-NEXT:    s_endpgm
2393;
2394; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
2395; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2396; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
2397; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2398; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2399; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2400; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2401; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2402; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2403; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2404; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2405; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2406; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2407; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2408; SKIP-CACHE-INV-NEXT:    s_endpgm
2409;
2410; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
2411; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2412; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2413; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2414; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2415; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2416; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2417; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2418; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2419; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2420; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2421; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2422;
2423; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
2424; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2425; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2426; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2427; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2428; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2429; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2430; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2431; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2432; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2433; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2434; GFX90A-TGSPLIT-NEXT:    s_endpgm
2435    i32* %out, i32 %in, i32 %old) {
2436entry:
2437  %gep = getelementptr i32, i32* %out, i32 4
2438  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
2439  ret void
2440}
2441
2442define amdgpu_kernel void @flat_agent_acquire_monotonic_ret_cmpxchg(
2443; GFX7-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
2444; GFX7:       ; %bb.0: ; %entry
2445; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2446; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2447; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2448; GFX7-NEXT:    s_add_u32 s4, s0, 16
2449; GFX7-NEXT:    s_addc_u32 s5, s1, 0
2450; GFX7-NEXT:    v_mov_b32_e32 v0, s4
2451; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2452; GFX7-NEXT:    v_mov_b32_e32 v1, s5
2453; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2454; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2455; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2456; GFX7-NEXT:    buffer_wbinvl1_vol
2457; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2458; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2459; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2460; GFX7-NEXT:    flat_store_dword v[0:1], v2
2461; GFX7-NEXT:    s_endpgm
2462;
2463; GFX10-WGP-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
2464; GFX10-WGP:       ; %bb.0: ; %entry
2465; GFX10-WGP-NEXT:    s_clause 0x1
2466; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2467; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2468; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2469; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
2470; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
2471; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
2472; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2473; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
2474; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2475; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2476; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2477; GFX10-WGP-NEXT:    buffer_gl0_inv
2478; GFX10-WGP-NEXT:    buffer_gl1_inv
2479; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2480; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2481; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2482; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
2483; GFX10-WGP-NEXT:    s_endpgm
2484;
2485; GFX10-CU-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
2486; GFX10-CU:       ; %bb.0: ; %entry
2487; GFX10-CU-NEXT:    s_clause 0x1
2488; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2489; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2490; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2491; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
2492; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
2493; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
2494; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2495; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
2496; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2497; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2498; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2499; GFX10-CU-NEXT:    buffer_gl0_inv
2500; GFX10-CU-NEXT:    buffer_gl1_inv
2501; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2502; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2503; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2504; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
2505; GFX10-CU-NEXT:    s_endpgm
2506;
2507; SKIP-CACHE-INV-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
2508; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2509; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
2510; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2511; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2512; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
2513; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
2514; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2515; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2516; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2517; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2518; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2519; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2520; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2521; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2522; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
2523; SKIP-CACHE-INV-NEXT:    s_endpgm
2524;
2525; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
2526; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2527; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2528; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2529; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2530; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2531; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2532; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
2533; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2534; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2535; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2536; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2537; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2538;
2539; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
2540; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2541; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2542; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2543; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2544; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2545; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2546; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
2547; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2548; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2549; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2550; GFX90A-TGSPLIT-NEXT:    s_endpgm
2551    i32* %out, i32 %in, i32 %old) {
2552entry:
2553  %gep = getelementptr i32, i32* %out, i32 4
2554  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire monotonic
2555  %val0 = extractvalue { i32, i1 } %val, 0
2556  store i32 %val0, i32* %out, align 4
2557  ret void
2558}
2559
2560define amdgpu_kernel void @flat_agent_acq_rel_monotonic_ret_cmpxchg(
2561; GFX7-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
2562; GFX7:       ; %bb.0: ; %entry
2563; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2564; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2565; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2566; GFX7-NEXT:    s_add_u32 s4, s0, 16
2567; GFX7-NEXT:    s_addc_u32 s5, s1, 0
2568; GFX7-NEXT:    v_mov_b32_e32 v0, s4
2569; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2570; GFX7-NEXT:    v_mov_b32_e32 v1, s5
2571; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2572; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2573; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2574; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2575; GFX7-NEXT:    buffer_wbinvl1_vol
2576; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2577; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2578; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2579; GFX7-NEXT:    flat_store_dword v[0:1], v2
2580; GFX7-NEXT:    s_endpgm
2581;
2582; GFX10-WGP-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
2583; GFX10-WGP:       ; %bb.0: ; %entry
2584; GFX10-WGP-NEXT:    s_clause 0x1
2585; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2586; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2587; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2588; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
2589; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
2590; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
2591; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2592; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
2593; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2594; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2595; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2596; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2597; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2598; GFX10-WGP-NEXT:    buffer_gl0_inv
2599; GFX10-WGP-NEXT:    buffer_gl1_inv
2600; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2601; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2602; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2603; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
2604; GFX10-WGP-NEXT:    s_endpgm
2605;
2606; GFX10-CU-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
2607; GFX10-CU:       ; %bb.0: ; %entry
2608; GFX10-CU-NEXT:    s_clause 0x1
2609; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2610; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2611; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2612; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
2613; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
2614; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
2615; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2616; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
2617; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2618; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2619; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2620; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2621; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2622; GFX10-CU-NEXT:    buffer_gl0_inv
2623; GFX10-CU-NEXT:    buffer_gl1_inv
2624; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2625; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2626; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2627; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
2628; GFX10-CU-NEXT:    s_endpgm
2629;
2630; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
2631; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2632; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
2633; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2634; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2635; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
2636; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
2637; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2638; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2639; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2640; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2641; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2642; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2643; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2644; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2645; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2646; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
2647; SKIP-CACHE-INV-NEXT:    s_endpgm
2648;
2649; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
2650; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2651; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2652; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2653; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2654; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2655; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2656; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2657; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
2658; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2659; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2660; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2661; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2662; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2663;
2664; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
2665; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2666; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2667; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2668; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2669; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2670; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2671; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2672; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
2673; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2674; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2675; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2676; GFX90A-TGSPLIT-NEXT:    s_endpgm
2677    i32* %out, i32 %in, i32 %old) {
2678entry:
2679  %gep = getelementptr i32, i32* %out, i32 4
2680  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel monotonic
2681  %val0 = extractvalue { i32, i1 } %val, 0
2682  store i32 %val0, i32* %out, align 4
2683  ret void
2684}
2685
2686define amdgpu_kernel void @flat_agent_seq_cst_monotonic_ret_cmpxchg(
2687; GFX7-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
2688; GFX7:       ; %bb.0: ; %entry
2689; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2690; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2691; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2692; GFX7-NEXT:    s_add_u32 s4, s0, 16
2693; GFX7-NEXT:    s_addc_u32 s5, s1, 0
2694; GFX7-NEXT:    v_mov_b32_e32 v0, s4
2695; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2696; GFX7-NEXT:    v_mov_b32_e32 v1, s5
2697; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2698; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2699; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2700; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2701; GFX7-NEXT:    buffer_wbinvl1_vol
2702; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2703; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2704; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2705; GFX7-NEXT:    flat_store_dword v[0:1], v2
2706; GFX7-NEXT:    s_endpgm
2707;
2708; GFX10-WGP-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
2709; GFX10-WGP:       ; %bb.0: ; %entry
2710; GFX10-WGP-NEXT:    s_clause 0x1
2711; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2712; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2713; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2714; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
2715; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
2716; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
2717; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2718; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
2719; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2720; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2721; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2722; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2723; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2724; GFX10-WGP-NEXT:    buffer_gl0_inv
2725; GFX10-WGP-NEXT:    buffer_gl1_inv
2726; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2727; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2728; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2729; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
2730; GFX10-WGP-NEXT:    s_endpgm
2731;
2732; GFX10-CU-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
2733; GFX10-CU:       ; %bb.0: ; %entry
2734; GFX10-CU-NEXT:    s_clause 0x1
2735; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2736; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2737; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2738; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
2739; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
2740; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
2741; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2742; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
2743; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2744; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2745; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2746; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2747; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2748; GFX10-CU-NEXT:    buffer_gl0_inv
2749; GFX10-CU-NEXT:    buffer_gl1_inv
2750; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2751; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2752; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2753; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
2754; GFX10-CU-NEXT:    s_endpgm
2755;
2756; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
2757; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2758; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
2759; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2760; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2761; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
2762; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
2763; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2764; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2765; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2766; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2767; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2768; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2769; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2770; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2771; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2772; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
2773; SKIP-CACHE-INV-NEXT:    s_endpgm
2774;
2775; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
2776; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2777; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2778; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2779; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2780; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2781; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2782; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2783; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
2784; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2785; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2786; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2787; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2788; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2789;
2790; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
2791; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2792; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2793; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2794; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2795; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2796; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2797; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2798; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
2799; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2800; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2801; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2802; GFX90A-TGSPLIT-NEXT:    s_endpgm
2803    i32* %out, i32 %in, i32 %old) {
2804entry:
2805  %gep = getelementptr i32, i32* %out, i32 4
2806  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst monotonic
2807  %val0 = extractvalue { i32, i1 } %val, 0
2808  store i32 %val0, i32* %out, align 4
2809  ret void
2810}
2811
2812define amdgpu_kernel void @flat_agent_acquire_acquire_ret_cmpxchg(
2813; GFX7-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
2814; GFX7:       ; %bb.0: ; %entry
2815; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2816; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2817; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2818; GFX7-NEXT:    s_add_u32 s4, s0, 16
2819; GFX7-NEXT:    s_addc_u32 s5, s1, 0
2820; GFX7-NEXT:    v_mov_b32_e32 v0, s4
2821; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2822; GFX7-NEXT:    v_mov_b32_e32 v1, s5
2823; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2824; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2825; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2826; GFX7-NEXT:    buffer_wbinvl1_vol
2827; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2828; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2829; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2830; GFX7-NEXT:    flat_store_dword v[0:1], v2
2831; GFX7-NEXT:    s_endpgm
2832;
2833; GFX10-WGP-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
2834; GFX10-WGP:       ; %bb.0: ; %entry
2835; GFX10-WGP-NEXT:    s_clause 0x1
2836; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2837; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2838; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2839; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
2840; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
2841; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
2842; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2843; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
2844; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2845; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2846; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2847; GFX10-WGP-NEXT:    buffer_gl0_inv
2848; GFX10-WGP-NEXT:    buffer_gl1_inv
2849; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2850; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2851; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2852; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
2853; GFX10-WGP-NEXT:    s_endpgm
2854;
2855; GFX10-CU-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
2856; GFX10-CU:       ; %bb.0: ; %entry
2857; GFX10-CU-NEXT:    s_clause 0x1
2858; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2859; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2860; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2861; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
2862; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
2863; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
2864; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2865; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
2866; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2867; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2868; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2869; GFX10-CU-NEXT:    buffer_gl0_inv
2870; GFX10-CU-NEXT:    buffer_gl1_inv
2871; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2872; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2873; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2874; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
2875; GFX10-CU-NEXT:    s_endpgm
2876;
2877; SKIP-CACHE-INV-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
2878; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2879; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
2880; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2881; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2882; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
2883; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
2884; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2885; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2886; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2887; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2888; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2889; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2890; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2891; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2892; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
2893; SKIP-CACHE-INV-NEXT:    s_endpgm
2894;
2895; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
2896; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2897; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2898; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2899; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2900; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2901; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2902; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
2903; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2904; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2905; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2906; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2907; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2908;
2909; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
2910; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2911; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2912; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2913; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2914; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2915; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2916; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
2917; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2918; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2919; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2920; GFX90A-TGSPLIT-NEXT:    s_endpgm
2921    i32* %out, i32 %in, i32 %old) {
2922entry:
2923  %gep = getelementptr i32, i32* %out, i32 4
2924  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire acquire
2925  %val0 = extractvalue { i32, i1 } %val, 0
2926  store i32 %val0, i32* %out, align 4
2927  ret void
2928}
2929
2930define amdgpu_kernel void @flat_agent_release_acquire_ret_cmpxchg(
2931; GFX7-LABEL: flat_agent_release_acquire_ret_cmpxchg:
2932; GFX7:       ; %bb.0: ; %entry
2933; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2934; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2935; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2936; GFX7-NEXT:    s_add_u32 s4, s0, 16
2937; GFX7-NEXT:    s_addc_u32 s5, s1, 0
2938; GFX7-NEXT:    v_mov_b32_e32 v0, s4
2939; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2940; GFX7-NEXT:    v_mov_b32_e32 v1, s5
2941; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2942; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2943; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2944; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2945; GFX7-NEXT:    buffer_wbinvl1_vol
2946; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2947; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2948; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2949; GFX7-NEXT:    flat_store_dword v[0:1], v2
2950; GFX7-NEXT:    s_endpgm
2951;
2952; GFX10-WGP-LABEL: flat_agent_release_acquire_ret_cmpxchg:
2953; GFX10-WGP:       ; %bb.0: ; %entry
2954; GFX10-WGP-NEXT:    s_clause 0x1
2955; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2956; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2957; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2958; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
2959; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
2960; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
2961; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2962; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
2963; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2964; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2965; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2966; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2967; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2968; GFX10-WGP-NEXT:    buffer_gl0_inv
2969; GFX10-WGP-NEXT:    buffer_gl1_inv
2970; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2971; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2972; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2973; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
2974; GFX10-WGP-NEXT:    s_endpgm
2975;
2976; GFX10-CU-LABEL: flat_agent_release_acquire_ret_cmpxchg:
2977; GFX10-CU:       ; %bb.0: ; %entry
2978; GFX10-CU-NEXT:    s_clause 0x1
2979; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2980; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2981; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2982; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
2983; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
2984; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
2985; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2986; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
2987; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2988; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2989; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2990; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2991; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2992; GFX10-CU-NEXT:    buffer_gl0_inv
2993; GFX10-CU-NEXT:    buffer_gl1_inv
2994; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2995; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2996; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2997; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
2998; GFX10-CU-NEXT:    s_endpgm
2999;
3000; SKIP-CACHE-INV-LABEL: flat_agent_release_acquire_ret_cmpxchg:
3001; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3002; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
3003; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
3004; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3005; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3006; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3007; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3008; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3009; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3010; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3011; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3012; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3013; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3014; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3015; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3016; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3017; SKIP-CACHE-INV-NEXT:    s_endpgm
3018;
3019; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_acquire_ret_cmpxchg:
3020; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3021; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3022; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3023; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3024; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3025; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3026; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3027; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3028; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3029; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3030; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3031; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3032; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3033;
3034; GFX90A-TGSPLIT-LABEL: flat_agent_release_acquire_ret_cmpxchg:
3035; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3036; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3037; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3038; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3039; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3040; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3041; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3042; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3043; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3044; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3045; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3046; GFX90A-TGSPLIT-NEXT:    s_endpgm
3047    i32* %out, i32 %in, i32 %old) {
3048entry:
3049  %gep = getelementptr i32, i32* %out, i32 4
3050  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release acquire
3051  %val0 = extractvalue { i32, i1 } %val, 0
3052  store i32 %val0, i32* %out, align 4
3053  ret void
3054}
3055
3056define amdgpu_kernel void @flat_agent_acq_rel_acquire_ret_cmpxchg(
3057; GFX7-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
3058; GFX7:       ; %bb.0: ; %entry
3059; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3060; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3061; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3062; GFX7-NEXT:    s_add_u32 s4, s0, 16
3063; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3064; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3065; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3066; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3067; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3068; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3069; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3070; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3071; GFX7-NEXT:    buffer_wbinvl1_vol
3072; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3073; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3074; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3075; GFX7-NEXT:    flat_store_dword v[0:1], v2
3076; GFX7-NEXT:    s_endpgm
3077;
3078; GFX10-WGP-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
3079; GFX10-WGP:       ; %bb.0: ; %entry
3080; GFX10-WGP-NEXT:    s_clause 0x1
3081; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3082; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3083; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3084; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3085; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3086; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3087; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3088; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
3089; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3090; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3091; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3092; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3093; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3094; GFX10-WGP-NEXT:    buffer_gl0_inv
3095; GFX10-WGP-NEXT:    buffer_gl1_inv
3096; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3097; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3098; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3099; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3100; GFX10-WGP-NEXT:    s_endpgm
3101;
3102; GFX10-CU-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
3103; GFX10-CU:       ; %bb.0: ; %entry
3104; GFX10-CU-NEXT:    s_clause 0x1
3105; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3106; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3107; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3108; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3109; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3110; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3111; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3112; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3113; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3114; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3115; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3116; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3117; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3118; GFX10-CU-NEXT:    buffer_gl0_inv
3119; GFX10-CU-NEXT:    buffer_gl1_inv
3120; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3121; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3122; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3123; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3124; GFX10-CU-NEXT:    s_endpgm
3125;
3126; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
3127; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3128; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
3129; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
3130; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3131; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3132; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3133; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3134; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3135; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3136; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3137; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3138; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3139; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3140; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3141; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3142; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3143; SKIP-CACHE-INV-NEXT:    s_endpgm
3144;
3145; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
3146; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3147; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3148; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3149; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3150; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3151; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3152; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3153; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3154; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3155; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3156; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3157; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3158; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3159;
3160; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
3161; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3162; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3163; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3164; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3165; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3166; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3167; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3168; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3169; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3170; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3171; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3172; GFX90A-TGSPLIT-NEXT:    s_endpgm
3173    i32* %out, i32 %in, i32 %old) {
3174entry:
3175  %gep = getelementptr i32, i32* %out, i32 4
3176  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel acquire
3177  %val0 = extractvalue { i32, i1 } %val, 0
3178  store i32 %val0, i32* %out, align 4
3179  ret void
3180}
3181
3182define amdgpu_kernel void @flat_agent_seq_cst_acquire_ret_cmpxchg(
3183; GFX7-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
3184; GFX7:       ; %bb.0: ; %entry
3185; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3186; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3187; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3188; GFX7-NEXT:    s_add_u32 s4, s0, 16
3189; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3190; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3191; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3192; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3193; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3194; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3195; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3196; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3197; GFX7-NEXT:    buffer_wbinvl1_vol
3198; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3199; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3200; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3201; GFX7-NEXT:    flat_store_dword v[0:1], v2
3202; GFX7-NEXT:    s_endpgm
3203;
3204; GFX10-WGP-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
3205; GFX10-WGP:       ; %bb.0: ; %entry
3206; GFX10-WGP-NEXT:    s_clause 0x1
3207; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3208; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3209; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3210; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3211; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3212; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3213; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3214; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
3215; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3216; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3217; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3218; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3219; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3220; GFX10-WGP-NEXT:    buffer_gl0_inv
3221; GFX10-WGP-NEXT:    buffer_gl1_inv
3222; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3223; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3224; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3225; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3226; GFX10-WGP-NEXT:    s_endpgm
3227;
3228; GFX10-CU-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
3229; GFX10-CU:       ; %bb.0: ; %entry
3230; GFX10-CU-NEXT:    s_clause 0x1
3231; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3232; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3233; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3234; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3235; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3236; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3237; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3238; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3239; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3240; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3241; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3242; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3243; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3244; GFX10-CU-NEXT:    buffer_gl0_inv
3245; GFX10-CU-NEXT:    buffer_gl1_inv
3246; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3247; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3248; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3249; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3250; GFX10-CU-NEXT:    s_endpgm
3251;
3252; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
3253; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3254; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
3255; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
3256; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3257; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3258; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3259; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3260; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3261; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3262; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3263; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3264; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3265; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3266; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3267; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3268; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3269; SKIP-CACHE-INV-NEXT:    s_endpgm
3270;
3271; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
3272; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3273; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3274; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3275; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3276; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3277; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3278; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3279; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3280; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3281; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3282; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3283; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3284; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3285;
3286; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
3287; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3288; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3289; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3290; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3291; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3292; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3293; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3294; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3295; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3296; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3297; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3298; GFX90A-TGSPLIT-NEXT:    s_endpgm
3299    i32* %out, i32 %in, i32 %old) {
3300entry:
3301  %gep = getelementptr i32, i32* %out, i32 4
3302  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst acquire
3303  %val0 = extractvalue { i32, i1 } %val, 0
3304  store i32 %val0, i32* %out, align 4
3305  ret void
3306}
3307
3308define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_ret_cmpxchg(
3309; GFX7-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
3310; GFX7:       ; %bb.0: ; %entry
3311; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3312; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3313; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3314; GFX7-NEXT:    s_add_u32 s4, s0, 16
3315; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3316; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3317; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3318; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3319; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3320; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3321; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3322; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3323; GFX7-NEXT:    buffer_wbinvl1_vol
3324; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3325; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3326; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3327; GFX7-NEXT:    flat_store_dword v[0:1], v2
3328; GFX7-NEXT:    s_endpgm
3329;
3330; GFX10-WGP-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
3331; GFX10-WGP:       ; %bb.0: ; %entry
3332; GFX10-WGP-NEXT:    s_clause 0x1
3333; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3334; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3335; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3336; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3337; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3338; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3339; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3340; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
3341; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3342; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3343; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3344; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3345; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3346; GFX10-WGP-NEXT:    buffer_gl0_inv
3347; GFX10-WGP-NEXT:    buffer_gl1_inv
3348; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3349; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3350; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3351; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3352; GFX10-WGP-NEXT:    s_endpgm
3353;
3354; GFX10-CU-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
3355; GFX10-CU:       ; %bb.0: ; %entry
3356; GFX10-CU-NEXT:    s_clause 0x1
3357; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3358; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3359; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3360; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3361; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3362; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3363; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3364; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3365; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3366; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3367; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3368; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3369; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3370; GFX10-CU-NEXT:    buffer_gl0_inv
3371; GFX10-CU-NEXT:    buffer_gl1_inv
3372; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3373; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3374; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3375; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3376; GFX10-CU-NEXT:    s_endpgm
3377;
3378; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
3379; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3380; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
3381; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
3382; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3383; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3384; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3385; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3386; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3387; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3388; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3389; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3390; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3391; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3392; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3393; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3394; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3395; SKIP-CACHE-INV-NEXT:    s_endpgm
3396;
3397; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
3398; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3399; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3400; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3401; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3402; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3403; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3404; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3405; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3406; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3407; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3408; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3409; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3410; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3411;
3412; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
3413; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3414; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3415; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3416; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3417; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3418; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3419; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3420; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3421; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3422; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3423; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3424; GFX90A-TGSPLIT-NEXT:    s_endpgm
3425    i32* %out, i32 %in, i32 %old) {
3426entry:
3427  %gep = getelementptr i32, i32* %out, i32 4
3428  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
3429  %val0 = extractvalue { i32, i1 } %val, 0
3430  store i32 %val0, i32* %out, align 4
3431  ret void
3432}
3433
3434define amdgpu_kernel void @flat_agent_one_as_unordered_load(
3435; GFX7-LABEL: flat_agent_one_as_unordered_load:
3436; GFX7:       ; %bb.0: ; %entry
3437; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3438; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3439; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3440; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3441; GFX7-NEXT:    flat_load_dword v0, v[0:1]
3442; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3443; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3444; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3445; GFX7-NEXT:    flat_store_dword v[2:3], v0
3446; GFX7-NEXT:    s_endpgm
3447;
3448; GFX10-WGP-LABEL: flat_agent_one_as_unordered_load:
3449; GFX10-WGP:       ; %bb.0: ; %entry
3450; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3451; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3452; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3453; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3454; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
3455; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
3456; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
3457; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3458; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3459; GFX10-WGP-NEXT:    s_endpgm
3460;
3461; GFX10-CU-LABEL: flat_agent_one_as_unordered_load:
3462; GFX10-CU:       ; %bb.0: ; %entry
3463; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3464; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3465; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3466; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3467; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
3468; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
3469; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
3470; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3471; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3472; GFX10-CU-NEXT:    s_endpgm
3473;
3474; SKIP-CACHE-INV-LABEL: flat_agent_one_as_unordered_load:
3475; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3476; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
3477; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3478; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3479; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
3480; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
3481; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
3482; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
3483; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3484; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
3485; SKIP-CACHE-INV-NEXT:    s_endpgm
3486;
3487; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_unordered_load:
3488; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3489; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3490; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3491; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3492; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
3493; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v0, v[0:1]
3494; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3495; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
3496; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3497; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[2:3], v0
3498; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3499;
3500; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_unordered_load:
3501; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3502; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3503; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3504; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3505; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
3506; GFX90A-TGSPLIT-NEXT:    flat_load_dword v0, v[0:1]
3507; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3508; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
3509; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3510; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[2:3], v0
3511; GFX90A-TGSPLIT-NEXT:    s_endpgm
3512    i32* %in, i32* %out) {
3513entry:
3514  %val = load atomic i32, i32* %in syncscope("agent-one-as") unordered, align 4
3515  store i32 %val, i32* %out
3516  ret void
3517}
3518
3519define amdgpu_kernel void @flat_agent_one_as_monotonic_load(
3520; GFX7-LABEL: flat_agent_one_as_monotonic_load:
3521; GFX7:       ; %bb.0: ; %entry
3522; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3523; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3524; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3525; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3526; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
3527; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3528; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3529; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3530; GFX7-NEXT:    flat_store_dword v[2:3], v0
3531; GFX7-NEXT:    s_endpgm
3532;
3533; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_load:
3534; GFX10-WGP:       ; %bb.0: ; %entry
3535; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3536; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3537; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3538; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3539; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
3540; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
3541; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
3542; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3543; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3544; GFX10-WGP-NEXT:    s_endpgm
3545;
3546; GFX10-CU-LABEL: flat_agent_one_as_monotonic_load:
3547; GFX10-CU:       ; %bb.0: ; %entry
3548; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3549; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3550; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3551; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3552; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
3553; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
3554; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
3555; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3556; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3557; GFX10-CU-NEXT:    s_endpgm
3558;
3559; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_load:
3560; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3561; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
3562; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3563; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3564; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
3565; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1] glc
3566; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
3567; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
3568; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3569; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
3570; SKIP-CACHE-INV-NEXT:    s_endpgm
3571;
3572; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_load:
3573; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3574; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3575; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3576; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3577; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
3578; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v0, v[0:1] glc
3579; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3580; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
3581; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3582; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[2:3], v0
3583; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3584;
3585; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_load:
3586; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3587; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3588; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3589; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3590; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
3591; GFX90A-TGSPLIT-NEXT:    flat_load_dword v0, v[0:1] glc
3592; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3593; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
3594; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3595; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[2:3], v0
3596; GFX90A-TGSPLIT-NEXT:    s_endpgm
3597    i32* %in, i32* %out) {
3598entry:
3599  %val = load atomic i32, i32* %in syncscope("agent-one-as") monotonic, align 4
3600  store i32 %val, i32* %out
3601  ret void
3602}
3603
3604define amdgpu_kernel void @flat_agent_one_as_acquire_load(
3605; GFX7-LABEL: flat_agent_one_as_acquire_load:
3606; GFX7:       ; %bb.0: ; %entry
3607; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3608; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3609; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3610; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3611; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
3612; GFX7-NEXT:    s_waitcnt vmcnt(0)
3613; GFX7-NEXT:    buffer_wbinvl1_vol
3614; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3615; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3616; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3617; GFX7-NEXT:    flat_store_dword v[2:3], v0
3618; GFX7-NEXT:    s_endpgm
3619;
3620; GFX10-WGP-LABEL: flat_agent_one_as_acquire_load:
3621; GFX10-WGP:       ; %bb.0: ; %entry
3622; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3623; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3624; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3625; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3626; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
3627; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3628; GFX10-WGP-NEXT:    buffer_gl0_inv
3629; GFX10-WGP-NEXT:    buffer_gl1_inv
3630; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
3631; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
3632; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3633; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3634; GFX10-WGP-NEXT:    s_endpgm
3635;
3636; GFX10-CU-LABEL: flat_agent_one_as_acquire_load:
3637; GFX10-CU:       ; %bb.0: ; %entry
3638; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3639; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3640; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3641; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3642; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
3643; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3644; GFX10-CU-NEXT:    buffer_gl0_inv
3645; GFX10-CU-NEXT:    buffer_gl1_inv
3646; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
3647; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
3648; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3649; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3650; GFX10-CU-NEXT:    s_endpgm
3651;
3652; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_load:
3653; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3654; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
3655; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3656; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3657; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
3658; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1] glc
3659; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3660; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
3661; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
3662; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3663; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
3664; SKIP-CACHE-INV-NEXT:    s_endpgm
3665;
3666; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_load:
3667; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3668; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3669; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3670; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3671; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
3672; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v0, v[0:1] glc
3673; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3674; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3675; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3676; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
3677; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3678; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[2:3], v0
3679; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3680;
3681; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_load:
3682; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3683; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3684; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3685; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3686; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
3687; GFX90A-TGSPLIT-NEXT:    flat_load_dword v0, v[0:1] glc
3688; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3689; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3690; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3691; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
3692; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[2:3], v0
3693; GFX90A-TGSPLIT-NEXT:    s_endpgm
3694    i32* %in, i32* %out) {
3695entry:
3696  %val = load atomic i32, i32* %in syncscope("agent-one-as") acquire, align 4
3697  store i32 %val, i32* %out
3698  ret void
3699}
3700
3701define amdgpu_kernel void @flat_agent_one_as_seq_cst_load(
3702; GFX7-LABEL: flat_agent_one_as_seq_cst_load:
3703; GFX7:       ; %bb.0: ; %entry
3704; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3705; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3706; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3707; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3708; GFX7-NEXT:    s_waitcnt vmcnt(0)
3709; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
3710; GFX7-NEXT:    s_waitcnt vmcnt(0)
3711; GFX7-NEXT:    buffer_wbinvl1_vol
3712; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3713; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3714; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3715; GFX7-NEXT:    flat_store_dword v[2:3], v0
3716; GFX7-NEXT:    s_endpgm
3717;
3718; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_load:
3719; GFX10-WGP:       ; %bb.0: ; %entry
3720; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3721; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3722; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3723; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3724; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3725; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3726; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
3727; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3728; GFX10-WGP-NEXT:    buffer_gl0_inv
3729; GFX10-WGP-NEXT:    buffer_gl1_inv
3730; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
3731; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
3732; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3733; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3734; GFX10-WGP-NEXT:    s_endpgm
3735;
3736; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_load:
3737; GFX10-CU:       ; %bb.0: ; %entry
3738; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3739; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3740; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3741; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3742; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3743; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3744; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
3745; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3746; GFX10-CU-NEXT:    buffer_gl0_inv
3747; GFX10-CU-NEXT:    buffer_gl1_inv
3748; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
3749; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
3750; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3751; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3752; GFX10-CU-NEXT:    s_endpgm
3753;
3754; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_load:
3755; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3756; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
3757; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3758; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3759; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
3760; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3761; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1] glc
3762; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3763; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
3764; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
3765; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3766; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
3767; SKIP-CACHE-INV-NEXT:    s_endpgm
3768;
3769; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_load:
3770; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3771; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3772; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3773; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3774; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
3775; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3776; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v0, v[0:1] glc
3777; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3778; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3779; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3780; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
3781; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3782; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[2:3], v0
3783; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3784;
3785; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_load:
3786; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3787; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3788; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3789; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3790; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
3791; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3792; GFX90A-TGSPLIT-NEXT:    flat_load_dword v0, v[0:1] glc
3793; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3794; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3795; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3796; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
3797; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[2:3], v0
3798; GFX90A-TGSPLIT-NEXT:    s_endpgm
3799    i32* %in, i32* %out) {
3800entry:
3801  %val = load atomic i32, i32* %in syncscope("agent-one-as") seq_cst, align 4
3802  store i32 %val, i32* %out
3803  ret void
3804}
3805
3806define amdgpu_kernel void @flat_agent_one_as_unordered_store(
3807; GFX7-LABEL: flat_agent_one_as_unordered_store:
3808; GFX7:       ; %bb.0: ; %entry
3809; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
3810; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
3811; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3812; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3813; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3814; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3815; GFX7-NEXT:    flat_store_dword v[0:1], v2
3816; GFX7-NEXT:    s_endpgm
3817;
3818; GFX10-WGP-LABEL: flat_agent_one_as_unordered_store:
3819; GFX10-WGP:       ; %bb.0: ; %entry
3820; GFX10-WGP-NEXT:    s_clause 0x1
3821; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3822; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
3823; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3824; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3825; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3826; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3827; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3828; GFX10-WGP-NEXT:    s_endpgm
3829;
3830; GFX10-CU-LABEL: flat_agent_one_as_unordered_store:
3831; GFX10-CU:       ; %bb.0: ; %entry
3832; GFX10-CU-NEXT:    s_clause 0x1
3833; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3834; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
3835; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3836; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3837; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3838; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3839; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3840; GFX10-CU-NEXT:    s_endpgm
3841;
3842; SKIP-CACHE-INV-LABEL: flat_agent_one_as_unordered_store:
3843; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3844; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
3845; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
3846; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3847; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
3848; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3849; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
3850; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3851; SKIP-CACHE-INV-NEXT:    s_endpgm
3852;
3853; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_unordered_store:
3854; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3855; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
3856; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3857; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3858; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3859; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3860; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3861; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3862;
3863; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_unordered_store:
3864; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3865; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
3866; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3867; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3868; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3869; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3870; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3871; GFX90A-TGSPLIT-NEXT:    s_endpgm
3872    i32 %in, i32* %out) {
3873entry:
3874  store atomic i32 %in, i32* %out syncscope("agent-one-as") unordered, align 4
3875  ret void
3876}
3877
3878define amdgpu_kernel void @flat_agent_one_as_monotonic_store(
3879; GFX7-LABEL: flat_agent_one_as_monotonic_store:
3880; GFX7:       ; %bb.0: ; %entry
3881; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
3882; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
3883; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3884; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3885; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3886; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3887; GFX7-NEXT:    flat_store_dword v[0:1], v2
3888; GFX7-NEXT:    s_endpgm
3889;
3890; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_store:
3891; GFX10-WGP:       ; %bb.0: ; %entry
3892; GFX10-WGP-NEXT:    s_clause 0x1
3893; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3894; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
3895; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3896; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3897; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3898; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3899; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3900; GFX10-WGP-NEXT:    s_endpgm
3901;
3902; GFX10-CU-LABEL: flat_agent_one_as_monotonic_store:
3903; GFX10-CU:       ; %bb.0: ; %entry
3904; GFX10-CU-NEXT:    s_clause 0x1
3905; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3906; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
3907; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3908; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3909; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3910; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3911; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3912; GFX10-CU-NEXT:    s_endpgm
3913;
3914; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_store:
3915; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3916; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
3917; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
3918; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3919; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
3920; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3921; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
3922; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3923; SKIP-CACHE-INV-NEXT:    s_endpgm
3924;
3925; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_store:
3926; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3927; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
3928; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3929; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3930; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3931; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3932; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3933; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3934;
3935; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_store:
3936; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3937; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
3938; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3939; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3940; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3941; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3942; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3943; GFX90A-TGSPLIT-NEXT:    s_endpgm
3944    i32 %in, i32* %out) {
3945entry:
3946  store atomic i32 %in, i32* %out syncscope("agent-one-as") monotonic, align 4
3947  ret void
3948}
3949
3950define amdgpu_kernel void @flat_agent_one_as_release_store(
3951; GFX7-LABEL: flat_agent_one_as_release_store:
3952; GFX7:       ; %bb.0: ; %entry
3953; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
3954; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
3955; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3956; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3957; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3958; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3959; GFX7-NEXT:    s_waitcnt vmcnt(0)
3960; GFX7-NEXT:    flat_store_dword v[0:1], v2
3961; GFX7-NEXT:    s_endpgm
3962;
3963; GFX10-WGP-LABEL: flat_agent_one_as_release_store:
3964; GFX10-WGP:       ; %bb.0: ; %entry
3965; GFX10-WGP-NEXT:    s_clause 0x1
3966; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3967; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
3968; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3969; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3970; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3971; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3972; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3973; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3974; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3975; GFX10-WGP-NEXT:    s_endpgm
3976;
3977; GFX10-CU-LABEL: flat_agent_one_as_release_store:
3978; GFX10-CU:       ; %bb.0: ; %entry
3979; GFX10-CU-NEXT:    s_clause 0x1
3980; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3981; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
3982; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3983; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3984; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3985; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3986; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3987; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3988; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3989; GFX10-CU-NEXT:    s_endpgm
3990;
3991; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_store:
3992; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3993; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
3994; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
3995; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3996; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
3997; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3998; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
3999; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4000; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4001; SKIP-CACHE-INV-NEXT:    s_endpgm
4002;
4003; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_store:
4004; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4005; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
4006; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4007; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4008; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4009; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4010; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4011; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4012; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4013;
4014; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_store:
4015; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4016; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
4017; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4018; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4019; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4020; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4021; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4022; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4023; GFX90A-TGSPLIT-NEXT:    s_endpgm
4024    i32 %in, i32* %out) {
4025entry:
4026  store atomic i32 %in, i32* %out syncscope("agent-one-as") release, align 4
4027  ret void
4028}
4029
4030define amdgpu_kernel void @flat_agent_one_as_seq_cst_store(
4031; GFX7-LABEL: flat_agent_one_as_seq_cst_store:
4032; GFX7:       ; %bb.0: ; %entry
4033; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
4034; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
4035; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4036; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4037; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4038; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4039; GFX7-NEXT:    s_waitcnt vmcnt(0)
4040; GFX7-NEXT:    flat_store_dword v[0:1], v2
4041; GFX7-NEXT:    s_endpgm
4042;
4043; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_store:
4044; GFX10-WGP:       ; %bb.0: ; %entry
4045; GFX10-WGP-NEXT:    s_clause 0x1
4046; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4047; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
4048; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4049; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4050; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4051; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4052; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4053; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4054; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4055; GFX10-WGP-NEXT:    s_endpgm
4056;
4057; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_store:
4058; GFX10-CU:       ; %bb.0: ; %entry
4059; GFX10-CU-NEXT:    s_clause 0x1
4060; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4061; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
4062; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4063; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4064; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4065; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4066; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4067; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4068; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4069; GFX10-CU-NEXT:    s_endpgm
4070;
4071; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_store:
4072; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4073; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
4074; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
4075; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4076; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
4077; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4078; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
4079; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4080; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4081; SKIP-CACHE-INV-NEXT:    s_endpgm
4082;
4083; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_store:
4084; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4085; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
4086; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4087; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4088; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4089; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4090; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4091; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4092; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4093;
4094; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_store:
4095; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4096; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
4097; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4098; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4099; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4100; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4101; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4102; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4103; GFX90A-TGSPLIT-NEXT:    s_endpgm
4104    i32 %in, i32* %out) {
4105entry:
4106  store atomic i32 %in, i32* %out syncscope("agent-one-as") seq_cst, align 4
4107  ret void
4108}
4109
4110define amdgpu_kernel void @flat_agent_one_as_monotonic_atomicrmw(
4111; GFX7-LABEL: flat_agent_one_as_monotonic_atomicrmw:
4112; GFX7:       ; %bb.0: ; %entry
4113; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4114; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
4115; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4116; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4117; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4118; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4119; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
4120; GFX7-NEXT:    s_endpgm
4121;
4122; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_atomicrmw:
4123; GFX10-WGP:       ; %bb.0: ; %entry
4124; GFX10-WGP-NEXT:    s_clause 0x1
4125; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4126; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
4127; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4128; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4129; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4130; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4131; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
4132; GFX10-WGP-NEXT:    s_endpgm
4133;
4134; GFX10-CU-LABEL: flat_agent_one_as_monotonic_atomicrmw:
4135; GFX10-CU:       ; %bb.0: ; %entry
4136; GFX10-CU-NEXT:    s_clause 0x1
4137; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4138; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
4139; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4140; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4141; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4142; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4143; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
4144; GFX10-CU-NEXT:    s_endpgm
4145;
4146; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_atomicrmw:
4147; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4148; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
4149; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
4150; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4151; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4152; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4153; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4154; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
4155; SKIP-CACHE-INV-NEXT:    s_endpgm
4156;
4157; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_atomicrmw:
4158; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4159; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4160; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4161; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4162; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4163; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4164; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
4165; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4166;
4167; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_atomicrmw:
4168; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4169; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4170; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4171; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4172; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4173; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4174; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
4175; GFX90A-TGSPLIT-NEXT:    s_endpgm
4176    i32* %out, i32 %in) {
4177entry:
4178  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") monotonic
4179  ret void
4180}
4181
4182define amdgpu_kernel void @flat_agent_one_as_acquire_atomicrmw(
4183; GFX7-LABEL: flat_agent_one_as_acquire_atomicrmw:
4184; GFX7:       ; %bb.0: ; %entry
4185; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4186; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
4187; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4188; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4189; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4190; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4191; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
4192; GFX7-NEXT:    s_waitcnt vmcnt(0)
4193; GFX7-NEXT:    buffer_wbinvl1_vol
4194; GFX7-NEXT:    s_endpgm
4195;
4196; GFX10-WGP-LABEL: flat_agent_one_as_acquire_atomicrmw:
4197; GFX10-WGP:       ; %bb.0: ; %entry
4198; GFX10-WGP-NEXT:    s_clause 0x1
4199; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4200; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
4201; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4202; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4203; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4204; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4205; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
4206; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4207; GFX10-WGP-NEXT:    buffer_gl0_inv
4208; GFX10-WGP-NEXT:    buffer_gl1_inv
4209; GFX10-WGP-NEXT:    s_endpgm
4210;
4211; GFX10-CU-LABEL: flat_agent_one_as_acquire_atomicrmw:
4212; GFX10-CU:       ; %bb.0: ; %entry
4213; GFX10-CU-NEXT:    s_clause 0x1
4214; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4215; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
4216; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4217; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4218; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4219; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4220; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
4221; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4222; GFX10-CU-NEXT:    buffer_gl0_inv
4223; GFX10-CU-NEXT:    buffer_gl1_inv
4224; GFX10-CU-NEXT:    s_endpgm
4225;
4226; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_atomicrmw:
4227; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4228; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
4229; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
4230; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4231; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4232; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4233; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4234; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
4235; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4236; SKIP-CACHE-INV-NEXT:    s_endpgm
4237;
4238; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_atomicrmw:
4239; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4240; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4241; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4242; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4243; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4244; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4245; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
4246; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4247; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4248; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4249;
4250; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_atomicrmw:
4251; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4252; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4253; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4254; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4255; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4256; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4257; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
4258; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4259; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4260; GFX90A-TGSPLIT-NEXT:    s_endpgm
4261    i32* %out, i32 %in) {
4262entry:
4263  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") acquire
4264  ret void
4265}
4266
4267define amdgpu_kernel void @flat_agent_one_as_release_atomicrmw(
4268; GFX7-LABEL: flat_agent_one_as_release_atomicrmw:
4269; GFX7:       ; %bb.0: ; %entry
4270; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4271; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
4272; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4273; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4274; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4275; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4276; GFX7-NEXT:    s_waitcnt vmcnt(0)
4277; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
4278; GFX7-NEXT:    s_endpgm
4279;
4280; GFX10-WGP-LABEL: flat_agent_one_as_release_atomicrmw:
4281; GFX10-WGP:       ; %bb.0: ; %entry
4282; GFX10-WGP-NEXT:    s_clause 0x1
4283; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4284; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
4285; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4286; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4287; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4288; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4289; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4290; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4291; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
4292; GFX10-WGP-NEXT:    s_endpgm
4293;
4294; GFX10-CU-LABEL: flat_agent_one_as_release_atomicrmw:
4295; GFX10-CU:       ; %bb.0: ; %entry
4296; GFX10-CU-NEXT:    s_clause 0x1
4297; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4298; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
4299; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4300; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4301; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4302; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4303; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4304; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4305; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
4306; GFX10-CU-NEXT:    s_endpgm
4307;
4308; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_atomicrmw:
4309; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4310; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
4311; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
4312; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4313; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4314; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4315; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4316; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4317; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
4318; SKIP-CACHE-INV-NEXT:    s_endpgm
4319;
4320; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_atomicrmw:
4321; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4322; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4323; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4324; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4325; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4326; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4327; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4328; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
4329; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4330;
4331; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_atomicrmw:
4332; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4333; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4334; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4335; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4336; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4337; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4338; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4339; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
4340; GFX90A-TGSPLIT-NEXT:    s_endpgm
4341    i32* %out, i32 %in) {
4342entry:
4343  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") release
4344  ret void
4345}
4346
4347define amdgpu_kernel void @flat_agent_one_as_acq_rel_atomicrmw(
4348; GFX7-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
4349; GFX7:       ; %bb.0: ; %entry
4350; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4351; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
4352; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4353; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4354; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4355; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4356; GFX7-NEXT:    s_waitcnt vmcnt(0)
4357; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
4358; GFX7-NEXT:    s_waitcnt vmcnt(0)
4359; GFX7-NEXT:    buffer_wbinvl1_vol
4360; GFX7-NEXT:    s_endpgm
4361;
4362; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
4363; GFX10-WGP:       ; %bb.0: ; %entry
4364; GFX10-WGP-NEXT:    s_clause 0x1
4365; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4366; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
4367; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4368; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4369; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4370; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4371; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4372; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4373; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
4374; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4375; GFX10-WGP-NEXT:    buffer_gl0_inv
4376; GFX10-WGP-NEXT:    buffer_gl1_inv
4377; GFX10-WGP-NEXT:    s_endpgm
4378;
4379; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
4380; GFX10-CU:       ; %bb.0: ; %entry
4381; GFX10-CU-NEXT:    s_clause 0x1
4382; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4383; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
4384; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4385; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4386; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4387; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4388; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4389; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4390; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
4391; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4392; GFX10-CU-NEXT:    buffer_gl0_inv
4393; GFX10-CU-NEXT:    buffer_gl1_inv
4394; GFX10-CU-NEXT:    s_endpgm
4395;
4396; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
4397; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4398; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
4399; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
4400; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4401; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4402; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4403; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4404; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4405; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
4406; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4407; SKIP-CACHE-INV-NEXT:    s_endpgm
4408;
4409; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
4410; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4411; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4412; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4413; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4414; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4415; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4416; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4417; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
4418; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4419; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4420; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4421;
4422; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
4423; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4424; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4425; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4426; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4427; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4428; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4429; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4430; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
4431; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4432; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4433; GFX90A-TGSPLIT-NEXT:    s_endpgm
4434    i32* %out, i32 %in) {
4435entry:
4436  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") acq_rel
4437  ret void
4438}
4439
4440define amdgpu_kernel void @flat_agent_one_as_seq_cst_atomicrmw(
4441; GFX7-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
4442; GFX7:       ; %bb.0: ; %entry
4443; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4444; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
4445; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4446; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4447; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4448; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4449; GFX7-NEXT:    s_waitcnt vmcnt(0)
4450; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
4451; GFX7-NEXT:    s_waitcnt vmcnt(0)
4452; GFX7-NEXT:    buffer_wbinvl1_vol
4453; GFX7-NEXT:    s_endpgm
4454;
4455; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
4456; GFX10-WGP:       ; %bb.0: ; %entry
4457; GFX10-WGP-NEXT:    s_clause 0x1
4458; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4459; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
4460; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4461; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4462; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4463; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4464; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4465; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4466; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
4467; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4468; GFX10-WGP-NEXT:    buffer_gl0_inv
4469; GFX10-WGP-NEXT:    buffer_gl1_inv
4470; GFX10-WGP-NEXT:    s_endpgm
4471;
4472; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
4473; GFX10-CU:       ; %bb.0: ; %entry
4474; GFX10-CU-NEXT:    s_clause 0x1
4475; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4476; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
4477; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4478; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4479; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4480; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4481; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4482; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4483; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
4484; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4485; GFX10-CU-NEXT:    buffer_gl0_inv
4486; GFX10-CU-NEXT:    buffer_gl1_inv
4487; GFX10-CU-NEXT:    s_endpgm
4488;
4489; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
4490; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4491; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
4492; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
4493; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4494; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4495; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4496; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4497; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4498; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
4499; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4500; SKIP-CACHE-INV-NEXT:    s_endpgm
4501;
4502; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
4503; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4504; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4505; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4506; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4507; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4508; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4509; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4510; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
4511; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4512; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4513; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4514;
4515; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
4516; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4517; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4518; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4519; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4520; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4521; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4522; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4523; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
4524; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4525; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4526; GFX90A-TGSPLIT-NEXT:    s_endpgm
4527    i32* %out, i32 %in) {
4528entry:
4529  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") seq_cst
4530  ret void
4531}
4532
4533define amdgpu_kernel void @flat_agent_one_as_acquire_ret_atomicrmw(
4534; GFX7-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
4535; GFX7:       ; %bb.0: ; %entry
4536; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4537; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
4538; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4539; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4540; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4541; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4542; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4543; GFX7-NEXT:    s_waitcnt vmcnt(0)
4544; GFX7-NEXT:    buffer_wbinvl1_vol
4545; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4546; GFX7-NEXT:    flat_store_dword v[0:1], v2
4547; GFX7-NEXT:    s_endpgm
4548;
4549; GFX10-WGP-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
4550; GFX10-WGP:       ; %bb.0: ; %entry
4551; GFX10-WGP-NEXT:    s_clause 0x1
4552; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4553; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
4554; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4555; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4556; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4557; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4558; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4559; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4560; GFX10-WGP-NEXT:    buffer_gl0_inv
4561; GFX10-WGP-NEXT:    buffer_gl1_inv
4562; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4563; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4564; GFX10-WGP-NEXT:    s_endpgm
4565;
4566; GFX10-CU-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
4567; GFX10-CU:       ; %bb.0: ; %entry
4568; GFX10-CU-NEXT:    s_clause 0x1
4569; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4570; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
4571; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4572; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4573; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4574; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4575; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4576; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4577; GFX10-CU-NEXT:    buffer_gl0_inv
4578; GFX10-CU-NEXT:    buffer_gl1_inv
4579; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4580; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4581; GFX10-CU-NEXT:    s_endpgm
4582;
4583; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
4584; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4585; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
4586; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
4587; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4588; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4589; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4590; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4591; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4592; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4593; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4594; SKIP-CACHE-INV-NEXT:    s_endpgm
4595;
4596; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
4597; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4598; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4599; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4600; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4601; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4602; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4603; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4604; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4605; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4606; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4607; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4608; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4609;
4610; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
4611; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4612; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4613; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4614; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4615; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4616; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4617; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4618; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4619; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4620; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4621; GFX90A-TGSPLIT-NEXT:    s_endpgm
4622    i32* %out, i32 %in) {
4623entry:
4624  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") acquire
4625  store i32 %val, i32* %out, align 4
4626  ret void
4627}
4628
4629define amdgpu_kernel void @flat_agent_one_as_acq_rel_ret_atomicrmw(
4630; GFX7-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
4631; GFX7:       ; %bb.0: ; %entry
4632; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4633; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
4634; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4635; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4636; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4637; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4638; GFX7-NEXT:    s_waitcnt vmcnt(0)
4639; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4640; GFX7-NEXT:    s_waitcnt vmcnt(0)
4641; GFX7-NEXT:    buffer_wbinvl1_vol
4642; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4643; GFX7-NEXT:    flat_store_dword v[0:1], v2
4644; GFX7-NEXT:    s_endpgm
4645;
4646; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
4647; GFX10-WGP:       ; %bb.0: ; %entry
4648; GFX10-WGP-NEXT:    s_clause 0x1
4649; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4650; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
4651; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4652; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4653; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4654; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4655; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4656; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4657; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4658; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4659; GFX10-WGP-NEXT:    buffer_gl0_inv
4660; GFX10-WGP-NEXT:    buffer_gl1_inv
4661; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4662; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4663; GFX10-WGP-NEXT:    s_endpgm
4664;
4665; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
4666; GFX10-CU:       ; %bb.0: ; %entry
4667; GFX10-CU-NEXT:    s_clause 0x1
4668; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4669; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
4670; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4671; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4672; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4673; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4674; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4675; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4676; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4677; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4678; GFX10-CU-NEXT:    buffer_gl0_inv
4679; GFX10-CU-NEXT:    buffer_gl1_inv
4680; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4681; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4682; GFX10-CU-NEXT:    s_endpgm
4683;
4684; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
4685; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4686; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
4687; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
4688; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4689; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4690; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4691; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4692; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4693; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4694; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4695; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4696; SKIP-CACHE-INV-NEXT:    s_endpgm
4697;
4698; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
4699; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4700; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4701; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4702; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4703; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4704; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4705; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4706; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4707; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4708; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4709; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4710; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4711; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4712;
4713; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
4714; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4715; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4716; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4717; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4718; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4719; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4720; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4721; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4722; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4723; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4724; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4725; GFX90A-TGSPLIT-NEXT:    s_endpgm
4726    i32* %out, i32 %in) {
4727entry:
4728  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") acq_rel
4729  store i32 %val, i32* %out, align 4
4730  ret void
4731}
4732
4733define amdgpu_kernel void @flat_agent_one_as_seq_cst_ret_atomicrmw(
4734; GFX7-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
4735; GFX7:       ; %bb.0: ; %entry
4736; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4737; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
4738; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4739; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4740; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4741; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4742; GFX7-NEXT:    s_waitcnt vmcnt(0)
4743; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4744; GFX7-NEXT:    s_waitcnt vmcnt(0)
4745; GFX7-NEXT:    buffer_wbinvl1_vol
4746; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4747; GFX7-NEXT:    flat_store_dword v[0:1], v2
4748; GFX7-NEXT:    s_endpgm
4749;
4750; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
4751; GFX10-WGP:       ; %bb.0: ; %entry
4752; GFX10-WGP-NEXT:    s_clause 0x1
4753; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4754; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
4755; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4756; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4757; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4758; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4759; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4760; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4761; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4762; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4763; GFX10-WGP-NEXT:    buffer_gl0_inv
4764; GFX10-WGP-NEXT:    buffer_gl1_inv
4765; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4766; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4767; GFX10-WGP-NEXT:    s_endpgm
4768;
4769; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
4770; GFX10-CU:       ; %bb.0: ; %entry
4771; GFX10-CU-NEXT:    s_clause 0x1
4772; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4773; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
4774; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4775; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4776; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4777; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4778; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4779; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4780; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4781; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4782; GFX10-CU-NEXT:    buffer_gl0_inv
4783; GFX10-CU-NEXT:    buffer_gl1_inv
4784; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4785; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4786; GFX10-CU-NEXT:    s_endpgm
4787;
4788; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
4789; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4790; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
4791; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
4792; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4793; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4794; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4795; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4796; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4797; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4798; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4799; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4800; SKIP-CACHE-INV-NEXT:    s_endpgm
4801;
4802; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
4803; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4804; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4805; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4806; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4807; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4808; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4809; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4810; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4811; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4812; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4813; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4814; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4815; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4816;
4817; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
4818; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4819; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4820; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4821; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4822; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4823; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4824; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4825; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4826; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4827; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4828; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4829; GFX90A-TGSPLIT-NEXT:    s_endpgm
4830    i32* %out, i32 %in) {
4831entry:
4832  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") seq_cst
4833  store i32 %val, i32* %out, align 4
4834  ret void
4835}
4836
4837define amdgpu_kernel void @flat_agent_one_as_monotonic_monotonic_cmpxchg(
4838; GFX7-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
4839; GFX7:       ; %bb.0: ; %entry
4840; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4841; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4842; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4843; GFX7-NEXT:    s_add_u32 s0, s0, 16
4844; GFX7-NEXT:    s_addc_u32 s1, s1, 0
4845; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4846; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4847; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4848; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4849; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4850; GFX7-NEXT:    s_endpgm
4851;
4852; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
4853; GFX10-WGP:       ; %bb.0: ; %entry
4854; GFX10-WGP-NEXT:    s_clause 0x1
4855; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4856; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4857; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4858; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
4859; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
4860; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4861; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4862; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4863; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4864; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4865; GFX10-WGP-NEXT:    s_endpgm
4866;
4867; GFX10-CU-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
4868; GFX10-CU:       ; %bb.0: ; %entry
4869; GFX10-CU-NEXT:    s_clause 0x1
4870; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4871; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4872; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4873; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
4874; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
4875; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4876; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4877; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4878; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4879; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4880; GFX10-CU-NEXT:    s_endpgm
4881;
4882; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
4883; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4884; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
4885; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
4886; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4887; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
4888; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
4889; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4890; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4891; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4892; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4893; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4894; SKIP-CACHE-INV-NEXT:    s_endpgm
4895;
4896; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
4897; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4898; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4899; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4900; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4901; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4902; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4903; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4904; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4905;
4906; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
4907; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4908; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4909; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4910; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4911; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4912; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4913; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4914; GFX90A-TGSPLIT-NEXT:    s_endpgm
4915    i32* %out, i32 %in, i32 %old) {
4916entry:
4917  %gep = getelementptr i32, i32* %out, i32 4
4918  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") monotonic monotonic
4919  ret void
4920}
4921
4922define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_cmpxchg(
4923; GFX7-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
4924; GFX7:       ; %bb.0: ; %entry
4925; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4926; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4927; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4928; GFX7-NEXT:    s_add_u32 s0, s0, 16
4929; GFX7-NEXT:    s_addc_u32 s1, s1, 0
4930; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4931; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4932; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4933; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4934; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4935; GFX7-NEXT:    s_waitcnt vmcnt(0)
4936; GFX7-NEXT:    buffer_wbinvl1_vol
4937; GFX7-NEXT:    s_endpgm
4938;
4939; GFX10-WGP-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
4940; GFX10-WGP:       ; %bb.0: ; %entry
4941; GFX10-WGP-NEXT:    s_clause 0x1
4942; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4943; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4944; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4945; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
4946; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
4947; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4948; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4949; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4950; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4951; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4952; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4953; GFX10-WGP-NEXT:    buffer_gl0_inv
4954; GFX10-WGP-NEXT:    buffer_gl1_inv
4955; GFX10-WGP-NEXT:    s_endpgm
4956;
4957; GFX10-CU-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
4958; GFX10-CU:       ; %bb.0: ; %entry
4959; GFX10-CU-NEXT:    s_clause 0x1
4960; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4961; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4962; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4963; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
4964; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
4965; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4966; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4967; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4968; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4969; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4970; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4971; GFX10-CU-NEXT:    buffer_gl0_inv
4972; GFX10-CU-NEXT:    buffer_gl1_inv
4973; GFX10-CU-NEXT:    s_endpgm
4974;
4975; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
4976; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4977; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
4978; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
4979; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4980; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
4981; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
4982; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4983; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4984; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4985; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4986; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
4987; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4988; SKIP-CACHE-INV-NEXT:    s_endpgm
4989;
4990; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
4991; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4992; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4993; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4994; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4995; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4996; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4997; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4998; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4999; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5000; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5001;
5002; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
5003; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5004; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5005; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5006; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5007; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5008; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5009; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5010; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5011; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5012; GFX90A-TGSPLIT-NEXT:    s_endpgm
5013    i32* %out, i32 %in, i32 %old) {
5014entry:
5015  %gep = getelementptr i32, i32* %out, i32 4
5016  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire monotonic
5017  ret void
5018}
5019
5020define amdgpu_kernel void @flat_agent_one_as_release_monotonic_cmpxchg(
5021; GFX7-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
5022; GFX7:       ; %bb.0: ; %entry
5023; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5024; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5025; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5026; GFX7-NEXT:    s_add_u32 s0, s0, 16
5027; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5028; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5029; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5030; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5031; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5032; GFX7-NEXT:    s_waitcnt vmcnt(0)
5033; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5034; GFX7-NEXT:    s_endpgm
5035;
5036; GFX10-WGP-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
5037; GFX10-WGP:       ; %bb.0: ; %entry
5038; GFX10-WGP-NEXT:    s_clause 0x1
5039; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5040; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5041; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5042; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
5043; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
5044; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5045; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5046; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5047; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5048; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5049; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5050; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5051; GFX10-WGP-NEXT:    s_endpgm
5052;
5053; GFX10-CU-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
5054; GFX10-CU:       ; %bb.0: ; %entry
5055; GFX10-CU-NEXT:    s_clause 0x1
5056; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5057; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5058; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5059; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
5060; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
5061; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5062; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5063; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5064; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5065; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5066; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5067; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5068; GFX10-CU-NEXT:    s_endpgm
5069;
5070; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
5071; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5072; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
5073; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5074; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5075; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
5076; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
5077; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5078; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5079; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5080; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5081; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5082; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5083; SKIP-CACHE-INV-NEXT:    s_endpgm
5084;
5085; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
5086; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5087; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5088; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5089; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5090; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5091; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5092; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5093; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5094; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5095;
5096; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
5097; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5098; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5099; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5100; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5101; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5102; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5103; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5104; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5105; GFX90A-TGSPLIT-NEXT:    s_endpgm
5106    i32* %out, i32 %in, i32 %old) {
5107entry:
5108  %gep = getelementptr i32, i32* %out, i32 4
5109  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release monotonic
5110  ret void
5111}
5112
5113define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_cmpxchg(
5114; GFX7-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
5115; GFX7:       ; %bb.0: ; %entry
5116; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5117; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5118; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5119; GFX7-NEXT:    s_add_u32 s0, s0, 16
5120; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5121; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5122; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5123; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5124; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5125; GFX7-NEXT:    s_waitcnt vmcnt(0)
5126; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5127; GFX7-NEXT:    s_waitcnt vmcnt(0)
5128; GFX7-NEXT:    buffer_wbinvl1_vol
5129; GFX7-NEXT:    s_endpgm
5130;
5131; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
5132; GFX10-WGP:       ; %bb.0: ; %entry
5133; GFX10-WGP-NEXT:    s_clause 0x1
5134; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5135; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5136; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5137; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
5138; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
5139; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5140; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5141; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5142; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5143; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5144; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5145; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5146; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5147; GFX10-WGP-NEXT:    buffer_gl0_inv
5148; GFX10-WGP-NEXT:    buffer_gl1_inv
5149; GFX10-WGP-NEXT:    s_endpgm
5150;
5151; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
5152; GFX10-CU:       ; %bb.0: ; %entry
5153; GFX10-CU-NEXT:    s_clause 0x1
5154; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5155; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5156; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5157; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
5158; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
5159; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5160; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5161; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5162; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5163; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5164; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5165; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5166; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5167; GFX10-CU-NEXT:    buffer_gl0_inv
5168; GFX10-CU-NEXT:    buffer_gl1_inv
5169; GFX10-CU-NEXT:    s_endpgm
5170;
5171; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
5172; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5173; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
5174; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5175; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5176; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
5177; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
5178; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5179; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5180; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5181; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5182; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5183; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5184; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5185; SKIP-CACHE-INV-NEXT:    s_endpgm
5186;
5187; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
5188; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5189; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5190; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5191; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5192; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5193; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5194; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5195; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5196; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5197; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5198; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5199;
5200; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
5201; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5202; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5203; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5204; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5205; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5206; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5207; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5208; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5209; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5210; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5211; GFX90A-TGSPLIT-NEXT:    s_endpgm
5212    i32* %out, i32 %in, i32 %old) {
5213entry:
5214  %gep = getelementptr i32, i32* %out, i32 4
5215  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel monotonic
5216  ret void
5217}
5218
5219define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_cmpxchg(
5220; GFX7-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
5221; GFX7:       ; %bb.0: ; %entry
5222; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5223; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5224; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5225; GFX7-NEXT:    s_add_u32 s0, s0, 16
5226; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5227; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5228; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5229; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5230; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5231; GFX7-NEXT:    s_waitcnt vmcnt(0)
5232; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5233; GFX7-NEXT:    s_waitcnt vmcnt(0)
5234; GFX7-NEXT:    buffer_wbinvl1_vol
5235; GFX7-NEXT:    s_endpgm
5236;
5237; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
5238; GFX10-WGP:       ; %bb.0: ; %entry
5239; GFX10-WGP-NEXT:    s_clause 0x1
5240; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5241; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5242; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5243; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
5244; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
5245; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5246; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5247; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5248; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5249; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5250; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5251; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5252; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5253; GFX10-WGP-NEXT:    buffer_gl0_inv
5254; GFX10-WGP-NEXT:    buffer_gl1_inv
5255; GFX10-WGP-NEXT:    s_endpgm
5256;
5257; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
5258; GFX10-CU:       ; %bb.0: ; %entry
5259; GFX10-CU-NEXT:    s_clause 0x1
5260; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5261; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5262; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5263; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
5264; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
5265; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5266; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5267; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5268; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5269; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5270; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5271; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5272; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5273; GFX10-CU-NEXT:    buffer_gl0_inv
5274; GFX10-CU-NEXT:    buffer_gl1_inv
5275; GFX10-CU-NEXT:    s_endpgm
5276;
5277; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
5278; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5279; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
5280; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5281; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5282; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
5283; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
5284; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5285; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5286; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5287; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5288; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5289; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5290; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5291; SKIP-CACHE-INV-NEXT:    s_endpgm
5292;
5293; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
5294; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5295; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5296; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5297; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5298; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5299; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5300; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5301; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5302; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5303; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5304; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5305;
5306; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
5307; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5308; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5309; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5310; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5311; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5312; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5313; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5314; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5315; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5316; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5317; GFX90A-TGSPLIT-NEXT:    s_endpgm
5318    i32* %out, i32 %in, i32 %old) {
5319entry:
5320  %gep = getelementptr i32, i32* %out, i32 4
5321  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst monotonic
5322  ret void
5323}
5324
5325define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_cmpxchg(
5326; GFX7-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
5327; GFX7:       ; %bb.0: ; %entry
5328; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5329; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5330; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5331; GFX7-NEXT:    s_add_u32 s0, s0, 16
5332; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5333; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5334; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5335; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5336; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5337; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5338; GFX7-NEXT:    s_waitcnt vmcnt(0)
5339; GFX7-NEXT:    buffer_wbinvl1_vol
5340; GFX7-NEXT:    s_endpgm
5341;
5342; GFX10-WGP-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
5343; GFX10-WGP:       ; %bb.0: ; %entry
5344; GFX10-WGP-NEXT:    s_clause 0x1
5345; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5346; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5347; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5348; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
5349; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
5350; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5351; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5352; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5353; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5354; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5355; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5356; GFX10-WGP-NEXT:    buffer_gl0_inv
5357; GFX10-WGP-NEXT:    buffer_gl1_inv
5358; GFX10-WGP-NEXT:    s_endpgm
5359;
5360; GFX10-CU-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
5361; GFX10-CU:       ; %bb.0: ; %entry
5362; GFX10-CU-NEXT:    s_clause 0x1
5363; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5364; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5365; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5366; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
5367; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
5368; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5369; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5370; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5371; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5372; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5373; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5374; GFX10-CU-NEXT:    buffer_gl0_inv
5375; GFX10-CU-NEXT:    buffer_gl1_inv
5376; GFX10-CU-NEXT:    s_endpgm
5377;
5378; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
5379; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5380; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
5381; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5382; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5383; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
5384; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
5385; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5386; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5387; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5388; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5389; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5390; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5391; SKIP-CACHE-INV-NEXT:    s_endpgm
5392;
5393; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
5394; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5395; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5396; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5397; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5398; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5399; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5400; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5401; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5402; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5403; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5404;
5405; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
5406; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5407; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5408; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5409; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5410; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5411; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5412; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5413; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5414; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5415; GFX90A-TGSPLIT-NEXT:    s_endpgm
5416    i32* %out, i32 %in, i32 %old) {
5417entry:
5418  %gep = getelementptr i32, i32* %out, i32 4
5419  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire acquire
5420  ret void
5421}
5422
5423define amdgpu_kernel void @flat_agent_one_as_release_acquire_cmpxchg(
5424; GFX7-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
5425; GFX7:       ; %bb.0: ; %entry
5426; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5427; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5428; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5429; GFX7-NEXT:    s_add_u32 s0, s0, 16
5430; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5431; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5432; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5433; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5434; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5435; GFX7-NEXT:    s_waitcnt vmcnt(0)
5436; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5437; GFX7-NEXT:    s_waitcnt vmcnt(0)
5438; GFX7-NEXT:    buffer_wbinvl1_vol
5439; GFX7-NEXT:    s_endpgm
5440;
5441; GFX10-WGP-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
5442; GFX10-WGP:       ; %bb.0: ; %entry
5443; GFX10-WGP-NEXT:    s_clause 0x1
5444; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5445; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5446; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5447; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
5448; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
5449; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5450; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5451; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5452; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5453; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5454; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5455; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5456; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5457; GFX10-WGP-NEXT:    buffer_gl0_inv
5458; GFX10-WGP-NEXT:    buffer_gl1_inv
5459; GFX10-WGP-NEXT:    s_endpgm
5460;
5461; GFX10-CU-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
5462; GFX10-CU:       ; %bb.0: ; %entry
5463; GFX10-CU-NEXT:    s_clause 0x1
5464; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5465; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5466; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5467; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
5468; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
5469; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5470; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5471; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5472; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5473; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5474; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5475; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5476; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5477; GFX10-CU-NEXT:    buffer_gl0_inv
5478; GFX10-CU-NEXT:    buffer_gl1_inv
5479; GFX10-CU-NEXT:    s_endpgm
5480;
5481; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
5482; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5483; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
5484; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5485; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5486; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
5487; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
5488; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5489; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5490; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5491; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5492; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5493; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5494; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5495; SKIP-CACHE-INV-NEXT:    s_endpgm
5496;
5497; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
5498; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5499; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5500; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5501; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5502; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5503; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5504; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5505; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5506; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5507; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5508; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5509;
5510; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
5511; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5512; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5513; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5514; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5515; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5516; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5517; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5518; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5519; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5520; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5521; GFX90A-TGSPLIT-NEXT:    s_endpgm
5522    i32* %out, i32 %in, i32 %old) {
5523entry:
5524  %gep = getelementptr i32, i32* %out, i32 4
5525  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release acquire
5526  ret void
5527}
5528
5529define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_cmpxchg(
5530; GFX7-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
5531; GFX7:       ; %bb.0: ; %entry
5532; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5533; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5534; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5535; GFX7-NEXT:    s_add_u32 s0, s0, 16
5536; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5537; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5538; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5539; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5540; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5541; GFX7-NEXT:    s_waitcnt vmcnt(0)
5542; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5543; GFX7-NEXT:    s_waitcnt vmcnt(0)
5544; GFX7-NEXT:    buffer_wbinvl1_vol
5545; GFX7-NEXT:    s_endpgm
5546;
5547; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
5548; GFX10-WGP:       ; %bb.0: ; %entry
5549; GFX10-WGP-NEXT:    s_clause 0x1
5550; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5551; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5552; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5553; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
5554; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
5555; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5556; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5557; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5558; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5559; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5560; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5561; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5562; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5563; GFX10-WGP-NEXT:    buffer_gl0_inv
5564; GFX10-WGP-NEXT:    buffer_gl1_inv
5565; GFX10-WGP-NEXT:    s_endpgm
5566;
5567; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
5568; GFX10-CU:       ; %bb.0: ; %entry
5569; GFX10-CU-NEXT:    s_clause 0x1
5570; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5571; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5572; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5573; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
5574; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
5575; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5576; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5577; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5578; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5579; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5580; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5581; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5582; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5583; GFX10-CU-NEXT:    buffer_gl0_inv
5584; GFX10-CU-NEXT:    buffer_gl1_inv
5585; GFX10-CU-NEXT:    s_endpgm
5586;
5587; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
5588; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5589; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
5590; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5591; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5592; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
5593; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
5594; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5595; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5596; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5597; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5598; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5599; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5600; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5601; SKIP-CACHE-INV-NEXT:    s_endpgm
5602;
5603; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
5604; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5605; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5606; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5607; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5608; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5609; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5610; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5611; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5612; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5613; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5614; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5615;
5616; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
5617; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5618; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5619; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5620; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5621; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5622; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5623; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5624; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5625; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5626; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5627; GFX90A-TGSPLIT-NEXT:    s_endpgm
5628    i32* %out, i32 %in, i32 %old) {
5629entry:
5630  %gep = getelementptr i32, i32* %out, i32 4
5631  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel acquire
5632  ret void
5633}
5634
5635define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_cmpxchg(
5636; GFX7-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
5637; GFX7:       ; %bb.0: ; %entry
5638; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5639; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5640; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5641; GFX7-NEXT:    s_add_u32 s0, s0, 16
5642; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5643; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5644; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5645; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5646; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5647; GFX7-NEXT:    s_waitcnt vmcnt(0)
5648; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5649; GFX7-NEXT:    s_waitcnt vmcnt(0)
5650; GFX7-NEXT:    buffer_wbinvl1_vol
5651; GFX7-NEXT:    s_endpgm
5652;
5653; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
5654; GFX10-WGP:       ; %bb.0: ; %entry
5655; GFX10-WGP-NEXT:    s_clause 0x1
5656; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5657; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5658; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5659; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
5660; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
5661; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5662; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5663; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5664; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5665; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5666; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5667; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5668; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5669; GFX10-WGP-NEXT:    buffer_gl0_inv
5670; GFX10-WGP-NEXT:    buffer_gl1_inv
5671; GFX10-WGP-NEXT:    s_endpgm
5672;
5673; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
5674; GFX10-CU:       ; %bb.0: ; %entry
5675; GFX10-CU-NEXT:    s_clause 0x1
5676; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5677; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5678; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5679; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
5680; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
5681; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5682; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5683; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5684; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5685; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5686; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5687; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5688; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5689; GFX10-CU-NEXT:    buffer_gl0_inv
5690; GFX10-CU-NEXT:    buffer_gl1_inv
5691; GFX10-CU-NEXT:    s_endpgm
5692;
5693; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
5694; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5695; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
5696; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5697; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5698; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
5699; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
5700; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5701; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5702; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5703; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5704; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5705; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5706; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5707; SKIP-CACHE-INV-NEXT:    s_endpgm
5708;
5709; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
5710; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5711; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5712; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5713; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5714; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5715; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5716; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5717; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5718; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5719; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5720; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5721;
5722; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
5723; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5724; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5725; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5726; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5727; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5728; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5729; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5730; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5731; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5732; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5733; GFX90A-TGSPLIT-NEXT:    s_endpgm
5734    i32* %out, i32 %in, i32 %old) {
5735entry:
5736  %gep = getelementptr i32, i32* %out, i32 4
5737  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst acquire
5738  ret void
5739}
5740
5741define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_cmpxchg(
5742; GFX7-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
5743; GFX7:       ; %bb.0: ; %entry
5744; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5745; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5746; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5747; GFX7-NEXT:    s_add_u32 s0, s0, 16
5748; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5749; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5750; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5751; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5752; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5753; GFX7-NEXT:    s_waitcnt vmcnt(0)
5754; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5755; GFX7-NEXT:    s_waitcnt vmcnt(0)
5756; GFX7-NEXT:    buffer_wbinvl1_vol
5757; GFX7-NEXT:    s_endpgm
5758;
5759; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
5760; GFX10-WGP:       ; %bb.0: ; %entry
5761; GFX10-WGP-NEXT:    s_clause 0x1
5762; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5763; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5764; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5765; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
5766; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
5767; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5768; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5769; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5770; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5771; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5772; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5773; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5774; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5775; GFX10-WGP-NEXT:    buffer_gl0_inv
5776; GFX10-WGP-NEXT:    buffer_gl1_inv
5777; GFX10-WGP-NEXT:    s_endpgm
5778;
5779; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
5780; GFX10-CU:       ; %bb.0: ; %entry
5781; GFX10-CU-NEXT:    s_clause 0x1
5782; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5783; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5784; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5785; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
5786; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
5787; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5788; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5789; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5790; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5791; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5792; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5793; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5794; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5795; GFX10-CU-NEXT:    buffer_gl0_inv
5796; GFX10-CU-NEXT:    buffer_gl1_inv
5797; GFX10-CU-NEXT:    s_endpgm
5798;
5799; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
5800; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5801; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
5802; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5803; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5804; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
5805; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
5806; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5807; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5808; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5809; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5810; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5811; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5812; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5813; SKIP-CACHE-INV-NEXT:    s_endpgm
5814;
5815; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
5816; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5817; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5818; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5819; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5820; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5821; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5822; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5823; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5824; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5825; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5826; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5827;
5828; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
5829; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5830; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5831; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5832; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5833; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5834; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5835; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5836; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5837; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5838; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5839; GFX90A-TGSPLIT-NEXT:    s_endpgm
5840    i32* %out, i32 %in, i32 %old) {
5841entry:
5842  %gep = getelementptr i32, i32* %out, i32 4
5843  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst seq_cst
5844  ret void
5845}
5846
5847define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_ret_cmpxchg(
5848; GFX7-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
5849; GFX7:       ; %bb.0: ; %entry
5850; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5851; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5852; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5853; GFX7-NEXT:    s_add_u32 s4, s0, 16
5854; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5855; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5856; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5857; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5858; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5859; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5860; GFX7-NEXT:    s_waitcnt vmcnt(0)
5861; GFX7-NEXT:    buffer_wbinvl1_vol
5862; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5863; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5864; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5865; GFX7-NEXT:    flat_store_dword v[0:1], v2
5866; GFX7-NEXT:    s_endpgm
5867;
5868; GFX10-WGP-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
5869; GFX10-WGP:       ; %bb.0: ; %entry
5870; GFX10-WGP-NEXT:    s_clause 0x1
5871; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5872; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5873; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5874; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5875; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5876; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5877; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5878; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5879; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5880; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5881; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5882; GFX10-WGP-NEXT:    buffer_gl0_inv
5883; GFX10-WGP-NEXT:    buffer_gl1_inv
5884; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5885; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5886; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5887; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5888; GFX10-WGP-NEXT:    s_endpgm
5889;
5890; GFX10-CU-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
5891; GFX10-CU:       ; %bb.0: ; %entry
5892; GFX10-CU-NEXT:    s_clause 0x1
5893; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5894; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5895; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5896; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5897; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5898; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5899; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5900; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5901; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5902; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5903; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5904; GFX10-CU-NEXT:    buffer_gl0_inv
5905; GFX10-CU-NEXT:    buffer_gl1_inv
5906; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5907; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5908; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5909; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5910; GFX10-CU-NEXT:    s_endpgm
5911;
5912; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
5913; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5914; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
5915; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5916; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5917; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5918; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5919; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5920; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5921; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5922; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5923; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5924; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5925; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5926; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5927; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5928; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5929; SKIP-CACHE-INV-NEXT:    s_endpgm
5930;
5931; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
5932; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5933; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5934; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5935; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5936; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5937; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5938; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5939; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5940; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5941; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5942; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5943; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5944;
5945; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
5946; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5947; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5948; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5949; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5950; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5951; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5952; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5953; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5954; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5955; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5956; GFX90A-TGSPLIT-NEXT:    s_endpgm
5957    i32* %out, i32 %in, i32 %old) {
5958entry:
5959  %gep = getelementptr i32, i32* %out, i32 4
5960  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire monotonic
5961  %val0 = extractvalue { i32, i1 } %val, 0
5962  store i32 %val0, i32* %out, align 4
5963  ret void
5964}
5965
5966define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg(
5967; GFX7-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
5968; GFX7:       ; %bb.0: ; %entry
5969; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5970; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5971; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5972; GFX7-NEXT:    s_add_u32 s4, s0, 16
5973; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5974; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5975; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5976; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5977; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5978; GFX7-NEXT:    s_waitcnt vmcnt(0)
5979; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5980; GFX7-NEXT:    s_waitcnt vmcnt(0)
5981; GFX7-NEXT:    buffer_wbinvl1_vol
5982; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5983; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5984; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5985; GFX7-NEXT:    flat_store_dword v[0:1], v2
5986; GFX7-NEXT:    s_endpgm
5987;
5988; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
5989; GFX10-WGP:       ; %bb.0: ; %entry
5990; GFX10-WGP-NEXT:    s_clause 0x1
5991; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5992; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5993; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5994; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5995; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5996; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5997; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5998; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5999; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6000; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6001; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6002; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6003; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6004; GFX10-WGP-NEXT:    buffer_gl0_inv
6005; GFX10-WGP-NEXT:    buffer_gl1_inv
6006; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6007; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6008; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6009; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6010; GFX10-WGP-NEXT:    s_endpgm
6011;
6012; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
6013; GFX10-CU:       ; %bb.0: ; %entry
6014; GFX10-CU-NEXT:    s_clause 0x1
6015; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6016; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6017; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6018; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6019; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6020; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6021; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6022; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6023; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6024; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6025; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6026; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6027; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6028; GFX10-CU-NEXT:    buffer_gl0_inv
6029; GFX10-CU-NEXT:    buffer_gl1_inv
6030; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6031; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6032; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6033; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6034; GFX10-CU-NEXT:    s_endpgm
6035;
6036; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
6037; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6038; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
6039; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
6040; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6041; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6042; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6043; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6044; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6045; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6046; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6047; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6048; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6049; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6050; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6051; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6052; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6053; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6054; SKIP-CACHE-INV-NEXT:    s_endpgm
6055;
6056; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
6057; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6058; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6059; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6060; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6061; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6062; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6063; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6064; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6065; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6066; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6067; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6068; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6069; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6070;
6071; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
6072; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6073; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6074; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6075; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6076; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6077; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6078; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6079; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6080; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6081; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6082; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6083; GFX90A-TGSPLIT-NEXT:    s_endpgm
6084    i32* %out, i32 %in, i32 %old) {
6085entry:
6086  %gep = getelementptr i32, i32* %out, i32 4
6087  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel monotonic
6088  %val0 = extractvalue { i32, i1 } %val, 0
6089  store i32 %val0, i32* %out, align 4
6090  ret void
6091}
6092
6093define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg(
6094; GFX7-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
6095; GFX7:       ; %bb.0: ; %entry
6096; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6097; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6098; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6099; GFX7-NEXT:    s_add_u32 s4, s0, 16
6100; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6101; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6102; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6103; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6104; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6105; GFX7-NEXT:    s_waitcnt vmcnt(0)
6106; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6107; GFX7-NEXT:    s_waitcnt vmcnt(0)
6108; GFX7-NEXT:    buffer_wbinvl1_vol
6109; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6110; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6111; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6112; GFX7-NEXT:    flat_store_dword v[0:1], v2
6113; GFX7-NEXT:    s_endpgm
6114;
6115; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
6116; GFX10-WGP:       ; %bb.0: ; %entry
6117; GFX10-WGP-NEXT:    s_clause 0x1
6118; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6119; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6120; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6121; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6122; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6123; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6124; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6125; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6126; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6127; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6128; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6129; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6130; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6131; GFX10-WGP-NEXT:    buffer_gl0_inv
6132; GFX10-WGP-NEXT:    buffer_gl1_inv
6133; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6134; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6135; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6136; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6137; GFX10-WGP-NEXT:    s_endpgm
6138;
6139; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
6140; GFX10-CU:       ; %bb.0: ; %entry
6141; GFX10-CU-NEXT:    s_clause 0x1
6142; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6143; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6144; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6145; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6146; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6147; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6148; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6149; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6150; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6151; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6152; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6153; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6154; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6155; GFX10-CU-NEXT:    buffer_gl0_inv
6156; GFX10-CU-NEXT:    buffer_gl1_inv
6157; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6158; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6159; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6160; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6161; GFX10-CU-NEXT:    s_endpgm
6162;
6163; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
6164; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6165; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
6166; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
6167; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6168; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6169; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6170; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6171; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6172; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6173; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6174; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6175; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6176; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6177; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6178; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6179; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6180; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6181; SKIP-CACHE-INV-NEXT:    s_endpgm
6182;
6183; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
6184; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6185; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6186; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6187; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6188; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6189; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6190; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6191; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6192; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6193; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6194; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6195; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6196; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6197;
6198; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
6199; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6200; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6201; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6202; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6203; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6204; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6205; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6206; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6207; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6208; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6209; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6210; GFX90A-TGSPLIT-NEXT:    s_endpgm
6211    i32* %out, i32 %in, i32 %old) {
6212entry:
6213  %gep = getelementptr i32, i32* %out, i32 4
6214  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst monotonic
6215  %val0 = extractvalue { i32, i1 } %val, 0
6216  store i32 %val0, i32* %out, align 4
6217  ret void
6218}
6219
6220define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_ret_cmpxchg(
6221; GFX7-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
6222; GFX7:       ; %bb.0: ; %entry
6223; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6224; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6225; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6226; GFX7-NEXT:    s_add_u32 s4, s0, 16
6227; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6228; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6229; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6230; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6231; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6232; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6233; GFX7-NEXT:    s_waitcnt vmcnt(0)
6234; GFX7-NEXT:    buffer_wbinvl1_vol
6235; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6236; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6237; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6238; GFX7-NEXT:    flat_store_dword v[0:1], v2
6239; GFX7-NEXT:    s_endpgm
6240;
6241; GFX10-WGP-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
6242; GFX10-WGP:       ; %bb.0: ; %entry
6243; GFX10-WGP-NEXT:    s_clause 0x1
6244; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6245; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6246; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6247; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6248; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6249; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6250; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6251; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6252; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6253; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6254; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6255; GFX10-WGP-NEXT:    buffer_gl0_inv
6256; GFX10-WGP-NEXT:    buffer_gl1_inv
6257; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6258; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6259; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6260; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6261; GFX10-WGP-NEXT:    s_endpgm
6262;
6263; GFX10-CU-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
6264; GFX10-CU:       ; %bb.0: ; %entry
6265; GFX10-CU-NEXT:    s_clause 0x1
6266; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6267; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6268; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6269; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6270; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6271; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6272; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6273; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6274; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6275; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6276; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6277; GFX10-CU-NEXT:    buffer_gl0_inv
6278; GFX10-CU-NEXT:    buffer_gl1_inv
6279; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6280; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6281; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6282; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6283; GFX10-CU-NEXT:    s_endpgm
6284;
6285; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
6286; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6287; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
6288; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
6289; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6290; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6291; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6292; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6293; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6294; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6295; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6296; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6297; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6298; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6299; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6300; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6301; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6302; SKIP-CACHE-INV-NEXT:    s_endpgm
6303;
6304; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
6305; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6306; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6307; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6308; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6309; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6310; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6311; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6312; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6313; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6314; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6315; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6316; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6317;
6318; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
6319; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6320; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6321; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6322; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6323; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6324; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6325; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6326; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6327; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6328; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6329; GFX90A-TGSPLIT-NEXT:    s_endpgm
6330    i32* %out, i32 %in, i32 %old) {
6331entry:
6332  %gep = getelementptr i32, i32* %out, i32 4
6333  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire acquire
6334  %val0 = extractvalue { i32, i1 } %val, 0
6335  store i32 %val0, i32* %out, align 4
6336  ret void
6337}
6338
6339define amdgpu_kernel void @flat_agent_one_as_release_acquire_ret_cmpxchg(
6340; GFX7-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
6341; GFX7:       ; %bb.0: ; %entry
6342; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6343; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6344; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6345; GFX7-NEXT:    s_add_u32 s4, s0, 16
6346; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6347; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6348; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6349; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6350; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6351; GFX7-NEXT:    s_waitcnt vmcnt(0)
6352; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6353; GFX7-NEXT:    s_waitcnt vmcnt(0)
6354; GFX7-NEXT:    buffer_wbinvl1_vol
6355; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6356; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6357; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6358; GFX7-NEXT:    flat_store_dword v[0:1], v2
6359; GFX7-NEXT:    s_endpgm
6360;
6361; GFX10-WGP-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
6362; GFX10-WGP:       ; %bb.0: ; %entry
6363; GFX10-WGP-NEXT:    s_clause 0x1
6364; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6365; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6366; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6367; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6368; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6369; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6370; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6371; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6372; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6373; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6374; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6375; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6376; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6377; GFX10-WGP-NEXT:    buffer_gl0_inv
6378; GFX10-WGP-NEXT:    buffer_gl1_inv
6379; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6380; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6381; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6382; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6383; GFX10-WGP-NEXT:    s_endpgm
6384;
6385; GFX10-CU-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
6386; GFX10-CU:       ; %bb.0: ; %entry
6387; GFX10-CU-NEXT:    s_clause 0x1
6388; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6389; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6390; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6391; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6392; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6393; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6394; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6395; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6396; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6397; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6398; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6399; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6400; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6401; GFX10-CU-NEXT:    buffer_gl0_inv
6402; GFX10-CU-NEXT:    buffer_gl1_inv
6403; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6404; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6405; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6406; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6407; GFX10-CU-NEXT:    s_endpgm
6408;
6409; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
6410; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6411; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
6412; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
6413; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6414; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6415; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6416; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6417; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6418; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6419; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6420; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6421; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6422; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6423; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6424; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6425; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6426; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6427; SKIP-CACHE-INV-NEXT:    s_endpgm
6428;
6429; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
6430; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6431; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6432; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6433; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6434; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6435; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6436; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6437; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6438; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6439; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6440; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6441; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6442; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6443;
6444; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
6445; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6446; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6447; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6448; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6449; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6450; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6451; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6452; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6453; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6454; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6455; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6456; GFX90A-TGSPLIT-NEXT:    s_endpgm
6457    i32* %out, i32 %in, i32 %old) {
6458entry:
6459  %gep = getelementptr i32, i32* %out, i32 4
6460  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release acquire
6461  %val0 = extractvalue { i32, i1 } %val, 0
6462  store i32 %val0, i32* %out, align 4
6463  ret void
6464}
6465
6466define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_ret_cmpxchg(
6467; GFX7-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
6468; GFX7:       ; %bb.0: ; %entry
6469; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6470; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6471; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6472; GFX7-NEXT:    s_add_u32 s4, s0, 16
6473; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6474; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6475; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6476; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6477; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6478; GFX7-NEXT:    s_waitcnt vmcnt(0)
6479; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6480; GFX7-NEXT:    s_waitcnt vmcnt(0)
6481; GFX7-NEXT:    buffer_wbinvl1_vol
6482; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6483; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6484; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6485; GFX7-NEXT:    flat_store_dword v[0:1], v2
6486; GFX7-NEXT:    s_endpgm
6487;
6488; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
6489; GFX10-WGP:       ; %bb.0: ; %entry
6490; GFX10-WGP-NEXT:    s_clause 0x1
6491; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6492; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6493; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6494; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6495; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6496; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6497; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6498; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6499; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6500; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6501; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6502; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6503; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6504; GFX10-WGP-NEXT:    buffer_gl0_inv
6505; GFX10-WGP-NEXT:    buffer_gl1_inv
6506; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6507; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6508; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6509; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6510; GFX10-WGP-NEXT:    s_endpgm
6511;
6512; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
6513; GFX10-CU:       ; %bb.0: ; %entry
6514; GFX10-CU-NEXT:    s_clause 0x1
6515; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6516; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6517; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6518; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6519; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6520; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6521; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6522; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6523; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6524; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6525; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6526; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6527; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6528; GFX10-CU-NEXT:    buffer_gl0_inv
6529; GFX10-CU-NEXT:    buffer_gl1_inv
6530; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6531; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6532; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6533; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6534; GFX10-CU-NEXT:    s_endpgm
6535;
6536; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
6537; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6538; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
6539; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
6540; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6541; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6542; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6543; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6544; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6545; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6546; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6547; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6548; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6549; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6550; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6551; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6552; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6553; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6554; SKIP-CACHE-INV-NEXT:    s_endpgm
6555;
6556; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
6557; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6558; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6559; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6560; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6561; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6562; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6563; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6564; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6565; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6566; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6567; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6568; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6569; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6570;
6571; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
6572; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6573; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6574; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6575; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6576; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6577; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6578; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6579; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6580; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6581; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6582; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6583; GFX90A-TGSPLIT-NEXT:    s_endpgm
6584    i32* %out, i32 %in, i32 %old) {
6585entry:
6586  %gep = getelementptr i32, i32* %out, i32 4
6587  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel acquire
6588  %val0 = extractvalue { i32, i1 } %val, 0
6589  store i32 %val0, i32* %out, align 4
6590  ret void
6591}
6592
6593define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_ret_cmpxchg(
6594; GFX7-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
6595; GFX7:       ; %bb.0: ; %entry
6596; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6597; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6598; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6599; GFX7-NEXT:    s_add_u32 s4, s0, 16
6600; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6601; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6602; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6603; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6604; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6605; GFX7-NEXT:    s_waitcnt vmcnt(0)
6606; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6607; GFX7-NEXT:    s_waitcnt vmcnt(0)
6608; GFX7-NEXT:    buffer_wbinvl1_vol
6609; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6610; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6611; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6612; GFX7-NEXT:    flat_store_dword v[0:1], v2
6613; GFX7-NEXT:    s_endpgm
6614;
6615; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
6616; GFX10-WGP:       ; %bb.0: ; %entry
6617; GFX10-WGP-NEXT:    s_clause 0x1
6618; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6619; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6620; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6621; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6622; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6623; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6624; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6625; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6626; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6627; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6628; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6629; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6630; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6631; GFX10-WGP-NEXT:    buffer_gl0_inv
6632; GFX10-WGP-NEXT:    buffer_gl1_inv
6633; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6634; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6635; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6636; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6637; GFX10-WGP-NEXT:    s_endpgm
6638;
6639; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
6640; GFX10-CU:       ; %bb.0: ; %entry
6641; GFX10-CU-NEXT:    s_clause 0x1
6642; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6643; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6644; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6645; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6646; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6647; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6648; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6649; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6650; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6651; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6652; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6653; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6654; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6655; GFX10-CU-NEXT:    buffer_gl0_inv
6656; GFX10-CU-NEXT:    buffer_gl1_inv
6657; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6658; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6659; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6660; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6661; GFX10-CU-NEXT:    s_endpgm
6662;
6663; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
6664; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6665; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
6666; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
6667; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6668; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6669; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6670; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6671; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6672; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6673; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6674; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6675; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6676; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6677; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6678; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6679; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6680; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6681; SKIP-CACHE-INV-NEXT:    s_endpgm
6682;
6683; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
6684; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6685; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6686; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6687; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6688; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6689; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6690; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6691; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6692; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6693; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6694; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6695; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6696; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6697;
6698; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
6699; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6700; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6701; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6702; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6703; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6704; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6705; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6706; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6707; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6708; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6709; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6710; GFX90A-TGSPLIT-NEXT:    s_endpgm
6711    i32* %out, i32 %in, i32 %old) {
6712entry:
6713  %gep = getelementptr i32, i32* %out, i32 4
6714  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst acquire
6715  %val0 = extractvalue { i32, i1 } %val, 0
6716  store i32 %val0, i32* %out, align 4
6717  ret void
6718}
6719
6720define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg(
6721; GFX7-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
6722; GFX7:       ; %bb.0: ; %entry
6723; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6724; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6725; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6726; GFX7-NEXT:    s_add_u32 s4, s0, 16
6727; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6728; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6729; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6730; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6731; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6732; GFX7-NEXT:    s_waitcnt vmcnt(0)
6733; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6734; GFX7-NEXT:    s_waitcnt vmcnt(0)
6735; GFX7-NEXT:    buffer_wbinvl1_vol
6736; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6737; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6738; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6739; GFX7-NEXT:    flat_store_dword v[0:1], v2
6740; GFX7-NEXT:    s_endpgm
6741;
6742; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
6743; GFX10-WGP:       ; %bb.0: ; %entry
6744; GFX10-WGP-NEXT:    s_clause 0x1
6745; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6746; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6747; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6748; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6749; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6750; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6751; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6752; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6753; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6754; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6755; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6756; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6757; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6758; GFX10-WGP-NEXT:    buffer_gl0_inv
6759; GFX10-WGP-NEXT:    buffer_gl1_inv
6760; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6761; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6762; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6763; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6764; GFX10-WGP-NEXT:    s_endpgm
6765;
6766; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
6767; GFX10-CU:       ; %bb.0: ; %entry
6768; GFX10-CU-NEXT:    s_clause 0x1
6769; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6770; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6771; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6772; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6773; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6774; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6775; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6776; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6777; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6778; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6779; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6780; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6781; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6782; GFX10-CU-NEXT:    buffer_gl0_inv
6783; GFX10-CU-NEXT:    buffer_gl1_inv
6784; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6785; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6786; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6787; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6788; GFX10-CU-NEXT:    s_endpgm
6789;
6790; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
6791; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6792; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
6793; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
6794; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6795; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6796; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6797; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6798; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6799; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6800; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6801; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6802; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6803; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6804; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6805; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6806; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6807; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6808; SKIP-CACHE-INV-NEXT:    s_endpgm
6809;
6810; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
6811; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6812; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6813; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6814; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6815; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6816; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6817; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6818; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6819; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6820; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6821; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6822; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6823; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6824;
6825; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
6826; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6827; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6828; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6829; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6830; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6831; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6832; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6833; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6834; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6835; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6836; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6837; GFX90A-TGSPLIT-NEXT:    s_endpgm
6838    i32* %out, i32 %in, i32 %old) {
6839entry:
6840  %gep = getelementptr i32, i32* %out, i32 4
6841  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst seq_cst
6842  %val0 = extractvalue { i32, i1 } %val, 0
6843  store i32 %val0, i32* %out, align 4
6844  ret void
6845}
6846
6847