1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
3; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
4; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
5; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
6; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-NOTTGSPLIT %s
7; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-TGSPLIT %s
8; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-NOTTGSPLIT %s
9; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
10
11define amdgpu_kernel void @flat_agent_unordered_load(
12; GFX7-LABEL: flat_agent_unordered_load:
13; GFX7:       ; %bb.0: ; %entry
14; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
15; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
16; GFX7-NEXT:    v_mov_b32_e32 v0, s0
17; GFX7-NEXT:    v_mov_b32_e32 v1, s1
18; GFX7-NEXT:    flat_load_dword v2, v[0:1]
19; GFX7-NEXT:    v_mov_b32_e32 v0, s2
20; GFX7-NEXT:    v_mov_b32_e32 v1, s3
21; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
22; GFX7-NEXT:    flat_store_dword v[0:1], v2
23; GFX7-NEXT:    s_endpgm
24;
25; GFX10-WGP-LABEL: flat_agent_unordered_load:
26; GFX10-WGP:       ; %bb.0: ; %entry
27; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
28; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
29; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
30; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
31; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
32; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
33; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
34; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
35; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
36; GFX10-WGP-NEXT:    s_endpgm
37;
38; GFX10-CU-LABEL: flat_agent_unordered_load:
39; GFX10-CU:       ; %bb.0: ; %entry
40; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
41; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
42; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
43; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
44; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
45; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
46; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
47; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
48; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
49; GFX10-CU-NEXT:    s_endpgm
50;
51; SKIP-CACHE-INV-LABEL: flat_agent_unordered_load:
52; SKIP-CACHE-INV:       ; %bb.0: ; %entry
53; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
54; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
55; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
56; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
57; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
58; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
59; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
60; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
61; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
62; SKIP-CACHE-INV-NEXT:    s_endpgm
63;
64; GFX90A-NOTTGSPLIT-LABEL: flat_agent_unordered_load:
65; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
66; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
67; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
68; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
69; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
70; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
71; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
72; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
73; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
74; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
75; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
76;
77; GFX90A-TGSPLIT-LABEL: flat_agent_unordered_load:
78; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
79; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
80; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
81; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
82; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
83; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
84; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
85; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
86; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
87; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
88; GFX90A-TGSPLIT-NEXT:    s_endpgm
89;
90; GFX940-NOTTGSPLIT-LABEL: flat_agent_unordered_load:
91; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
92; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
93; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
94; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
95; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
96; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
97; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
98; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
99; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
100; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
101; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
102;
103; GFX940-TGSPLIT-LABEL: flat_agent_unordered_load:
104; GFX940-TGSPLIT:       ; %bb.0: ; %entry
105; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
106; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
107; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
108; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
109; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
110; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
111; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
112; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
113; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
114; GFX940-TGSPLIT-NEXT:    s_endpgm
115    i32* %in, i32* %out) {
116entry:
117  %val = load atomic i32, i32* %in syncscope("agent") unordered, align 4
118  store i32 %val, i32* %out
119  ret void
120}
121
122define amdgpu_kernel void @flat_agent_monotonic_load(
123; GFX7-LABEL: flat_agent_monotonic_load:
124; GFX7:       ; %bb.0: ; %entry
125; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
126; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
127; GFX7-NEXT:    v_mov_b32_e32 v0, s0
128; GFX7-NEXT:    v_mov_b32_e32 v1, s1
129; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
130; GFX7-NEXT:    v_mov_b32_e32 v0, s2
131; GFX7-NEXT:    v_mov_b32_e32 v1, s3
132; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
133; GFX7-NEXT:    flat_store_dword v[0:1], v2
134; GFX7-NEXT:    s_endpgm
135;
136; GFX10-WGP-LABEL: flat_agent_monotonic_load:
137; GFX10-WGP:       ; %bb.0: ; %entry
138; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
139; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
140; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
141; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
142; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
143; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
144; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
145; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
146; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
147; GFX10-WGP-NEXT:    s_endpgm
148;
149; GFX10-CU-LABEL: flat_agent_monotonic_load:
150; GFX10-CU:       ; %bb.0: ; %entry
151; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
152; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
153; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
154; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
155; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
156; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
157; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
158; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
159; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
160; GFX10-CU-NEXT:    s_endpgm
161;
162; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_load:
163; SKIP-CACHE-INV:       ; %bb.0: ; %entry
164; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
165; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
166; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
167; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
168; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
169; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
170; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
171; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
172; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
173; SKIP-CACHE-INV-NEXT:    s_endpgm
174;
175; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_load:
176; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
177; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
178; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
179; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
180; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
181; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
182; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
183; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
184; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
185; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
186; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
187;
188; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_load:
189; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
190; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
191; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
192; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
193; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
194; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
195; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
196; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
197; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
198; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
199; GFX90A-TGSPLIT-NEXT:    s_endpgm
200;
201; GFX940-NOTTGSPLIT-LABEL: flat_agent_monotonic_load:
202; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
203; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
204; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
205; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
206; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
207; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc1
208; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
209; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
210; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
211; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
212; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
213;
214; GFX940-TGSPLIT-LABEL: flat_agent_monotonic_load:
215; GFX940-TGSPLIT:       ; %bb.0: ; %entry
216; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
217; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
218; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
219; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
220; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc1
221; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
222; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
223; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
224; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
225; GFX940-TGSPLIT-NEXT:    s_endpgm
226    i32* %in, i32* %out) {
227entry:
228  %val = load atomic i32, i32* %in syncscope("agent") monotonic, align 4
229  store i32 %val, i32* %out
230  ret void
231}
232
233define amdgpu_kernel void @flat_agent_acquire_load(
234; GFX7-LABEL: flat_agent_acquire_load:
235; GFX7:       ; %bb.0: ; %entry
236; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
237; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
238; GFX7-NEXT:    v_mov_b32_e32 v0, s0
239; GFX7-NEXT:    v_mov_b32_e32 v1, s1
240; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
241; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
242; GFX7-NEXT:    buffer_wbinvl1_vol
243; GFX7-NEXT:    v_mov_b32_e32 v0, s2
244; GFX7-NEXT:    v_mov_b32_e32 v1, s3
245; GFX7-NEXT:    flat_store_dword v[0:1], v2
246; GFX7-NEXT:    s_endpgm
247;
248; GFX10-WGP-LABEL: flat_agent_acquire_load:
249; GFX10-WGP:       ; %bb.0: ; %entry
250; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
251; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
252; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
253; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
254; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
255; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
256; GFX10-WGP-NEXT:    buffer_gl0_inv
257; GFX10-WGP-NEXT:    buffer_gl1_inv
258; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
259; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
260; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
261; GFX10-WGP-NEXT:    s_endpgm
262;
263; GFX10-CU-LABEL: flat_agent_acquire_load:
264; GFX10-CU:       ; %bb.0: ; %entry
265; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
266; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
267; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
268; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
269; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
270; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
271; GFX10-CU-NEXT:    buffer_gl0_inv
272; GFX10-CU-NEXT:    buffer_gl1_inv
273; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
274; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
275; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
276; GFX10-CU-NEXT:    s_endpgm
277;
278; SKIP-CACHE-INV-LABEL: flat_agent_acquire_load:
279; SKIP-CACHE-INV:       ; %bb.0: ; %entry
280; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
281; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
282; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
283; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
284; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
285; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
286; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
287; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
288; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
289; SKIP-CACHE-INV-NEXT:    s_endpgm
290;
291; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_load:
292; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
293; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
294; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
295; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
296; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
297; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
298; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
299; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
300; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
301; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
302; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
303; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
304;
305; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_load:
306; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
307; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
308; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
309; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
310; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
311; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
312; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
313; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
314; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
315; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
316; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
317; GFX90A-TGSPLIT-NEXT:    s_endpgm
318;
319; GFX940-NOTTGSPLIT-LABEL: flat_agent_acquire_load:
320; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
321; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
322; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
323; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
324; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
325; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc1
326; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
327; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
328; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
329; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
330; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
331; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
332;
333; GFX940-TGSPLIT-LABEL: flat_agent_acquire_load:
334; GFX940-TGSPLIT:       ; %bb.0: ; %entry
335; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
336; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
337; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
338; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
339; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc1
340; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
341; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
342; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
343; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
344; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
345; GFX940-TGSPLIT-NEXT:    s_endpgm
346    i32* %in, i32* %out) {
347entry:
348  %val = load atomic i32, i32* %in syncscope("agent") acquire, align 4
349  store i32 %val, i32* %out
350  ret void
351}
352
353define amdgpu_kernel void @flat_agent_seq_cst_load(
354; GFX7-LABEL: flat_agent_seq_cst_load:
355; GFX7:       ; %bb.0: ; %entry
356; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
357; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
358; GFX7-NEXT:    v_mov_b32_e32 v0, s0
359; GFX7-NEXT:    v_mov_b32_e32 v1, s1
360; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
361; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
362; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
363; GFX7-NEXT:    buffer_wbinvl1_vol
364; GFX7-NEXT:    v_mov_b32_e32 v0, s2
365; GFX7-NEXT:    v_mov_b32_e32 v1, s3
366; GFX7-NEXT:    flat_store_dword v[0:1], v2
367; GFX7-NEXT:    s_endpgm
368;
369; GFX10-WGP-LABEL: flat_agent_seq_cst_load:
370; GFX10-WGP:       ; %bb.0: ; %entry
371; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
372; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
373; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
374; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
375; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
376; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
377; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
378; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
379; GFX10-WGP-NEXT:    buffer_gl0_inv
380; GFX10-WGP-NEXT:    buffer_gl1_inv
381; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
382; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
383; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
384; GFX10-WGP-NEXT:    s_endpgm
385;
386; GFX10-CU-LABEL: flat_agent_seq_cst_load:
387; GFX10-CU:       ; %bb.0: ; %entry
388; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
389; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
390; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
391; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
392; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
393; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
394; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
395; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
396; GFX10-CU-NEXT:    buffer_gl0_inv
397; GFX10-CU-NEXT:    buffer_gl1_inv
398; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
399; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
400; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
401; GFX10-CU-NEXT:    s_endpgm
402;
403; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_load:
404; SKIP-CACHE-INV:       ; %bb.0: ; %entry
405; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
406; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
407; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
408; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
409; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
410; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
411; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
412; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
413; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
414; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
415; SKIP-CACHE-INV-NEXT:    s_endpgm
416;
417; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_load:
418; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
419; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
420; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
421; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
422; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
423; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
424; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
425; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
426; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
427; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
428; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
429; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
430; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
431;
432; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_load:
433; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
434; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
435; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
436; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
437; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
438; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
439; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
440; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
441; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
442; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
443; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
444; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
445; GFX90A-TGSPLIT-NEXT:    s_endpgm
446;
447; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_load:
448; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
449; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
450; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
451; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
452; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
453; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
454; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc1
455; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
456; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
457; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
458; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
459; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
460; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
461;
462; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_load:
463; GFX940-TGSPLIT:       ; %bb.0: ; %entry
464; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
465; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
466; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
467; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
468; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
469; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc1
470; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
471; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
472; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
473; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
474; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
475; GFX940-TGSPLIT-NEXT:    s_endpgm
476    i32* %in, i32* %out) {
477entry:
478  %val = load atomic i32, i32* %in syncscope("agent") seq_cst, align 4
479  store i32 %val, i32* %out
480  ret void
481}
482
483define amdgpu_kernel void @flat_agent_unordered_store(
484; GFX7-LABEL: flat_agent_unordered_store:
485; GFX7:       ; %bb.0: ; %entry
486; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
487; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
488; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
489; GFX7-NEXT:    v_mov_b32_e32 v0, s0
490; GFX7-NEXT:    v_mov_b32_e32 v1, s1
491; GFX7-NEXT:    v_mov_b32_e32 v2, s2
492; GFX7-NEXT:    flat_store_dword v[0:1], v2
493; GFX7-NEXT:    s_endpgm
494;
495; GFX10-WGP-LABEL: flat_agent_unordered_store:
496; GFX10-WGP:       ; %bb.0: ; %entry
497; GFX10-WGP-NEXT:    s_clause 0x1
498; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
499; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
500; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
501; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
502; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
503; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
504; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
505; GFX10-WGP-NEXT:    s_endpgm
506;
507; GFX10-CU-LABEL: flat_agent_unordered_store:
508; GFX10-CU:       ; %bb.0: ; %entry
509; GFX10-CU-NEXT:    s_clause 0x1
510; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
511; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
512; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
513; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
514; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
515; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
516; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
517; GFX10-CU-NEXT:    s_endpgm
518;
519; SKIP-CACHE-INV-LABEL: flat_agent_unordered_store:
520; SKIP-CACHE-INV:       ; %bb.0: ; %entry
521; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
522; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
523; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
524; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
525; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
526; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
527; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
528; SKIP-CACHE-INV-NEXT:    s_endpgm
529;
530; GFX90A-NOTTGSPLIT-LABEL: flat_agent_unordered_store:
531; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
532; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
533; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
534; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
535; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
536; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
537; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
538; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
539;
540; GFX90A-TGSPLIT-LABEL: flat_agent_unordered_store:
541; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
542; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
543; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
544; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
545; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
546; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
547; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
548; GFX90A-TGSPLIT-NEXT:    s_endpgm
549;
550; GFX940-NOTTGSPLIT-LABEL: flat_agent_unordered_store:
551; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
552; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
553; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
554; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
555; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
556; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
557; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
558; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
559;
560; GFX940-TGSPLIT-LABEL: flat_agent_unordered_store:
561; GFX940-TGSPLIT:       ; %bb.0: ; %entry
562; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
563; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
564; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
565; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
566; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
567; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
568; GFX940-TGSPLIT-NEXT:    s_endpgm
569    i32 %in, i32* %out) {
570entry:
571  store atomic i32 %in, i32* %out syncscope("agent") unordered, align 4
572  ret void
573}
574
575define amdgpu_kernel void @flat_agent_monotonic_store(
576; GFX7-LABEL: flat_agent_monotonic_store:
577; GFX7:       ; %bb.0: ; %entry
578; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
579; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
580; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
581; GFX7-NEXT:    v_mov_b32_e32 v0, s0
582; GFX7-NEXT:    v_mov_b32_e32 v1, s1
583; GFX7-NEXT:    v_mov_b32_e32 v2, s2
584; GFX7-NEXT:    flat_store_dword v[0:1], v2
585; GFX7-NEXT:    s_endpgm
586;
587; GFX10-WGP-LABEL: flat_agent_monotonic_store:
588; GFX10-WGP:       ; %bb.0: ; %entry
589; GFX10-WGP-NEXT:    s_clause 0x1
590; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
591; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
592; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
593; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
594; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
595; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
596; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
597; GFX10-WGP-NEXT:    s_endpgm
598;
599; GFX10-CU-LABEL: flat_agent_monotonic_store:
600; GFX10-CU:       ; %bb.0: ; %entry
601; GFX10-CU-NEXT:    s_clause 0x1
602; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
603; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
604; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
605; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
606; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
607; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
608; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
609; GFX10-CU-NEXT:    s_endpgm
610;
611; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_store:
612; SKIP-CACHE-INV:       ; %bb.0: ; %entry
613; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
614; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
615; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
616; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
617; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
618; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
619; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
620; SKIP-CACHE-INV-NEXT:    s_endpgm
621;
622; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_store:
623; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
624; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
625; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
626; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
627; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
628; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
629; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
630; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
631;
632; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_store:
633; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
634; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
635; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
636; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
637; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
638; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
639; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
640; GFX90A-TGSPLIT-NEXT:    s_endpgm
641;
642; GFX940-NOTTGSPLIT-LABEL: flat_agent_monotonic_store:
643; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
644; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
645; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
646; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
647; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
648; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
649; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc1
650; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
651;
652; GFX940-TGSPLIT-LABEL: flat_agent_monotonic_store:
653; GFX940-TGSPLIT:       ; %bb.0: ; %entry
654; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
655; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
656; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
657; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
658; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
659; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc1
660; GFX940-TGSPLIT-NEXT:    s_endpgm
661    i32 %in, i32* %out) {
662entry:
663  store atomic i32 %in, i32* %out syncscope("agent") monotonic, align 4
664  ret void
665}
666
667define amdgpu_kernel void @flat_agent_release_store(
668; GFX7-LABEL: flat_agent_release_store:
669; GFX7:       ; %bb.0: ; %entry
670; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
671; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
672; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
673; GFX7-NEXT:    v_mov_b32_e32 v0, s0
674; GFX7-NEXT:    v_mov_b32_e32 v1, s1
675; GFX7-NEXT:    v_mov_b32_e32 v2, s2
676; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
677; GFX7-NEXT:    flat_store_dword v[0:1], v2
678; GFX7-NEXT:    s_endpgm
679;
680; GFX10-WGP-LABEL: flat_agent_release_store:
681; GFX10-WGP:       ; %bb.0: ; %entry
682; GFX10-WGP-NEXT:    s_clause 0x1
683; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
684; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
685; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
686; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
687; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
688; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
689; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
690; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
691; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
692; GFX10-WGP-NEXT:    s_endpgm
693;
694; GFX10-CU-LABEL: flat_agent_release_store:
695; GFX10-CU:       ; %bb.0: ; %entry
696; GFX10-CU-NEXT:    s_clause 0x1
697; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
698; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
699; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
700; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
701; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
702; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
703; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
704; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
705; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
706; GFX10-CU-NEXT:    s_endpgm
707;
708; SKIP-CACHE-INV-LABEL: flat_agent_release_store:
709; SKIP-CACHE-INV:       ; %bb.0: ; %entry
710; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
711; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
712; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
713; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
714; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
715; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
716; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
717; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
718; SKIP-CACHE-INV-NEXT:    s_endpgm
719;
720; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_store:
721; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
722; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
723; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
724; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
725; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
726; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
727; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
728; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
729; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
730;
731; GFX90A-TGSPLIT-LABEL: flat_agent_release_store:
732; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
733; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
734; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
735; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
736; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
737; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
738; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
739; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
740; GFX90A-TGSPLIT-NEXT:    s_endpgm
741;
742; GFX940-NOTTGSPLIT-LABEL: flat_agent_release_store:
743; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
744; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
745; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
746; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
747; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
748; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
749; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
750; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
751; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc1
752; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
753;
754; GFX940-TGSPLIT-LABEL: flat_agent_release_store:
755; GFX940-TGSPLIT:       ; %bb.0: ; %entry
756; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
757; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
758; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
759; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
760; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
761; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
762; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
763; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc1
764; GFX940-TGSPLIT-NEXT:    s_endpgm
765    i32 %in, i32* %out) {
766entry:
767  store atomic i32 %in, i32* %out syncscope("agent") release, align 4
768  ret void
769}
770
771define amdgpu_kernel void @flat_agent_seq_cst_store(
772; GFX7-LABEL: flat_agent_seq_cst_store:
773; GFX7:       ; %bb.0: ; %entry
774; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
775; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
776; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
777; GFX7-NEXT:    v_mov_b32_e32 v0, s0
778; GFX7-NEXT:    v_mov_b32_e32 v1, s1
779; GFX7-NEXT:    v_mov_b32_e32 v2, s2
780; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
781; GFX7-NEXT:    flat_store_dword v[0:1], v2
782; GFX7-NEXT:    s_endpgm
783;
784; GFX10-WGP-LABEL: flat_agent_seq_cst_store:
785; GFX10-WGP:       ; %bb.0: ; %entry
786; GFX10-WGP-NEXT:    s_clause 0x1
787; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
788; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
789; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
790; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
791; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
792; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
793; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
794; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
795; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
796; GFX10-WGP-NEXT:    s_endpgm
797;
798; GFX10-CU-LABEL: flat_agent_seq_cst_store:
799; GFX10-CU:       ; %bb.0: ; %entry
800; GFX10-CU-NEXT:    s_clause 0x1
801; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
802; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
803; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
804; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
805; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
806; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
807; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
808; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
809; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
810; GFX10-CU-NEXT:    s_endpgm
811;
812; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_store:
813; SKIP-CACHE-INV:       ; %bb.0: ; %entry
814; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
815; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
816; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
817; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
818; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
819; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
820; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
821; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
822; SKIP-CACHE-INV-NEXT:    s_endpgm
823;
824; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_store:
825; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
826; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
827; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
828; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
829; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
830; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
831; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
832; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
833; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
834;
835; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_store:
836; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
837; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
838; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
839; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
840; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
841; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
842; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
843; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
844; GFX90A-TGSPLIT-NEXT:    s_endpgm
845;
846; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_store:
847; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
848; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
849; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
850; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
851; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
852; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
853; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
854; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
855; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc1
856; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
857;
858; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_store:
859; GFX940-TGSPLIT:       ; %bb.0: ; %entry
860; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
861; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
862; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
863; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
864; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
865; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
866; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
867; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc1
868; GFX940-TGSPLIT-NEXT:    s_endpgm
869    i32 %in, i32* %out) {
870entry:
871  store atomic i32 %in, i32* %out syncscope("agent") seq_cst, align 4
872  ret void
873}
874
875define amdgpu_kernel void @flat_agent_monotonic_atomicrmw(
876; GFX7-LABEL: flat_agent_monotonic_atomicrmw:
877; GFX7:       ; %bb.0: ; %entry
878; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
879; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
880; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
881; GFX7-NEXT:    v_mov_b32_e32 v0, s0
882; GFX7-NEXT:    v_mov_b32_e32 v1, s1
883; GFX7-NEXT:    v_mov_b32_e32 v2, s2
884; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
885; GFX7-NEXT:    s_endpgm
886;
887; GFX10-WGP-LABEL: flat_agent_monotonic_atomicrmw:
888; GFX10-WGP:       ; %bb.0: ; %entry
889; GFX10-WGP-NEXT:    s_clause 0x1
890; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
891; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
892; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
893; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
894; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
895; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
896; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
897; GFX10-WGP-NEXT:    s_endpgm
898;
899; GFX10-CU-LABEL: flat_agent_monotonic_atomicrmw:
900; GFX10-CU:       ; %bb.0: ; %entry
901; GFX10-CU-NEXT:    s_clause 0x1
902; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
903; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
904; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
905; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
906; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
907; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
908; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
909; GFX10-CU-NEXT:    s_endpgm
910;
911; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_atomicrmw:
912; SKIP-CACHE-INV:       ; %bb.0: ; %entry
913; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
914; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
915; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
916; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
917; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
918; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
919; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
920; SKIP-CACHE-INV-NEXT:    s_endpgm
921;
922; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_atomicrmw:
923; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
924; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
925; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
926; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
927; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
928; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
929; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
930; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
931;
932; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_atomicrmw:
933; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
934; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
935; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
936; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
937; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
938; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
939; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
940; GFX90A-TGSPLIT-NEXT:    s_endpgm
941;
942; GFX940-NOTTGSPLIT-LABEL: flat_agent_monotonic_atomicrmw:
943; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
944; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
945; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
946; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
947; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
948; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
949; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
950; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
951;
952; GFX940-TGSPLIT-LABEL: flat_agent_monotonic_atomicrmw:
953; GFX940-TGSPLIT:       ; %bb.0: ; %entry
954; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
955; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
956; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
957; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
958; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
959; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
960; GFX940-TGSPLIT-NEXT:    s_endpgm
961    i32* %out, i32 %in) {
962entry:
963  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") monotonic
964  ret void
965}
966
967define amdgpu_kernel void @flat_agent_acquire_atomicrmw(
968; GFX7-LABEL: flat_agent_acquire_atomicrmw:
969; GFX7:       ; %bb.0: ; %entry
970; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
971; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
972; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
973; GFX7-NEXT:    v_mov_b32_e32 v0, s0
974; GFX7-NEXT:    v_mov_b32_e32 v1, s1
975; GFX7-NEXT:    v_mov_b32_e32 v2, s2
976; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
977; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
978; GFX7-NEXT:    buffer_wbinvl1_vol
979; GFX7-NEXT:    s_endpgm
980;
981; GFX10-WGP-LABEL: flat_agent_acquire_atomicrmw:
982; GFX10-WGP:       ; %bb.0: ; %entry
983; GFX10-WGP-NEXT:    s_clause 0x1
984; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
985; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
986; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
987; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
988; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
989; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
990; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
991; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
992; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
993; GFX10-WGP-NEXT:    buffer_gl0_inv
994; GFX10-WGP-NEXT:    buffer_gl1_inv
995; GFX10-WGP-NEXT:    s_endpgm
996;
997; GFX10-CU-LABEL: flat_agent_acquire_atomicrmw:
998; GFX10-CU:       ; %bb.0: ; %entry
999; GFX10-CU-NEXT:    s_clause 0x1
1000; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1001; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1002; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1003; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1004; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1005; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1006; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1007; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1008; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1009; GFX10-CU-NEXT:    buffer_gl0_inv
1010; GFX10-CU-NEXT:    buffer_gl1_inv
1011; GFX10-CU-NEXT:    s_endpgm
1012;
1013; SKIP-CACHE-INV-LABEL: flat_agent_acquire_atomicrmw:
1014; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1015; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1016; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1017; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1018; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1019; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1020; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1021; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1022; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1023; SKIP-CACHE-INV-NEXT:    s_endpgm
1024;
1025; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_atomicrmw:
1026; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1027; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1028; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1029; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1030; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1031; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1032; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1033; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1034; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1035; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1036;
1037; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_atomicrmw:
1038; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1039; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1040; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1041; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1042; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1043; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1044; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1045; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1046; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1047; GFX90A-TGSPLIT-NEXT:    s_endpgm
1048;
1049; GFX940-NOTTGSPLIT-LABEL: flat_agent_acquire_atomicrmw:
1050; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1051; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1052; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1053; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1054; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1055; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1056; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1057; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1058; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
1059; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1060;
1061; GFX940-TGSPLIT-LABEL: flat_agent_acquire_atomicrmw:
1062; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1063; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1064; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1065; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1066; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1067; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1068; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1069; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1070; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
1071; GFX940-TGSPLIT-NEXT:    s_endpgm
1072    i32* %out, i32 %in) {
1073entry:
1074  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") acquire
1075  ret void
1076}
1077
1078define amdgpu_kernel void @flat_agent_release_atomicrmw(
1079; GFX7-LABEL: flat_agent_release_atomicrmw:
1080; GFX7:       ; %bb.0: ; %entry
1081; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1082; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1083; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1084; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1085; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1086; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1087; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1088; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1089; GFX7-NEXT:    s_endpgm
1090;
1091; GFX10-WGP-LABEL: flat_agent_release_atomicrmw:
1092; GFX10-WGP:       ; %bb.0: ; %entry
1093; GFX10-WGP-NEXT:    s_clause 0x1
1094; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1095; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1096; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1097; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1098; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1099; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1100; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1101; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1102; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1103; GFX10-WGP-NEXT:    s_endpgm
1104;
1105; GFX10-CU-LABEL: flat_agent_release_atomicrmw:
1106; GFX10-CU:       ; %bb.0: ; %entry
1107; GFX10-CU-NEXT:    s_clause 0x1
1108; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1109; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1110; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1111; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1112; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1113; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1114; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1115; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1116; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1117; GFX10-CU-NEXT:    s_endpgm
1118;
1119; SKIP-CACHE-INV-LABEL: flat_agent_release_atomicrmw:
1120; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1121; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1122; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1123; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1124; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1125; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1126; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1127; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1128; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1129; SKIP-CACHE-INV-NEXT:    s_endpgm
1130;
1131; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_atomicrmw:
1132; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1133; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1134; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1135; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1136; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1137; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1138; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1139; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1140; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1141;
1142; GFX90A-TGSPLIT-LABEL: flat_agent_release_atomicrmw:
1143; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1144; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1145; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1146; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1147; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1148; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1149; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1150; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1151; GFX90A-TGSPLIT-NEXT:    s_endpgm
1152;
1153; GFX940-NOTTGSPLIT-LABEL: flat_agent_release_atomicrmw:
1154; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1155; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1156; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1157; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1158; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1159; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1160; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
1161; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1162; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1163; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1164;
1165; GFX940-TGSPLIT-LABEL: flat_agent_release_atomicrmw:
1166; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1167; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1168; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1169; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1170; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1171; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1172; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
1173; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1174; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1175; GFX940-TGSPLIT-NEXT:    s_endpgm
1176    i32* %out, i32 %in) {
1177entry:
1178  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") release
1179  ret void
1180}
1181
1182define amdgpu_kernel void @flat_agent_acq_rel_atomicrmw(
1183; GFX7-LABEL: flat_agent_acq_rel_atomicrmw:
1184; GFX7:       ; %bb.0: ; %entry
1185; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1186; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1187; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1188; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1189; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1190; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1191; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1192; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1193; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1194; GFX7-NEXT:    buffer_wbinvl1_vol
1195; GFX7-NEXT:    s_endpgm
1196;
1197; GFX10-WGP-LABEL: flat_agent_acq_rel_atomicrmw:
1198; GFX10-WGP:       ; %bb.0: ; %entry
1199; GFX10-WGP-NEXT:    s_clause 0x1
1200; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1201; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1202; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1203; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1204; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1205; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1206; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1207; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1208; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1209; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1210; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1211; GFX10-WGP-NEXT:    buffer_gl0_inv
1212; GFX10-WGP-NEXT:    buffer_gl1_inv
1213; GFX10-WGP-NEXT:    s_endpgm
1214;
1215; GFX10-CU-LABEL: flat_agent_acq_rel_atomicrmw:
1216; GFX10-CU:       ; %bb.0: ; %entry
1217; GFX10-CU-NEXT:    s_clause 0x1
1218; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1219; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1220; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1221; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1222; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1223; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1224; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1225; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1226; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1227; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1228; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1229; GFX10-CU-NEXT:    buffer_gl0_inv
1230; GFX10-CU-NEXT:    buffer_gl1_inv
1231; GFX10-CU-NEXT:    s_endpgm
1232;
1233; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_atomicrmw:
1234; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1235; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1236; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1237; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1238; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1239; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1240; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1241; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1242; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1243; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1244; SKIP-CACHE-INV-NEXT:    s_endpgm
1245;
1246; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_atomicrmw:
1247; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1248; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1249; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1250; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1251; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1252; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1253; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1254; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1255; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1256; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1257; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1258;
1259; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_atomicrmw:
1260; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1261; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1262; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1263; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1264; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1265; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1266; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1267; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1268; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1269; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1270; GFX90A-TGSPLIT-NEXT:    s_endpgm
1271;
1272; GFX940-NOTTGSPLIT-LABEL: flat_agent_acq_rel_atomicrmw:
1273; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1274; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1275; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1276; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1277; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1278; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1279; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
1280; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1281; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1282; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1283; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
1284; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1285;
1286; GFX940-TGSPLIT-LABEL: flat_agent_acq_rel_atomicrmw:
1287; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1288; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1289; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1290; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1291; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1292; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1293; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
1294; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1295; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1296; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1297; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
1298; GFX940-TGSPLIT-NEXT:    s_endpgm
1299    i32* %out, i32 %in) {
1300entry:
1301  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") acq_rel
1302  ret void
1303}
1304
1305define amdgpu_kernel void @flat_agent_seq_cst_atomicrmw(
1306; GFX7-LABEL: flat_agent_seq_cst_atomicrmw:
1307; GFX7:       ; %bb.0: ; %entry
1308; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1309; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1310; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1311; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1312; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1313; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1314; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1315; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1316; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1317; GFX7-NEXT:    buffer_wbinvl1_vol
1318; GFX7-NEXT:    s_endpgm
1319;
1320; GFX10-WGP-LABEL: flat_agent_seq_cst_atomicrmw:
1321; GFX10-WGP:       ; %bb.0: ; %entry
1322; GFX10-WGP-NEXT:    s_clause 0x1
1323; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1324; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1325; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1326; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1327; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1328; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1329; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1330; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1331; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1332; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1333; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1334; GFX10-WGP-NEXT:    buffer_gl0_inv
1335; GFX10-WGP-NEXT:    buffer_gl1_inv
1336; GFX10-WGP-NEXT:    s_endpgm
1337;
1338; GFX10-CU-LABEL: flat_agent_seq_cst_atomicrmw:
1339; GFX10-CU:       ; %bb.0: ; %entry
1340; GFX10-CU-NEXT:    s_clause 0x1
1341; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1342; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1343; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1344; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1345; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1346; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1347; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1348; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1349; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1350; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1351; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1352; GFX10-CU-NEXT:    buffer_gl0_inv
1353; GFX10-CU-NEXT:    buffer_gl1_inv
1354; GFX10-CU-NEXT:    s_endpgm
1355;
1356; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_atomicrmw:
1357; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1358; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1359; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1360; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1361; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1362; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1363; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1364; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1365; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1366; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1367; SKIP-CACHE-INV-NEXT:    s_endpgm
1368;
1369; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_atomicrmw:
1370; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1371; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1372; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1373; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1374; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1375; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1376; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1377; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1378; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1379; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1380; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1381;
1382; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_atomicrmw:
1383; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1384; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1385; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1386; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1387; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1388; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1389; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1390; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1391; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1392; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1393; GFX90A-TGSPLIT-NEXT:    s_endpgm
1394;
1395; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_atomicrmw:
1396; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1397; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1398; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1399; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1400; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1401; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1402; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
1403; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1404; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1405; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1406; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
1407; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1408;
1409; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_atomicrmw:
1410; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1411; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1412; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1413; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1414; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1415; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1416; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
1417; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1418; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1419; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1420; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
1421; GFX940-TGSPLIT-NEXT:    s_endpgm
1422    i32* %out, i32 %in) {
1423entry:
1424  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") seq_cst
1425  ret void
1426}
1427
1428define amdgpu_kernel void @flat_agent_acquire_ret_atomicrmw(
1429; GFX7-LABEL: flat_agent_acquire_ret_atomicrmw:
1430; GFX7:       ; %bb.0: ; %entry
1431; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1432; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1433; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1434; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1435; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1436; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1437; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1438; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1439; GFX7-NEXT:    buffer_wbinvl1_vol
1440; GFX7-NEXT:    flat_store_dword v[0:1], v2
1441; GFX7-NEXT:    s_endpgm
1442;
1443; GFX10-WGP-LABEL: flat_agent_acquire_ret_atomicrmw:
1444; GFX10-WGP:       ; %bb.0: ; %entry
1445; GFX10-WGP-NEXT:    s_clause 0x1
1446; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1447; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1448; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1449; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1450; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1451; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1452; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1453; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1454; GFX10-WGP-NEXT:    buffer_gl0_inv
1455; GFX10-WGP-NEXT:    buffer_gl1_inv
1456; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1457; GFX10-WGP-NEXT:    s_endpgm
1458;
1459; GFX10-CU-LABEL: flat_agent_acquire_ret_atomicrmw:
1460; GFX10-CU:       ; %bb.0: ; %entry
1461; GFX10-CU-NEXT:    s_clause 0x1
1462; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1463; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1464; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1465; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1466; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1467; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1468; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1469; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1470; GFX10-CU-NEXT:    buffer_gl0_inv
1471; GFX10-CU-NEXT:    buffer_gl1_inv
1472; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1473; GFX10-CU-NEXT:    s_endpgm
1474;
1475; SKIP-CACHE-INV-LABEL: flat_agent_acquire_ret_atomicrmw:
1476; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1477; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1478; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1479; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1480; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1481; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1482; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1483; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1484; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1485; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1486; SKIP-CACHE-INV-NEXT:    s_endpgm
1487;
1488; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_ret_atomicrmw:
1489; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1490; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1491; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1492; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1493; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1494; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1495; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1496; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1497; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1498; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1499; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1500;
1501; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_ret_atomicrmw:
1502; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1503; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1504; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1505; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1506; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1507; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1508; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1509; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1510; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1511; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1512; GFX90A-TGSPLIT-NEXT:    s_endpgm
1513;
1514; GFX940-NOTTGSPLIT-LABEL: flat_agent_acquire_ret_atomicrmw:
1515; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1516; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1517; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1518; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1519; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1520; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1521; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
1522; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1523; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
1524; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1525; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1526;
1527; GFX940-TGSPLIT-LABEL: flat_agent_acquire_ret_atomicrmw:
1528; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1529; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1530; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1531; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1532; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1533; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1534; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
1535; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1536; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
1537; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1538; GFX940-TGSPLIT-NEXT:    s_endpgm
1539    i32* %out, i32 %in) {
1540entry:
1541  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") acquire
1542  store i32 %val, i32* %out, align 4
1543  ret void
1544}
1545
1546define amdgpu_kernel void @flat_agent_acq_rel_ret_atomicrmw(
1547; GFX7-LABEL: flat_agent_acq_rel_ret_atomicrmw:
1548; GFX7:       ; %bb.0: ; %entry
1549; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1550; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1551; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1552; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1553; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1554; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1555; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1556; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1557; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1558; GFX7-NEXT:    buffer_wbinvl1_vol
1559; GFX7-NEXT:    flat_store_dword v[0:1], v2
1560; GFX7-NEXT:    s_endpgm
1561;
1562; GFX10-WGP-LABEL: flat_agent_acq_rel_ret_atomicrmw:
1563; GFX10-WGP:       ; %bb.0: ; %entry
1564; GFX10-WGP-NEXT:    s_clause 0x1
1565; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1566; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1567; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1568; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1569; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1570; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1571; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1572; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1573; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1574; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1575; GFX10-WGP-NEXT:    buffer_gl0_inv
1576; GFX10-WGP-NEXT:    buffer_gl1_inv
1577; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1578; GFX10-WGP-NEXT:    s_endpgm
1579;
1580; GFX10-CU-LABEL: flat_agent_acq_rel_ret_atomicrmw:
1581; GFX10-CU:       ; %bb.0: ; %entry
1582; GFX10-CU-NEXT:    s_clause 0x1
1583; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1584; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1585; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1586; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1587; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1588; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1589; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1590; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1591; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1592; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1593; GFX10-CU-NEXT:    buffer_gl0_inv
1594; GFX10-CU-NEXT:    buffer_gl1_inv
1595; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1596; GFX10-CU-NEXT:    s_endpgm
1597;
1598; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_ret_atomicrmw:
1599; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1600; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1601; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1602; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1603; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1604; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1605; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1606; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1607; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1608; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1609; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1610; SKIP-CACHE-INV-NEXT:    s_endpgm
1611;
1612; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_ret_atomicrmw:
1613; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1614; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1615; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1616; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1617; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1618; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1619; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1620; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1621; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1622; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1623; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1624; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1625;
1626; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_ret_atomicrmw:
1627; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1628; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1629; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1630; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1631; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1632; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1633; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1634; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1635; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1636; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1637; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1638; GFX90A-TGSPLIT-NEXT:    s_endpgm
1639;
1640; GFX940-NOTTGSPLIT-LABEL: flat_agent_acq_rel_ret_atomicrmw:
1641; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1642; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1643; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1644; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1645; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1646; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1647; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
1648; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1649; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
1650; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1651; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
1652; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1653; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1654;
1655; GFX940-TGSPLIT-LABEL: flat_agent_acq_rel_ret_atomicrmw:
1656; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1657; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1658; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1659; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1660; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1661; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1662; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
1663; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1664; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
1665; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1666; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
1667; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1668; GFX940-TGSPLIT-NEXT:    s_endpgm
1669    i32* %out, i32 %in) {
1670entry:
1671  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") acq_rel
1672  store i32 %val, i32* %out, align 4
1673  ret void
1674}
1675
1676define amdgpu_kernel void @flat_agent_seq_cst_ret_atomicrmw(
1677; GFX7-LABEL: flat_agent_seq_cst_ret_atomicrmw:
1678; GFX7:       ; %bb.0: ; %entry
1679; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1680; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1681; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1682; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1683; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1684; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1685; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1686; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1687; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1688; GFX7-NEXT:    buffer_wbinvl1_vol
1689; GFX7-NEXT:    flat_store_dword v[0:1], v2
1690; GFX7-NEXT:    s_endpgm
1691;
1692; GFX10-WGP-LABEL: flat_agent_seq_cst_ret_atomicrmw:
1693; GFX10-WGP:       ; %bb.0: ; %entry
1694; GFX10-WGP-NEXT:    s_clause 0x1
1695; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1696; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1697; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1698; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1699; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1700; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1701; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1702; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1703; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1704; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1705; GFX10-WGP-NEXT:    buffer_gl0_inv
1706; GFX10-WGP-NEXT:    buffer_gl1_inv
1707; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1708; GFX10-WGP-NEXT:    s_endpgm
1709;
1710; GFX10-CU-LABEL: flat_agent_seq_cst_ret_atomicrmw:
1711; GFX10-CU:       ; %bb.0: ; %entry
1712; GFX10-CU-NEXT:    s_clause 0x1
1713; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1714; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1715; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1716; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1717; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1718; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1719; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1720; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1721; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1722; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1723; GFX10-CU-NEXT:    buffer_gl0_inv
1724; GFX10-CU-NEXT:    buffer_gl1_inv
1725; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1726; GFX10-CU-NEXT:    s_endpgm
1727;
1728; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_ret_atomicrmw:
1729; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1730; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1731; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1732; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1733; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1734; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1735; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1736; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1737; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1738; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1739; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1740; SKIP-CACHE-INV-NEXT:    s_endpgm
1741;
1742; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_ret_atomicrmw:
1743; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1744; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1745; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1746; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1747; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1748; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1749; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1750; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1751; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1752; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1753; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1754; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1755;
1756; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_ret_atomicrmw:
1757; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1758; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1759; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1760; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1761; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1762; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1763; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1764; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1765; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1766; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1767; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1768; GFX90A-TGSPLIT-NEXT:    s_endpgm
1769;
1770; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_ret_atomicrmw:
1771; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1772; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1773; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1774; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1775; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1776; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1777; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
1778; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1779; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
1780; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1781; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
1782; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1783; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1784;
1785; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_ret_atomicrmw:
1786; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1787; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1788; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1789; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1790; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1791; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1792; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
1793; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1794; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
1795; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1796; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
1797; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1798; GFX940-TGSPLIT-NEXT:    s_endpgm
1799    i32* %out, i32 %in) {
1800entry:
1801  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") seq_cst
1802  store i32 %val, i32* %out, align 4
1803  ret void
1804}
1805
1806define amdgpu_kernel void @flat_agent_monotonic_monotonic_cmpxchg(
1807; GFX7-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
1808; GFX7:       ; %bb.0: ; %entry
1809; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1810; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1811; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1812; GFX7-NEXT:    s_add_u32 s0, s0, 16
1813; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1814; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1815; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1816; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1817; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1818; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1819; GFX7-NEXT:    s_endpgm
1820;
1821; GFX10-WGP-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
1822; GFX10-WGP:       ; %bb.0: ; %entry
1823; GFX10-WGP-NEXT:    s_clause 0x1
1824; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1825; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1826; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1827; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1828; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1829; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1830; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1831; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1832; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1833; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1834; GFX10-WGP-NEXT:    s_endpgm
1835;
1836; GFX10-CU-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
1837; GFX10-CU:       ; %bb.0: ; %entry
1838; GFX10-CU-NEXT:    s_clause 0x1
1839; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1840; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1841; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1842; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1843; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1844; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1845; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1846; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1847; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1848; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1849; GFX10-CU-NEXT:    s_endpgm
1850;
1851; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
1852; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1853; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1854; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
1855; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1856; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1857; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1858; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1859; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1860; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1861; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1862; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1863; SKIP-CACHE-INV-NEXT:    s_endpgm
1864;
1865; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
1866; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1867; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1868; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1869; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1870; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1871; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1872; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1873; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1874;
1875; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
1876; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1877; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1878; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1879; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1880; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1881; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1882; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1883; GFX90A-TGSPLIT-NEXT:    s_endpgm
1884;
1885; GFX940-NOTTGSPLIT-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
1886; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1887; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1888; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
1889; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1890; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1891; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
1892; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1893; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1894;
1895; GFX940-TGSPLIT-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
1896; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1897; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1898; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
1899; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1900; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1901; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
1902; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1903; GFX940-TGSPLIT-NEXT:    s_endpgm
1904    i32* %out, i32 %in, i32 %old) {
1905entry:
1906  %gep = getelementptr i32, i32* %out, i32 4
1907  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") monotonic monotonic
1908  ret void
1909}
1910
1911define amdgpu_kernel void @flat_agent_acquire_monotonic_cmpxchg(
1912; GFX7-LABEL: flat_agent_acquire_monotonic_cmpxchg:
1913; GFX7:       ; %bb.0: ; %entry
1914; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1915; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1916; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1917; GFX7-NEXT:    s_add_u32 s0, s0, 16
1918; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1919; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1920; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1921; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1922; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1923; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1924; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1925; GFX7-NEXT:    buffer_wbinvl1_vol
1926; GFX7-NEXT:    s_endpgm
1927;
1928; GFX10-WGP-LABEL: flat_agent_acquire_monotonic_cmpxchg:
1929; GFX10-WGP:       ; %bb.0: ; %entry
1930; GFX10-WGP-NEXT:    s_clause 0x1
1931; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1932; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1933; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1934; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1935; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1936; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1937; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1938; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1939; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1940; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1941; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1942; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1943; GFX10-WGP-NEXT:    buffer_gl0_inv
1944; GFX10-WGP-NEXT:    buffer_gl1_inv
1945; GFX10-WGP-NEXT:    s_endpgm
1946;
1947; GFX10-CU-LABEL: flat_agent_acquire_monotonic_cmpxchg:
1948; GFX10-CU:       ; %bb.0: ; %entry
1949; GFX10-CU-NEXT:    s_clause 0x1
1950; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1951; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1952; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1953; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1954; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1955; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1956; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1957; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1958; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1959; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1960; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1961; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1962; GFX10-CU-NEXT:    buffer_gl0_inv
1963; GFX10-CU-NEXT:    buffer_gl1_inv
1964; GFX10-CU-NEXT:    s_endpgm
1965;
1966; SKIP-CACHE-INV-LABEL: flat_agent_acquire_monotonic_cmpxchg:
1967; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1968; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1969; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
1970; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1971; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1972; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1973; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1974; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1975; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1976; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1977; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1978; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1979; SKIP-CACHE-INV-NEXT:    s_endpgm
1980;
1981; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_monotonic_cmpxchg:
1982; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1983; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1984; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1985; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1986; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1987; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1988; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1989; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1990; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1991; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1992;
1993; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_monotonic_cmpxchg:
1994; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1995; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1996; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1997; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1998; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1999; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2000; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2001; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2002; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2003; GFX90A-TGSPLIT-NEXT:    s_endpgm
2004;
2005; GFX940-NOTTGSPLIT-LABEL: flat_agent_acquire_monotonic_cmpxchg:
2006; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2007; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2008; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2009; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2010; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2011; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2012; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2013; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2014; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
2015; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2016;
2017; GFX940-TGSPLIT-LABEL: flat_agent_acquire_monotonic_cmpxchg:
2018; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2019; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2020; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2021; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2022; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2023; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2024; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2025; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2026; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
2027; GFX940-TGSPLIT-NEXT:    s_endpgm
2028    i32* %out, i32 %in, i32 %old) {
2029entry:
2030  %gep = getelementptr i32, i32* %out, i32 4
2031  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire monotonic
2032  ret void
2033}
2034
2035define amdgpu_kernel void @flat_agent_release_monotonic_cmpxchg(
2036; GFX7-LABEL: flat_agent_release_monotonic_cmpxchg:
2037; GFX7:       ; %bb.0: ; %entry
2038; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2039; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2040; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2041; GFX7-NEXT:    s_add_u32 s0, s0, 16
2042; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2043; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2044; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2045; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2046; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2047; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2048; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2049; GFX7-NEXT:    s_endpgm
2050;
2051; GFX10-WGP-LABEL: flat_agent_release_monotonic_cmpxchg:
2052; GFX10-WGP:       ; %bb.0: ; %entry
2053; GFX10-WGP-NEXT:    s_clause 0x1
2054; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2055; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2056; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2057; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2058; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2059; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2060; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2061; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2062; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2063; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2064; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2065; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2066; GFX10-WGP-NEXT:    s_endpgm
2067;
2068; GFX10-CU-LABEL: flat_agent_release_monotonic_cmpxchg:
2069; GFX10-CU:       ; %bb.0: ; %entry
2070; GFX10-CU-NEXT:    s_clause 0x1
2071; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2072; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2073; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2074; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2075; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2076; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2077; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2078; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2079; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2080; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2081; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2082; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2083; GFX10-CU-NEXT:    s_endpgm
2084;
2085; SKIP-CACHE-INV-LABEL: flat_agent_release_monotonic_cmpxchg:
2086; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2087; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2088; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2089; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2090; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2091; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2092; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2093; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2094; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2095; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2096; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2097; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2098; SKIP-CACHE-INV-NEXT:    s_endpgm
2099;
2100; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_monotonic_cmpxchg:
2101; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2102; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2103; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2104; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2105; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2106; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2107; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2108; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2109; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2110;
2111; GFX90A-TGSPLIT-LABEL: flat_agent_release_monotonic_cmpxchg:
2112; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2113; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2114; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2115; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2116; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2117; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2118; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2119; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2120; GFX90A-TGSPLIT-NEXT:    s_endpgm
2121;
2122; GFX940-NOTTGSPLIT-LABEL: flat_agent_release_monotonic_cmpxchg:
2123; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2124; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2125; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2126; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2127; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2128; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2129; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
2130; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2131; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2132; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2133;
2134; GFX940-TGSPLIT-LABEL: flat_agent_release_monotonic_cmpxchg:
2135; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2136; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2137; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2138; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2139; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2140; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2141; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
2142; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2143; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2144; GFX940-TGSPLIT-NEXT:    s_endpgm
2145    i32* %out, i32 %in, i32 %old) {
2146entry:
2147  %gep = getelementptr i32, i32* %out, i32 4
2148  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release monotonic
2149  ret void
2150}
2151
2152define amdgpu_kernel void @flat_agent_acq_rel_monotonic_cmpxchg(
2153; GFX7-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
2154; GFX7:       ; %bb.0: ; %entry
2155; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2156; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2157; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2158; GFX7-NEXT:    s_add_u32 s0, s0, 16
2159; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2160; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2161; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2162; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2163; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2164; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2165; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2166; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2167; GFX7-NEXT:    buffer_wbinvl1_vol
2168; GFX7-NEXT:    s_endpgm
2169;
2170; GFX10-WGP-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
2171; GFX10-WGP:       ; %bb.0: ; %entry
2172; GFX10-WGP-NEXT:    s_clause 0x1
2173; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2174; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2175; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2176; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2177; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2178; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2179; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2180; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2181; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2182; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2183; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2184; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2185; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2186; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2187; GFX10-WGP-NEXT:    buffer_gl0_inv
2188; GFX10-WGP-NEXT:    buffer_gl1_inv
2189; GFX10-WGP-NEXT:    s_endpgm
2190;
2191; GFX10-CU-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
2192; GFX10-CU:       ; %bb.0: ; %entry
2193; GFX10-CU-NEXT:    s_clause 0x1
2194; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2195; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2196; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2197; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2198; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2199; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2200; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2201; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2202; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2203; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2204; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2205; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2206; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2207; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2208; GFX10-CU-NEXT:    buffer_gl0_inv
2209; GFX10-CU-NEXT:    buffer_gl1_inv
2210; GFX10-CU-NEXT:    s_endpgm
2211;
2212; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
2213; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2214; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2215; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2216; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2217; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2218; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2219; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2220; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2221; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2222; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2223; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2224; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2225; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2226; SKIP-CACHE-INV-NEXT:    s_endpgm
2227;
2228; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
2229; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2230; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2231; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2232; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2233; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2234; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2235; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2236; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2237; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2238; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2239; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2240;
2241; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
2242; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2243; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2244; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2245; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2246; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2247; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2248; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2249; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2250; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2251; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2252; GFX90A-TGSPLIT-NEXT:    s_endpgm
2253;
2254; GFX940-NOTTGSPLIT-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
2255; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2256; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2257; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2258; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2259; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2260; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2261; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
2262; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2263; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2264; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2265; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
2266; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2267;
2268; GFX940-TGSPLIT-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
2269; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2270; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2271; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2272; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2273; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2274; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2275; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
2276; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2277; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2278; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2279; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
2280; GFX940-TGSPLIT-NEXT:    s_endpgm
2281    i32* %out, i32 %in, i32 %old) {
2282entry:
2283  %gep = getelementptr i32, i32* %out, i32 4
2284  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel monotonic
2285  ret void
2286}
2287
2288define amdgpu_kernel void @flat_agent_seq_cst_monotonic_cmpxchg(
2289; GFX7-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
2290; GFX7:       ; %bb.0: ; %entry
2291; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2292; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2293; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2294; GFX7-NEXT:    s_add_u32 s0, s0, 16
2295; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2296; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2297; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2298; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2299; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2300; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2301; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2302; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2303; GFX7-NEXT:    buffer_wbinvl1_vol
2304; GFX7-NEXT:    s_endpgm
2305;
2306; GFX10-WGP-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
2307; GFX10-WGP:       ; %bb.0: ; %entry
2308; GFX10-WGP-NEXT:    s_clause 0x1
2309; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2310; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2311; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2312; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2313; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2314; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2315; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2316; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2317; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2318; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2319; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2320; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2321; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2322; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2323; GFX10-WGP-NEXT:    buffer_gl0_inv
2324; GFX10-WGP-NEXT:    buffer_gl1_inv
2325; GFX10-WGP-NEXT:    s_endpgm
2326;
2327; GFX10-CU-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
2328; GFX10-CU:       ; %bb.0: ; %entry
2329; GFX10-CU-NEXT:    s_clause 0x1
2330; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2331; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2332; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2333; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2334; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2335; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2336; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2337; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2338; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2339; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2340; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2341; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2342; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2343; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2344; GFX10-CU-NEXT:    buffer_gl0_inv
2345; GFX10-CU-NEXT:    buffer_gl1_inv
2346; GFX10-CU-NEXT:    s_endpgm
2347;
2348; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
2349; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2350; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2351; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2352; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2353; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2354; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2355; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2356; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2357; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2358; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2359; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2360; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2361; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2362; SKIP-CACHE-INV-NEXT:    s_endpgm
2363;
2364; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
2365; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2366; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2367; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2368; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2369; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2370; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2371; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2372; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2373; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2374; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2375; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2376;
2377; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
2378; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2379; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2380; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2381; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2382; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2383; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2384; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2385; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2386; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2387; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2388; GFX90A-TGSPLIT-NEXT:    s_endpgm
2389;
2390; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
2391; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2392; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2393; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2394; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2395; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2396; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2397; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
2398; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2399; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2400; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2401; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
2402; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2403;
2404; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
2405; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2406; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2407; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2408; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2409; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2410; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2411; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
2412; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2413; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2414; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2415; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
2416; GFX940-TGSPLIT-NEXT:    s_endpgm
2417    i32* %out, i32 %in, i32 %old) {
2418entry:
2419  %gep = getelementptr i32, i32* %out, i32 4
2420  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst monotonic
2421  ret void
2422}
2423
2424define amdgpu_kernel void @flat_agent_monotonic_acquire_cmpxchg(
2425; GFX7-LABEL: flat_agent_monotonic_acquire_cmpxchg:
2426; GFX7:       ; %bb.0: ; %entry
2427; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2428; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2429; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2430; GFX7-NEXT:    s_add_u32 s0, s0, 16
2431; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2432; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2433; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2434; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2435; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2436; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2437; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2438; GFX7-NEXT:    buffer_wbinvl1_vol
2439; GFX7-NEXT:    s_endpgm
2440;
2441; GFX10-WGP-LABEL: flat_agent_monotonic_acquire_cmpxchg:
2442; GFX10-WGP:       ; %bb.0: ; %entry
2443; GFX10-WGP-NEXT:    s_clause 0x1
2444; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2445; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2446; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2447; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2448; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2449; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2450; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2451; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2452; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2453; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2454; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2455; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2456; GFX10-WGP-NEXT:    buffer_gl0_inv
2457; GFX10-WGP-NEXT:    buffer_gl1_inv
2458; GFX10-WGP-NEXT:    s_endpgm
2459;
2460; GFX10-CU-LABEL: flat_agent_monotonic_acquire_cmpxchg:
2461; GFX10-CU:       ; %bb.0: ; %entry
2462; GFX10-CU-NEXT:    s_clause 0x1
2463; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2464; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2465; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2466; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2467; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2468; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2469; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2470; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2471; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2472; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2473; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2474; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2475; GFX10-CU-NEXT:    buffer_gl0_inv
2476; GFX10-CU-NEXT:    buffer_gl1_inv
2477; GFX10-CU-NEXT:    s_endpgm
2478;
2479; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_acquire_cmpxchg:
2480; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2481; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2482; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2483; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2484; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2485; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2486; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2487; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2488; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2489; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2490; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2491; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2492; SKIP-CACHE-INV-NEXT:    s_endpgm
2493;
2494; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_acquire_cmpxchg:
2495; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2496; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2497; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2498; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2499; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2500; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2501; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2502; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2503; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2504; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2505;
2506; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_acquire_cmpxchg:
2507; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2508; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2509; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2510; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2511; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2512; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2513; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2514; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2515; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2516; GFX90A-TGSPLIT-NEXT:    s_endpgm
2517;
2518; GFX940-NOTTGSPLIT-LABEL: flat_agent_monotonic_acquire_cmpxchg:
2519; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2520; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2521; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2522; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2523; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2524; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2525; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2526; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2527; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
2528; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2529;
2530; GFX940-TGSPLIT-LABEL: flat_agent_monotonic_acquire_cmpxchg:
2531; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2532; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2533; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2534; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2535; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2536; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2537; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2538; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2539; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
2540; GFX940-TGSPLIT-NEXT:    s_endpgm
2541    i32* %out, i32 %in, i32 %old) {
2542entry:
2543  %gep = getelementptr i32, i32* %out, i32 4
2544  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") monotonic acquire
2545  ret void
2546}
2547
2548define amdgpu_kernel void @flat_agent_acquire_acquire_cmpxchg(
2549; GFX7-LABEL: flat_agent_acquire_acquire_cmpxchg:
2550; GFX7:       ; %bb.0: ; %entry
2551; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2552; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2553; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2554; GFX7-NEXT:    s_add_u32 s0, s0, 16
2555; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2556; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2557; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2558; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2559; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2560; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2561; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2562; GFX7-NEXT:    buffer_wbinvl1_vol
2563; GFX7-NEXT:    s_endpgm
2564;
2565; GFX10-WGP-LABEL: flat_agent_acquire_acquire_cmpxchg:
2566; GFX10-WGP:       ; %bb.0: ; %entry
2567; GFX10-WGP-NEXT:    s_clause 0x1
2568; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2569; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2570; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2571; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2572; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2573; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2574; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2575; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2576; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2577; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2578; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2579; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2580; GFX10-WGP-NEXT:    buffer_gl0_inv
2581; GFX10-WGP-NEXT:    buffer_gl1_inv
2582; GFX10-WGP-NEXT:    s_endpgm
2583;
2584; GFX10-CU-LABEL: flat_agent_acquire_acquire_cmpxchg:
2585; GFX10-CU:       ; %bb.0: ; %entry
2586; GFX10-CU-NEXT:    s_clause 0x1
2587; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2588; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2589; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2590; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2591; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2592; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2593; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2594; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2595; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2596; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2597; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2598; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2599; GFX10-CU-NEXT:    buffer_gl0_inv
2600; GFX10-CU-NEXT:    buffer_gl1_inv
2601; GFX10-CU-NEXT:    s_endpgm
2602;
2603; SKIP-CACHE-INV-LABEL: flat_agent_acquire_acquire_cmpxchg:
2604; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2605; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2606; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2607; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2608; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2609; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2610; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2611; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2612; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2613; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2614; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2615; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2616; SKIP-CACHE-INV-NEXT:    s_endpgm
2617;
2618; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_acquire_cmpxchg:
2619; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2620; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2621; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2622; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2623; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2624; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2625; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2626; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2627; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2628; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2629;
2630; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_acquire_cmpxchg:
2631; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2632; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2633; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2634; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2635; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2636; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2637; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2638; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2639; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2640; GFX90A-TGSPLIT-NEXT:    s_endpgm
2641;
2642; GFX940-NOTTGSPLIT-LABEL: flat_agent_acquire_acquire_cmpxchg:
2643; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2644; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2645; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2646; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2647; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2648; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2649; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2650; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2651; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
2652; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2653;
2654; GFX940-TGSPLIT-LABEL: flat_agent_acquire_acquire_cmpxchg:
2655; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2656; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2657; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2658; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2659; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2660; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2661; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2662; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2663; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
2664; GFX940-TGSPLIT-NEXT:    s_endpgm
2665    i32* %out, i32 %in, i32 %old) {
2666entry:
2667  %gep = getelementptr i32, i32* %out, i32 4
2668  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire acquire
2669  ret void
2670}
2671
2672define amdgpu_kernel void @flat_agent_release_acquire_cmpxchg(
2673; GFX7-LABEL: flat_agent_release_acquire_cmpxchg:
2674; GFX7:       ; %bb.0: ; %entry
2675; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2676; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2677; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2678; GFX7-NEXT:    s_add_u32 s0, s0, 16
2679; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2680; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2681; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2682; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2683; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2684; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2685; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2686; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2687; GFX7-NEXT:    buffer_wbinvl1_vol
2688; GFX7-NEXT:    s_endpgm
2689;
2690; GFX10-WGP-LABEL: flat_agent_release_acquire_cmpxchg:
2691; GFX10-WGP:       ; %bb.0: ; %entry
2692; GFX10-WGP-NEXT:    s_clause 0x1
2693; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2694; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2695; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2696; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2697; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2698; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2699; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2700; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2701; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2702; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2703; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2704; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2705; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2706; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2707; GFX10-WGP-NEXT:    buffer_gl0_inv
2708; GFX10-WGP-NEXT:    buffer_gl1_inv
2709; GFX10-WGP-NEXT:    s_endpgm
2710;
2711; GFX10-CU-LABEL: flat_agent_release_acquire_cmpxchg:
2712; GFX10-CU:       ; %bb.0: ; %entry
2713; GFX10-CU-NEXT:    s_clause 0x1
2714; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2715; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2716; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2717; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2718; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2719; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2720; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2721; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2722; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2723; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2724; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2725; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2726; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2727; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2728; GFX10-CU-NEXT:    buffer_gl0_inv
2729; GFX10-CU-NEXT:    buffer_gl1_inv
2730; GFX10-CU-NEXT:    s_endpgm
2731;
2732; SKIP-CACHE-INV-LABEL: flat_agent_release_acquire_cmpxchg:
2733; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2734; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2735; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2736; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2737; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2738; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2739; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2740; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2741; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2742; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2743; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2744; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2745; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2746; SKIP-CACHE-INV-NEXT:    s_endpgm
2747;
2748; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_acquire_cmpxchg:
2749; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2750; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2751; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2752; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2753; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2754; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2755; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2756; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2757; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2758; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2759; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2760;
2761; GFX90A-TGSPLIT-LABEL: flat_agent_release_acquire_cmpxchg:
2762; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2763; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2764; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2765; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2766; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2767; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2768; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2769; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2770; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2771; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2772; GFX90A-TGSPLIT-NEXT:    s_endpgm
2773;
2774; GFX940-NOTTGSPLIT-LABEL: flat_agent_release_acquire_cmpxchg:
2775; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2776; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2777; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2778; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2779; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2780; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2781; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
2782; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2783; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2784; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2785; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
2786; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2787;
2788; GFX940-TGSPLIT-LABEL: flat_agent_release_acquire_cmpxchg:
2789; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2790; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2791; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2792; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2793; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2794; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2795; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
2796; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2797; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2798; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2799; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
2800; GFX940-TGSPLIT-NEXT:    s_endpgm
2801    i32* %out, i32 %in, i32 %old) {
2802entry:
2803  %gep = getelementptr i32, i32* %out, i32 4
2804  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release acquire
2805  ret void
2806}
2807
2808define amdgpu_kernel void @flat_agent_acq_rel_acquire_cmpxchg(
2809; GFX7-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
2810; GFX7:       ; %bb.0: ; %entry
2811; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2812; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2813; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2814; GFX7-NEXT:    s_add_u32 s0, s0, 16
2815; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2816; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2817; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2818; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2819; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2820; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2821; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2822; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2823; GFX7-NEXT:    buffer_wbinvl1_vol
2824; GFX7-NEXT:    s_endpgm
2825;
2826; GFX10-WGP-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
2827; GFX10-WGP:       ; %bb.0: ; %entry
2828; GFX10-WGP-NEXT:    s_clause 0x1
2829; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2830; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2831; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2832; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2833; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2834; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2835; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2836; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2837; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2838; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2839; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2840; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2841; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2842; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2843; GFX10-WGP-NEXT:    buffer_gl0_inv
2844; GFX10-WGP-NEXT:    buffer_gl1_inv
2845; GFX10-WGP-NEXT:    s_endpgm
2846;
2847; GFX10-CU-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
2848; GFX10-CU:       ; %bb.0: ; %entry
2849; GFX10-CU-NEXT:    s_clause 0x1
2850; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2851; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2852; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2853; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2854; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2855; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2856; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2857; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2858; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2859; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2860; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2861; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2862; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2863; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2864; GFX10-CU-NEXT:    buffer_gl0_inv
2865; GFX10-CU-NEXT:    buffer_gl1_inv
2866; GFX10-CU-NEXT:    s_endpgm
2867;
2868; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
2869; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2870; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2871; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2872; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2873; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2874; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2875; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2876; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2877; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2878; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2879; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2880; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2881; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2882; SKIP-CACHE-INV-NEXT:    s_endpgm
2883;
2884; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
2885; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2886; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2887; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2888; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2889; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2890; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2891; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2892; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2893; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2894; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2895; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2896;
2897; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
2898; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2899; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2900; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2901; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2902; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2903; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2904; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2905; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2906; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2907; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2908; GFX90A-TGSPLIT-NEXT:    s_endpgm
2909;
2910; GFX940-NOTTGSPLIT-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
2911; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2912; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2913; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2914; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2915; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2916; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2917; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
2918; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2919; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2920; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2921; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
2922; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2923;
2924; GFX940-TGSPLIT-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
2925; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2926; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2927; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2928; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2929; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2930; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2931; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
2932; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2933; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2934; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2935; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
2936; GFX940-TGSPLIT-NEXT:    s_endpgm
2937    i32* %out, i32 %in, i32 %old) {
2938entry:
2939  %gep = getelementptr i32, i32* %out, i32 4
2940  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel acquire
2941  ret void
2942}
2943
2944define amdgpu_kernel void @flat_agent_seq_cst_acquire_cmpxchg(
2945; GFX7-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
2946; GFX7:       ; %bb.0: ; %entry
2947; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2948; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2949; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2950; GFX7-NEXT:    s_add_u32 s0, s0, 16
2951; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2952; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2953; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2954; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2955; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2956; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2957; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2958; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2959; GFX7-NEXT:    buffer_wbinvl1_vol
2960; GFX7-NEXT:    s_endpgm
2961;
2962; GFX10-WGP-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
2963; GFX10-WGP:       ; %bb.0: ; %entry
2964; GFX10-WGP-NEXT:    s_clause 0x1
2965; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2966; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2967; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2968; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2969; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2970; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2971; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2972; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2973; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2974; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2975; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2976; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2977; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2978; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2979; GFX10-WGP-NEXT:    buffer_gl0_inv
2980; GFX10-WGP-NEXT:    buffer_gl1_inv
2981; GFX10-WGP-NEXT:    s_endpgm
2982;
2983; GFX10-CU-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
2984; GFX10-CU:       ; %bb.0: ; %entry
2985; GFX10-CU-NEXT:    s_clause 0x1
2986; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2987; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2988; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2989; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2990; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2991; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2992; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2993; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2994; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2995; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2996; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2997; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2998; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2999; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3000; GFX10-CU-NEXT:    buffer_gl0_inv
3001; GFX10-CU-NEXT:    buffer_gl1_inv
3002; GFX10-CU-NEXT:    s_endpgm
3003;
3004; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
3005; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3006; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3007; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3008; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3009; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3010; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3011; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3012; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3013; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3014; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3015; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3016; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3017; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3018; SKIP-CACHE-INV-NEXT:    s_endpgm
3019;
3020; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
3021; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3022; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3023; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3024; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3025; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3026; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3027; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3028; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3029; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3030; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3031; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3032;
3033; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
3034; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3035; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3036; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3037; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3038; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3039; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3040; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3041; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3042; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3043; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3044; GFX90A-TGSPLIT-NEXT:    s_endpgm
3045;
3046; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
3047; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3048; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3049; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3050; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3051; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3052; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3053; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
3054; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3055; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3056; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3057; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
3058; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3059;
3060; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
3061; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3062; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3063; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3064; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3065; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3066; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3067; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
3068; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3069; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3070; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3071; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
3072; GFX940-TGSPLIT-NEXT:    s_endpgm
3073    i32* %out, i32 %in, i32 %old) {
3074entry:
3075  %gep = getelementptr i32, i32* %out, i32 4
3076  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst acquire
3077  ret void
3078}
3079
3080define amdgpu_kernel void @flat_agent_monotonic_seq_cst_cmpxchg(
3081; GFX7-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
3082; GFX7:       ; %bb.0: ; %entry
3083; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3084; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3085; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3086; GFX7-NEXT:    s_add_u32 s0, s0, 16
3087; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3088; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3089; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3090; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3091; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3092; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3093; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3094; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3095; GFX7-NEXT:    buffer_wbinvl1_vol
3096; GFX7-NEXT:    s_endpgm
3097;
3098; GFX10-WGP-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
3099; GFX10-WGP:       ; %bb.0: ; %entry
3100; GFX10-WGP-NEXT:    s_clause 0x1
3101; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3102; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3103; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3104; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3105; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3106; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3107; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3108; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3109; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3110; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3111; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3112; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3113; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3114; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3115; GFX10-WGP-NEXT:    buffer_gl0_inv
3116; GFX10-WGP-NEXT:    buffer_gl1_inv
3117; GFX10-WGP-NEXT:    s_endpgm
3118;
3119; GFX10-CU-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
3120; GFX10-CU:       ; %bb.0: ; %entry
3121; GFX10-CU-NEXT:    s_clause 0x1
3122; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3123; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3124; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3125; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3126; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3127; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3128; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3129; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3130; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3131; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3132; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3133; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3134; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3135; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3136; GFX10-CU-NEXT:    buffer_gl0_inv
3137; GFX10-CU-NEXT:    buffer_gl1_inv
3138; GFX10-CU-NEXT:    s_endpgm
3139;
3140; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
3141; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3142; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3143; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3144; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3145; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3146; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3147; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3148; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3149; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3150; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3151; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3152; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3153; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3154; SKIP-CACHE-INV-NEXT:    s_endpgm
3155;
3156; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
3157; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3158; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3159; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3160; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3161; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3162; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3163; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3164; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3165; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3166; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3167; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3168;
3169; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
3170; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3171; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3172; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3173; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3174; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3175; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3176; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3177; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3178; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3179; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3180; GFX90A-TGSPLIT-NEXT:    s_endpgm
3181;
3182; GFX940-NOTTGSPLIT-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
3183; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3184; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3185; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3186; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3187; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3188; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3189; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
3190; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3191; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3192; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3193; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
3194; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3195;
3196; GFX940-TGSPLIT-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
3197; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3198; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3199; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3200; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3201; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3202; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3203; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
3204; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3205; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3206; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3207; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
3208; GFX940-TGSPLIT-NEXT:    s_endpgm
3209    i32* %out, i32 %in, i32 %old) {
3210entry:
3211  %gep = getelementptr i32, i32* %out, i32 4
3212  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") monotonic seq_cst
3213  ret void
3214}
3215
3216define amdgpu_kernel void @flat_agent_acquire_seq_cst_cmpxchg(
3217; GFX7-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
3218; GFX7:       ; %bb.0: ; %entry
3219; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3220; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3221; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3222; GFX7-NEXT:    s_add_u32 s0, s0, 16
3223; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3224; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3225; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3226; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3227; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3228; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3229; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3230; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3231; GFX7-NEXT:    buffer_wbinvl1_vol
3232; GFX7-NEXT:    s_endpgm
3233;
3234; GFX10-WGP-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
3235; GFX10-WGP:       ; %bb.0: ; %entry
3236; GFX10-WGP-NEXT:    s_clause 0x1
3237; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3238; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3239; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3240; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3241; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3242; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3243; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3244; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3245; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3246; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3247; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3248; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3249; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3250; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3251; GFX10-WGP-NEXT:    buffer_gl0_inv
3252; GFX10-WGP-NEXT:    buffer_gl1_inv
3253; GFX10-WGP-NEXT:    s_endpgm
3254;
3255; GFX10-CU-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
3256; GFX10-CU:       ; %bb.0: ; %entry
3257; GFX10-CU-NEXT:    s_clause 0x1
3258; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3259; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3260; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3261; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3262; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3263; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3264; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3265; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3266; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3267; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3268; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3269; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3270; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3271; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3272; GFX10-CU-NEXT:    buffer_gl0_inv
3273; GFX10-CU-NEXT:    buffer_gl1_inv
3274; GFX10-CU-NEXT:    s_endpgm
3275;
3276; SKIP-CACHE-INV-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
3277; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3278; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3279; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3280; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3281; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3282; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3283; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3284; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3285; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3286; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3287; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3288; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3289; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3290; SKIP-CACHE-INV-NEXT:    s_endpgm
3291;
3292; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
3293; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3294; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3295; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3296; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3297; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3298; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3299; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3300; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3301; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3302; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3303; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3304;
3305; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
3306; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3307; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3308; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3309; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3310; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3311; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3312; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3313; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3314; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3315; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3316; GFX90A-TGSPLIT-NEXT:    s_endpgm
3317;
3318; GFX940-NOTTGSPLIT-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
3319; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3320; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3321; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3322; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3323; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3324; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3325; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
3326; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3327; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3328; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3329; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
3330; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3331;
3332; GFX940-TGSPLIT-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
3333; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3334; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3335; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3336; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3337; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3338; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3339; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
3340; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3341; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3342; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3343; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
3344; GFX940-TGSPLIT-NEXT:    s_endpgm
3345    i32* %out, i32 %in, i32 %old) {
3346entry:
3347  %gep = getelementptr i32, i32* %out, i32 4
3348  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire seq_cst
3349  ret void
3350}
3351
3352define amdgpu_kernel void @flat_agent_release_seq_cst_cmpxchg(
3353; GFX7-LABEL: flat_agent_release_seq_cst_cmpxchg:
3354; GFX7:       ; %bb.0: ; %entry
3355; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3356; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3357; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3358; GFX7-NEXT:    s_add_u32 s0, s0, 16
3359; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3360; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3361; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3362; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3363; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3364; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3365; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3366; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3367; GFX7-NEXT:    buffer_wbinvl1_vol
3368; GFX7-NEXT:    s_endpgm
3369;
3370; GFX10-WGP-LABEL: flat_agent_release_seq_cst_cmpxchg:
3371; GFX10-WGP:       ; %bb.0: ; %entry
3372; GFX10-WGP-NEXT:    s_clause 0x1
3373; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3374; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3375; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3376; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3377; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3378; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3379; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3380; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3381; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3382; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3383; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3384; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3385; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3386; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3387; GFX10-WGP-NEXT:    buffer_gl0_inv
3388; GFX10-WGP-NEXT:    buffer_gl1_inv
3389; GFX10-WGP-NEXT:    s_endpgm
3390;
3391; GFX10-CU-LABEL: flat_agent_release_seq_cst_cmpxchg:
3392; GFX10-CU:       ; %bb.0: ; %entry
3393; GFX10-CU-NEXT:    s_clause 0x1
3394; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3395; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3396; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3397; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3398; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3399; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3400; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3401; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3402; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3403; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3404; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3405; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3406; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3407; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3408; GFX10-CU-NEXT:    buffer_gl0_inv
3409; GFX10-CU-NEXT:    buffer_gl1_inv
3410; GFX10-CU-NEXT:    s_endpgm
3411;
3412; SKIP-CACHE-INV-LABEL: flat_agent_release_seq_cst_cmpxchg:
3413; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3414; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3415; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3416; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3417; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3418; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3419; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3420; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3421; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3422; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3423; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3424; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3425; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3426; SKIP-CACHE-INV-NEXT:    s_endpgm
3427;
3428; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_seq_cst_cmpxchg:
3429; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3430; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3431; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3432; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3433; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3434; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3435; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3436; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3437; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3438; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3439; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3440;
3441; GFX90A-TGSPLIT-LABEL: flat_agent_release_seq_cst_cmpxchg:
3442; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3443; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3444; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3445; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3446; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3447; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3448; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3449; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3450; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3451; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3452; GFX90A-TGSPLIT-NEXT:    s_endpgm
3453;
3454; GFX940-NOTTGSPLIT-LABEL: flat_agent_release_seq_cst_cmpxchg:
3455; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3456; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3457; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3458; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3459; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3460; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3461; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
3462; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3463; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3464; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3465; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
3466; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3467;
3468; GFX940-TGSPLIT-LABEL: flat_agent_release_seq_cst_cmpxchg:
3469; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3470; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3471; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3472; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3473; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3474; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3475; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
3476; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3477; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3478; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3479; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
3480; GFX940-TGSPLIT-NEXT:    s_endpgm
3481    i32* %out, i32 %in, i32 %old) {
3482entry:
3483  %gep = getelementptr i32, i32* %out, i32 4
3484  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release seq_cst
3485  ret void
3486}
3487
3488define amdgpu_kernel void @flat_agent_acq_rel_seq_cst_cmpxchg(
3489; GFX7-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
3490; GFX7:       ; %bb.0: ; %entry
3491; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3492; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3493; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3494; GFX7-NEXT:    s_add_u32 s0, s0, 16
3495; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3496; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3497; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3498; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3499; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3500; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3501; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3502; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3503; GFX7-NEXT:    buffer_wbinvl1_vol
3504; GFX7-NEXT:    s_endpgm
3505;
3506; GFX10-WGP-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
3507; GFX10-WGP:       ; %bb.0: ; %entry
3508; GFX10-WGP-NEXT:    s_clause 0x1
3509; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3510; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3511; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3512; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3513; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3514; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3515; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3516; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3517; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3518; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3519; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3520; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3521; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3522; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3523; GFX10-WGP-NEXT:    buffer_gl0_inv
3524; GFX10-WGP-NEXT:    buffer_gl1_inv
3525; GFX10-WGP-NEXT:    s_endpgm
3526;
3527; GFX10-CU-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
3528; GFX10-CU:       ; %bb.0: ; %entry
3529; GFX10-CU-NEXT:    s_clause 0x1
3530; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3531; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3532; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3533; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3534; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3535; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3536; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3537; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3538; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3539; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3540; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3541; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3542; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3543; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3544; GFX10-CU-NEXT:    buffer_gl0_inv
3545; GFX10-CU-NEXT:    buffer_gl1_inv
3546; GFX10-CU-NEXT:    s_endpgm
3547;
3548; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
3549; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3550; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3551; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3552; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3553; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3554; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3555; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3556; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3557; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3558; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3559; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3560; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3561; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3562; SKIP-CACHE-INV-NEXT:    s_endpgm
3563;
3564; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
3565; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3566; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3567; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3568; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3569; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3570; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3571; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3572; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3573; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3574; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3575; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3576;
3577; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
3578; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3579; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3580; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3581; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3582; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3583; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3584; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3585; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3586; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3587; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3588; GFX90A-TGSPLIT-NEXT:    s_endpgm
3589;
3590; GFX940-NOTTGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
3591; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3592; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3593; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3594; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3595; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3596; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3597; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
3598; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3599; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3600; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3601; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
3602; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3603;
3604; GFX940-TGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
3605; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3606; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3607; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3608; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3609; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3610; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3611; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
3612; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3613; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3614; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3615; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
3616; GFX940-TGSPLIT-NEXT:    s_endpgm
3617    i32* %out, i32 %in, i32 %old) {
3618entry:
3619  %gep = getelementptr i32, i32* %out, i32 4
3620  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel seq_cst
3621  ret void
3622}
3623
3624define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_cmpxchg(
3625; GFX7-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
3626; GFX7:       ; %bb.0: ; %entry
3627; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3628; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3629; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3630; GFX7-NEXT:    s_add_u32 s0, s0, 16
3631; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3632; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3633; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3634; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3635; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3636; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3637; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3638; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3639; GFX7-NEXT:    buffer_wbinvl1_vol
3640; GFX7-NEXT:    s_endpgm
3641;
3642; GFX10-WGP-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
3643; GFX10-WGP:       ; %bb.0: ; %entry
3644; GFX10-WGP-NEXT:    s_clause 0x1
3645; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3646; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3647; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3648; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3649; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3650; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3651; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3652; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3653; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3654; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3655; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3656; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3657; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3658; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3659; GFX10-WGP-NEXT:    buffer_gl0_inv
3660; GFX10-WGP-NEXT:    buffer_gl1_inv
3661; GFX10-WGP-NEXT:    s_endpgm
3662;
3663; GFX10-CU-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
3664; GFX10-CU:       ; %bb.0: ; %entry
3665; GFX10-CU-NEXT:    s_clause 0x1
3666; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3667; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3668; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3669; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3670; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3671; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3672; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3673; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3674; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3675; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3676; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3677; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3678; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3679; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3680; GFX10-CU-NEXT:    buffer_gl0_inv
3681; GFX10-CU-NEXT:    buffer_gl1_inv
3682; GFX10-CU-NEXT:    s_endpgm
3683;
3684; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
3685; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3686; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3687; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3688; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3689; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3690; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3691; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3692; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3693; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3694; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3695; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3696; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3697; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3698; SKIP-CACHE-INV-NEXT:    s_endpgm
3699;
3700; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
3701; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3702; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3703; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3704; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3705; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3706; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3707; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3708; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3709; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3710; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3711; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3712;
3713; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
3714; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3715; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3716; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3717; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3718; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3719; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3720; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3721; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3722; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3723; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3724; GFX90A-TGSPLIT-NEXT:    s_endpgm
3725;
3726; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
3727; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3728; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3729; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3730; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3731; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3732; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3733; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
3734; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3735; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3736; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3737; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
3738; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3739;
3740; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
3741; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3742; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3743; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3744; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3745; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3746; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3747; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
3748; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3749; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3750; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3751; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
3752; GFX940-TGSPLIT-NEXT:    s_endpgm
3753    i32* %out, i32 %in, i32 %old) {
3754entry:
3755  %gep = getelementptr i32, i32* %out, i32 4
3756  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
3757  ret void
3758}
3759
3760define amdgpu_kernel void @flat_agent_monotonic_monotonic_ret_cmpxchg(
3761; GFX7-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
3762; GFX7:       ; %bb.0: ; %entry
3763; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3764; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3765; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3766; GFX7-NEXT:    s_add_u32 s4, s0, 16
3767; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3768; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3769; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3770; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3771; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3772; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3773; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3774; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3775; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3776; GFX7-NEXT:    flat_store_dword v[0:1], v2
3777; GFX7-NEXT:    s_endpgm
3778;
3779; GFX10-WGP-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
3780; GFX10-WGP:       ; %bb.0: ; %entry
3781; GFX10-WGP-NEXT:    s_clause 0x1
3782; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3783; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3784; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3785; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3786; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3787; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3788; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3789; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
3790; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3791; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3792; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3793; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3794; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3795; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3796; GFX10-WGP-NEXT:    s_endpgm
3797;
3798; GFX10-CU-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
3799; GFX10-CU:       ; %bb.0: ; %entry
3800; GFX10-CU-NEXT:    s_clause 0x1
3801; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3802; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3803; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3804; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3805; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3806; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3807; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3808; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3809; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3810; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3811; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3812; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3813; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3814; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3815; GFX10-CU-NEXT:    s_endpgm
3816;
3817; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
3818; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3819; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3820; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3821; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3822; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3823; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3824; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3825; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3826; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3827; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3828; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3829; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3830; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3831; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3832; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3833; SKIP-CACHE-INV-NEXT:    s_endpgm
3834;
3835; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
3836; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3837; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3838; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3839; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3840; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3841; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3842; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3843; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3844; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3845; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3846;
3847; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
3848; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3849; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3850; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3851; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3852; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3853; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3854; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3855; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3856; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3857; GFX90A-TGSPLIT-NEXT:    s_endpgm
3858;
3859; GFX940-NOTTGSPLIT-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
3860; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3861; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3862; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3863; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3864; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3865; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3866; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
3867; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3868; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3869; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3870;
3871; GFX940-TGSPLIT-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
3872; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3873; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3874; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3875; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3876; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3877; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3878; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
3879; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3880; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3881; GFX940-TGSPLIT-NEXT:    s_endpgm
3882    i32* %out, i32 %in, i32 %old) {
3883entry:
3884  %gep = getelementptr i32, i32* %out, i32 4
3885  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") monotonic monotonic
3886  %val0 = extractvalue { i32, i1 } %val, 0
3887  store i32 %val0, i32* %out, align 4
3888  ret void
3889}
3890
3891define amdgpu_kernel void @flat_agent_acquire_monotonic_ret_cmpxchg(
3892; GFX7-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
3893; GFX7:       ; %bb.0: ; %entry
3894; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3895; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3896; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3897; GFX7-NEXT:    s_add_u32 s4, s0, 16
3898; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3899; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3900; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3901; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3902; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3903; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3904; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3905; GFX7-NEXT:    buffer_wbinvl1_vol
3906; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3907; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3908; GFX7-NEXT:    flat_store_dword v[0:1], v2
3909; GFX7-NEXT:    s_endpgm
3910;
3911; GFX10-WGP-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
3912; GFX10-WGP:       ; %bb.0: ; %entry
3913; GFX10-WGP-NEXT:    s_clause 0x1
3914; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3915; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3916; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3917; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3918; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3919; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3920; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3921; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
3922; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3923; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3924; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3925; GFX10-WGP-NEXT:    buffer_gl0_inv
3926; GFX10-WGP-NEXT:    buffer_gl1_inv
3927; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3928; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3929; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3930; GFX10-WGP-NEXT:    s_endpgm
3931;
3932; GFX10-CU-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
3933; GFX10-CU:       ; %bb.0: ; %entry
3934; GFX10-CU-NEXT:    s_clause 0x1
3935; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3936; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3937; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3938; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3939; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3940; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3941; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3942; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3943; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3944; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3945; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3946; GFX10-CU-NEXT:    buffer_gl0_inv
3947; GFX10-CU-NEXT:    buffer_gl1_inv
3948; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3949; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3950; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3951; GFX10-CU-NEXT:    s_endpgm
3952;
3953; SKIP-CACHE-INV-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
3954; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3955; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3956; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3957; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3958; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3959; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3960; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3961; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3962; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3963; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3964; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3965; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3966; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3967; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3968; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3969; SKIP-CACHE-INV-NEXT:    s_endpgm
3970;
3971; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
3972; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3973; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3974; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3975; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3976; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3977; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3978; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3979; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3980; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3981; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3982; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3983;
3984; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
3985; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3986; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3987; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3988; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3989; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3990; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3991; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3992; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3993; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3994; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3995; GFX90A-TGSPLIT-NEXT:    s_endpgm
3996;
3997; GFX940-NOTTGSPLIT-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
3998; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3999; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4000; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4001; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4002; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4003; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4004; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4005; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4006; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
4007; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4008; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4009;
4010; GFX940-TGSPLIT-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
4011; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4012; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4013; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4014; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4015; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4016; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4017; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4018; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4019; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
4020; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4021; GFX940-TGSPLIT-NEXT:    s_endpgm
4022    i32* %out, i32 %in, i32 %old) {
4023entry:
4024  %gep = getelementptr i32, i32* %out, i32 4
4025  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire monotonic
4026  %val0 = extractvalue { i32, i1 } %val, 0
4027  store i32 %val0, i32* %out, align 4
4028  ret void
4029}
4030
4031define amdgpu_kernel void @flat_agent_release_monotonic_ret_cmpxchg(
4032; GFX7-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
4033; GFX7:       ; %bb.0: ; %entry
4034; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4035; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4036; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4037; GFX7-NEXT:    s_add_u32 s4, s0, 16
4038; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4039; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4040; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4041; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4042; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4043; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4044; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4045; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4046; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4047; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4048; GFX7-NEXT:    flat_store_dword v[0:1], v2
4049; GFX7-NEXT:    s_endpgm
4050;
4051; GFX10-WGP-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
4052; GFX10-WGP:       ; %bb.0: ; %entry
4053; GFX10-WGP-NEXT:    s_clause 0x1
4054; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4055; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4056; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4057; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4058; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4059; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4060; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4061; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4062; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4063; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4064; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4065; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4066; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4067; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4068; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4069; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4070; GFX10-WGP-NEXT:    s_endpgm
4071;
4072; GFX10-CU-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
4073; GFX10-CU:       ; %bb.0: ; %entry
4074; GFX10-CU-NEXT:    s_clause 0x1
4075; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4076; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4077; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4078; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4079; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4080; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4081; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4082; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4083; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4084; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4085; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4086; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4087; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4088; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4089; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4090; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4091; GFX10-CU-NEXT:    s_endpgm
4092;
4093; SKIP-CACHE-INV-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
4094; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4095; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4096; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4097; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4098; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4099; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4100; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4101; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4102; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4103; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4104; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4105; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4106; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4107; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4108; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4109; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4110; SKIP-CACHE-INV-NEXT:    s_endpgm
4111;
4112; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
4113; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4114; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4115; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4116; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4117; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4118; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4119; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4120; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4121; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4122; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4123; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4124;
4125; GFX90A-TGSPLIT-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
4126; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4127; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4128; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4129; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4130; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4131; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4132; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4133; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4134; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4135; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4136; GFX90A-TGSPLIT-NEXT:    s_endpgm
4137;
4138; GFX940-NOTTGSPLIT-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
4139; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4140; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4141; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4142; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4143; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4144; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4145; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
4146; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4147; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4148; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4149; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4150; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4151;
4152; GFX940-TGSPLIT-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
4153; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4154; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4155; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4156; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4157; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4158; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4159; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
4160; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4161; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4162; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4163; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4164; GFX940-TGSPLIT-NEXT:    s_endpgm
4165    i32* %out, i32 %in, i32 %old) {
4166entry:
4167  %gep = getelementptr i32, i32* %out, i32 4
4168  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release monotonic
4169  %val0 = extractvalue { i32, i1 } %val, 0
4170  store i32 %val0, i32* %out, align 4
4171  ret void
4172}
4173
4174define amdgpu_kernel void @flat_agent_acq_rel_monotonic_ret_cmpxchg(
4175; GFX7-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
4176; GFX7:       ; %bb.0: ; %entry
4177; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4178; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4179; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4180; GFX7-NEXT:    s_add_u32 s4, s0, 16
4181; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4182; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4183; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4184; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4185; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4186; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4187; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4188; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4189; GFX7-NEXT:    buffer_wbinvl1_vol
4190; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4191; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4192; GFX7-NEXT:    flat_store_dword v[0:1], v2
4193; GFX7-NEXT:    s_endpgm
4194;
4195; GFX10-WGP-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
4196; GFX10-WGP:       ; %bb.0: ; %entry
4197; GFX10-WGP-NEXT:    s_clause 0x1
4198; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4199; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4200; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4201; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4202; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4203; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4204; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4205; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4206; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4207; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4208; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4209; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4210; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4211; GFX10-WGP-NEXT:    buffer_gl0_inv
4212; GFX10-WGP-NEXT:    buffer_gl1_inv
4213; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4214; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4215; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4216; GFX10-WGP-NEXT:    s_endpgm
4217;
4218; GFX10-CU-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
4219; GFX10-CU:       ; %bb.0: ; %entry
4220; GFX10-CU-NEXT:    s_clause 0x1
4221; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4222; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4223; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4224; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4225; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4226; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4227; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4228; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4229; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4230; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4231; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4232; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4233; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4234; GFX10-CU-NEXT:    buffer_gl0_inv
4235; GFX10-CU-NEXT:    buffer_gl1_inv
4236; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4237; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4238; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4239; GFX10-CU-NEXT:    s_endpgm
4240;
4241; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
4242; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4243; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4244; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4245; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4246; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4247; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4248; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4249; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4250; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4251; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4252; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4253; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4254; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4255; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4256; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4257; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4258; SKIP-CACHE-INV-NEXT:    s_endpgm
4259;
4260; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
4261; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4262; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4263; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4264; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4265; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4266; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4267; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4268; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4269; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4270; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4271; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4272; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4273;
4274; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
4275; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4276; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4277; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4278; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4279; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4280; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4281; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4282; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4283; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4284; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4285; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4286; GFX90A-TGSPLIT-NEXT:    s_endpgm
4287;
4288; GFX940-NOTTGSPLIT-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
4289; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4290; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4291; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4292; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4293; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4294; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4295; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
4296; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4297; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4298; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4299; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
4300; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4301; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4302;
4303; GFX940-TGSPLIT-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
4304; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4305; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4306; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4307; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4308; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4309; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4310; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
4311; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4312; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4313; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4314; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
4315; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4316; GFX940-TGSPLIT-NEXT:    s_endpgm
4317    i32* %out, i32 %in, i32 %old) {
4318entry:
4319  %gep = getelementptr i32, i32* %out, i32 4
4320  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel monotonic
4321  %val0 = extractvalue { i32, i1 } %val, 0
4322  store i32 %val0, i32* %out, align 4
4323  ret void
4324}
4325
4326define amdgpu_kernel void @flat_agent_seq_cst_monotonic_ret_cmpxchg(
4327; GFX7-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
4328; GFX7:       ; %bb.0: ; %entry
4329; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4330; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4331; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4332; GFX7-NEXT:    s_add_u32 s4, s0, 16
4333; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4334; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4335; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4336; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4337; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4338; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4339; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4340; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4341; GFX7-NEXT:    buffer_wbinvl1_vol
4342; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4343; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4344; GFX7-NEXT:    flat_store_dword v[0:1], v2
4345; GFX7-NEXT:    s_endpgm
4346;
4347; GFX10-WGP-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
4348; GFX10-WGP:       ; %bb.0: ; %entry
4349; GFX10-WGP-NEXT:    s_clause 0x1
4350; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4351; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4352; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4353; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4354; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4355; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4356; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4357; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4358; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4359; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4360; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4361; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4362; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4363; GFX10-WGP-NEXT:    buffer_gl0_inv
4364; GFX10-WGP-NEXT:    buffer_gl1_inv
4365; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4366; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4367; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4368; GFX10-WGP-NEXT:    s_endpgm
4369;
4370; GFX10-CU-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
4371; GFX10-CU:       ; %bb.0: ; %entry
4372; GFX10-CU-NEXT:    s_clause 0x1
4373; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4374; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4375; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4376; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4377; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4378; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4379; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4380; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4381; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4382; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4383; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4384; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4385; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4386; GFX10-CU-NEXT:    buffer_gl0_inv
4387; GFX10-CU-NEXT:    buffer_gl1_inv
4388; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4389; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4390; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4391; GFX10-CU-NEXT:    s_endpgm
4392;
4393; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
4394; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4395; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4396; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4397; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4398; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4399; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4400; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4401; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4402; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4403; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4404; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4405; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4406; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4407; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4408; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4409; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4410; SKIP-CACHE-INV-NEXT:    s_endpgm
4411;
4412; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
4413; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4414; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4415; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4416; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4417; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4418; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4419; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4420; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4421; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4422; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4423; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4424; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4425;
4426; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
4427; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4428; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4429; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4430; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4431; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4432; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4433; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4434; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4435; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4436; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4437; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4438; GFX90A-TGSPLIT-NEXT:    s_endpgm
4439;
4440; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
4441; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4442; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4443; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4444; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4445; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4446; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4447; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
4448; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4449; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4450; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4451; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
4452; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4453; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4454;
4455; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
4456; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4457; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4458; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4459; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4460; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4461; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4462; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
4463; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4464; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4465; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4466; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
4467; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4468; GFX940-TGSPLIT-NEXT:    s_endpgm
4469    i32* %out, i32 %in, i32 %old) {
4470entry:
4471  %gep = getelementptr i32, i32* %out, i32 4
4472  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst monotonic
4473  %val0 = extractvalue { i32, i1 } %val, 0
4474  store i32 %val0, i32* %out, align 4
4475  ret void
4476}
4477
4478define amdgpu_kernel void @flat_agent_monotonic_acquire_ret_cmpxchg(
4479; GFX7-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
4480; GFX7:       ; %bb.0: ; %entry
4481; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4482; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4483; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4484; GFX7-NEXT:    s_add_u32 s4, s0, 16
4485; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4486; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4487; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4488; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4489; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4490; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4491; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4492; GFX7-NEXT:    buffer_wbinvl1_vol
4493; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4494; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4495; GFX7-NEXT:    flat_store_dword v[0:1], v2
4496; GFX7-NEXT:    s_endpgm
4497;
4498; GFX10-WGP-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
4499; GFX10-WGP:       ; %bb.0: ; %entry
4500; GFX10-WGP-NEXT:    s_clause 0x1
4501; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4502; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4503; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4504; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4505; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4506; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4507; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4508; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4509; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4510; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4511; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4512; GFX10-WGP-NEXT:    buffer_gl0_inv
4513; GFX10-WGP-NEXT:    buffer_gl1_inv
4514; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4515; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4516; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4517; GFX10-WGP-NEXT:    s_endpgm
4518;
4519; GFX10-CU-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
4520; GFX10-CU:       ; %bb.0: ; %entry
4521; GFX10-CU-NEXT:    s_clause 0x1
4522; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4523; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4524; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4525; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4526; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4527; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4528; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4529; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4530; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4531; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4532; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4533; GFX10-CU-NEXT:    buffer_gl0_inv
4534; GFX10-CU-NEXT:    buffer_gl1_inv
4535; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4536; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4537; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4538; GFX10-CU-NEXT:    s_endpgm
4539;
4540; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
4541; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4542; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4543; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4544; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4545; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4546; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4547; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4548; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4549; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4550; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4551; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4552; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4553; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4554; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4555; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4556; SKIP-CACHE-INV-NEXT:    s_endpgm
4557;
4558; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
4559; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4560; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4561; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4562; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4563; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4564; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4565; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4566; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4567; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4568; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4569; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4570;
4571; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
4572; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4573; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4574; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4575; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4576; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4577; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4578; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4579; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4580; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4581; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4582; GFX90A-TGSPLIT-NEXT:    s_endpgm
4583;
4584; GFX940-NOTTGSPLIT-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
4585; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4586; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4587; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4588; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4589; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4590; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4591; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4592; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4593; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
4594; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4595; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4596;
4597; GFX940-TGSPLIT-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
4598; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4599; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4600; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4601; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4602; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4603; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4604; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4605; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4606; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
4607; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4608; GFX940-TGSPLIT-NEXT:    s_endpgm
4609    i32* %out, i32 %in, i32 %old) {
4610entry:
4611  %gep = getelementptr i32, i32* %out, i32 4
4612  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") monotonic acquire
4613  %val0 = extractvalue { i32, i1 } %val, 0
4614  store i32 %val0, i32* %out, align 4
4615  ret void
4616}
4617
4618define amdgpu_kernel void @flat_agent_acquire_acquire_ret_cmpxchg(
4619; GFX7-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
4620; GFX7:       ; %bb.0: ; %entry
4621; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4622; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4623; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4624; GFX7-NEXT:    s_add_u32 s4, s0, 16
4625; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4626; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4627; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4628; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4629; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4630; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4631; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4632; GFX7-NEXT:    buffer_wbinvl1_vol
4633; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4634; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4635; GFX7-NEXT:    flat_store_dword v[0:1], v2
4636; GFX7-NEXT:    s_endpgm
4637;
4638; GFX10-WGP-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
4639; GFX10-WGP:       ; %bb.0: ; %entry
4640; GFX10-WGP-NEXT:    s_clause 0x1
4641; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4642; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4643; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4644; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4645; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4646; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4647; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4648; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4649; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4650; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4651; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4652; GFX10-WGP-NEXT:    buffer_gl0_inv
4653; GFX10-WGP-NEXT:    buffer_gl1_inv
4654; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4655; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4656; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4657; GFX10-WGP-NEXT:    s_endpgm
4658;
4659; GFX10-CU-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
4660; GFX10-CU:       ; %bb.0: ; %entry
4661; GFX10-CU-NEXT:    s_clause 0x1
4662; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4663; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4664; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4665; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4666; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4667; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4668; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4669; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4670; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4671; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4672; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4673; GFX10-CU-NEXT:    buffer_gl0_inv
4674; GFX10-CU-NEXT:    buffer_gl1_inv
4675; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4676; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4677; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4678; GFX10-CU-NEXT:    s_endpgm
4679;
4680; SKIP-CACHE-INV-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
4681; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4682; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4683; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4684; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4685; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4686; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4687; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4688; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4689; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4690; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4691; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4692; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4693; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4694; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4695; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4696; SKIP-CACHE-INV-NEXT:    s_endpgm
4697;
4698; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
4699; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4700; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4701; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4702; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4703; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4704; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4705; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4706; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4707; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4708; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4709; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4710;
4711; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
4712; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4713; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4714; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4715; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4716; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4717; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4718; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4719; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4720; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4721; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4722; GFX90A-TGSPLIT-NEXT:    s_endpgm
4723;
4724; GFX940-NOTTGSPLIT-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
4725; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4726; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4727; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4728; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4729; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4730; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4731; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4732; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4733; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
4734; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4735; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4736;
4737; GFX940-TGSPLIT-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
4738; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4739; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4740; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4741; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4742; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4743; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4744; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4745; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4746; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
4747; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4748; GFX940-TGSPLIT-NEXT:    s_endpgm
4749    i32* %out, i32 %in, i32 %old) {
4750entry:
4751  %gep = getelementptr i32, i32* %out, i32 4
4752  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire acquire
4753  %val0 = extractvalue { i32, i1 } %val, 0
4754  store i32 %val0, i32* %out, align 4
4755  ret void
4756}
4757
4758define amdgpu_kernel void @flat_agent_release_acquire_ret_cmpxchg(
4759; GFX7-LABEL: flat_agent_release_acquire_ret_cmpxchg:
4760; GFX7:       ; %bb.0: ; %entry
4761; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4762; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4763; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4764; GFX7-NEXT:    s_add_u32 s4, s0, 16
4765; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4766; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4767; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4768; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4769; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4770; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4771; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4772; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4773; GFX7-NEXT:    buffer_wbinvl1_vol
4774; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4775; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4776; GFX7-NEXT:    flat_store_dword v[0:1], v2
4777; GFX7-NEXT:    s_endpgm
4778;
4779; GFX10-WGP-LABEL: flat_agent_release_acquire_ret_cmpxchg:
4780; GFX10-WGP:       ; %bb.0: ; %entry
4781; GFX10-WGP-NEXT:    s_clause 0x1
4782; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4783; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4784; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4785; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4786; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4787; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4788; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4789; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4790; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4791; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4792; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4793; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4794; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4795; GFX10-WGP-NEXT:    buffer_gl0_inv
4796; GFX10-WGP-NEXT:    buffer_gl1_inv
4797; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4798; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4799; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4800; GFX10-WGP-NEXT:    s_endpgm
4801;
4802; GFX10-CU-LABEL: flat_agent_release_acquire_ret_cmpxchg:
4803; GFX10-CU:       ; %bb.0: ; %entry
4804; GFX10-CU-NEXT:    s_clause 0x1
4805; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4806; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4807; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4808; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4809; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4810; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4811; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4812; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4813; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4814; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4815; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4816; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4817; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4818; GFX10-CU-NEXT:    buffer_gl0_inv
4819; GFX10-CU-NEXT:    buffer_gl1_inv
4820; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4821; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4822; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4823; GFX10-CU-NEXT:    s_endpgm
4824;
4825; SKIP-CACHE-INV-LABEL: flat_agent_release_acquire_ret_cmpxchg:
4826; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4827; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4828; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4829; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4830; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4831; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4832; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4833; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4834; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4835; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4836; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4837; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4838; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4839; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4840; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4841; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4842; SKIP-CACHE-INV-NEXT:    s_endpgm
4843;
4844; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_acquire_ret_cmpxchg:
4845; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4846; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4847; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4848; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4849; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4850; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4851; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4852; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4853; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4854; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4855; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4856; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4857;
4858; GFX90A-TGSPLIT-LABEL: flat_agent_release_acquire_ret_cmpxchg:
4859; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4860; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4861; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4862; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4863; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4864; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4865; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4866; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4867; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4868; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4869; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4870; GFX90A-TGSPLIT-NEXT:    s_endpgm
4871;
4872; GFX940-NOTTGSPLIT-LABEL: flat_agent_release_acquire_ret_cmpxchg:
4873; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4874; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4875; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4876; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4877; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4878; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4879; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
4880; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4881; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4882; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4883; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
4884; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4885; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4886;
4887; GFX940-TGSPLIT-LABEL: flat_agent_release_acquire_ret_cmpxchg:
4888; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4889; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4890; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4891; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4892; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4893; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4894; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
4895; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4896; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4897; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4898; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
4899; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4900; GFX940-TGSPLIT-NEXT:    s_endpgm
4901    i32* %out, i32 %in, i32 %old) {
4902entry:
4903  %gep = getelementptr i32, i32* %out, i32 4
4904  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release acquire
4905  %val0 = extractvalue { i32, i1 } %val, 0
4906  store i32 %val0, i32* %out, align 4
4907  ret void
4908}
4909
4910define amdgpu_kernel void @flat_agent_acq_rel_acquire_ret_cmpxchg(
4911; GFX7-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
4912; GFX7:       ; %bb.0: ; %entry
4913; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4914; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4915; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4916; GFX7-NEXT:    s_add_u32 s4, s0, 16
4917; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4918; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4919; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4920; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4921; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4922; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4923; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4924; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4925; GFX7-NEXT:    buffer_wbinvl1_vol
4926; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4927; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4928; GFX7-NEXT:    flat_store_dword v[0:1], v2
4929; GFX7-NEXT:    s_endpgm
4930;
4931; GFX10-WGP-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
4932; GFX10-WGP:       ; %bb.0: ; %entry
4933; GFX10-WGP-NEXT:    s_clause 0x1
4934; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4935; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4936; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4937; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4938; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4939; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4940; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4941; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4942; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4943; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4944; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4945; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4946; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4947; GFX10-WGP-NEXT:    buffer_gl0_inv
4948; GFX10-WGP-NEXT:    buffer_gl1_inv
4949; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4950; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4951; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4952; GFX10-WGP-NEXT:    s_endpgm
4953;
4954; GFX10-CU-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
4955; GFX10-CU:       ; %bb.0: ; %entry
4956; GFX10-CU-NEXT:    s_clause 0x1
4957; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4958; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4959; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4960; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4961; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4962; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4963; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4964; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4965; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4966; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4967; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4968; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4969; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4970; GFX10-CU-NEXT:    buffer_gl0_inv
4971; GFX10-CU-NEXT:    buffer_gl1_inv
4972; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4973; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4974; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4975; GFX10-CU-NEXT:    s_endpgm
4976;
4977; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
4978; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4979; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4980; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4981; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4982; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4983; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4984; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4985; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4986; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4987; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4988; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4989; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4990; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4991; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4992; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4993; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4994; SKIP-CACHE-INV-NEXT:    s_endpgm
4995;
4996; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
4997; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4998; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4999; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5000; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5001; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5002; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5003; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5004; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5005; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5006; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5007; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5008; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5009;
5010; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
5011; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5012; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5013; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5014; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5015; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5016; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5017; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5018; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5019; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5020; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5021; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5022; GFX90A-TGSPLIT-NEXT:    s_endpgm
5023;
5024; GFX940-NOTTGSPLIT-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
5025; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5026; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5027; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5028; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5029; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5030; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5031; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
5032; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5033; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5034; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5035; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
5036; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5037; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5038;
5039; GFX940-TGSPLIT-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
5040; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5041; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5042; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5043; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5044; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5045; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5046; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
5047; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5048; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5049; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5050; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
5051; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5052; GFX940-TGSPLIT-NEXT:    s_endpgm
5053    i32* %out, i32 %in, i32 %old) {
5054entry:
5055  %gep = getelementptr i32, i32* %out, i32 4
5056  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel acquire
5057  %val0 = extractvalue { i32, i1 } %val, 0
5058  store i32 %val0, i32* %out, align 4
5059  ret void
5060}
5061
5062define amdgpu_kernel void @flat_agent_seq_cst_acquire_ret_cmpxchg(
5063; GFX7-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
5064; GFX7:       ; %bb.0: ; %entry
5065; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5066; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5067; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5068; GFX7-NEXT:    s_add_u32 s4, s0, 16
5069; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5070; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5071; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5072; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5073; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5074; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5075; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5076; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5077; GFX7-NEXT:    buffer_wbinvl1_vol
5078; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5079; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5080; GFX7-NEXT:    flat_store_dword v[0:1], v2
5081; GFX7-NEXT:    s_endpgm
5082;
5083; GFX10-WGP-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
5084; GFX10-WGP:       ; %bb.0: ; %entry
5085; GFX10-WGP-NEXT:    s_clause 0x1
5086; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5087; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5088; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5089; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5090; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5091; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5092; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5093; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5094; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5095; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5096; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5097; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5098; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5099; GFX10-WGP-NEXT:    buffer_gl0_inv
5100; GFX10-WGP-NEXT:    buffer_gl1_inv
5101; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5102; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5103; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5104; GFX10-WGP-NEXT:    s_endpgm
5105;
5106; GFX10-CU-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
5107; GFX10-CU:       ; %bb.0: ; %entry
5108; GFX10-CU-NEXT:    s_clause 0x1
5109; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5110; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5111; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5112; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5113; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5114; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5115; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5116; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5117; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5118; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5119; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5120; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5121; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5122; GFX10-CU-NEXT:    buffer_gl0_inv
5123; GFX10-CU-NEXT:    buffer_gl1_inv
5124; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5125; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5126; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5127; GFX10-CU-NEXT:    s_endpgm
5128;
5129; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
5130; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5131; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5132; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5133; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5134; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5135; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5136; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5137; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5138; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5139; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5140; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5141; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5142; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5143; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5144; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5145; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5146; SKIP-CACHE-INV-NEXT:    s_endpgm
5147;
5148; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
5149; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5150; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5151; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5152; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5153; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5154; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5155; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5156; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5157; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5158; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5159; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5160; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5161;
5162; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
5163; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5164; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5165; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5166; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5167; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5168; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5169; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5170; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5171; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5172; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5173; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5174; GFX90A-TGSPLIT-NEXT:    s_endpgm
5175;
5176; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
5177; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5178; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5179; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5180; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5181; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5182; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5183; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
5184; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5185; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5186; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5187; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
5188; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5189; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5190;
5191; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
5192; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5193; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5194; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5195; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5196; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5197; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5198; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
5199; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5200; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5201; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5202; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
5203; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5204; GFX940-TGSPLIT-NEXT:    s_endpgm
5205    i32* %out, i32 %in, i32 %old) {
5206entry:
5207  %gep = getelementptr i32, i32* %out, i32 4
5208  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst acquire
5209  %val0 = extractvalue { i32, i1 } %val, 0
5210  store i32 %val0, i32* %out, align 4
5211  ret void
5212}
5213
5214define amdgpu_kernel void @flat_agent_monotonic_seq_cst_ret_cmpxchg(
5215; GFX7-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
5216; GFX7:       ; %bb.0: ; %entry
5217; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5218; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5219; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5220; GFX7-NEXT:    s_add_u32 s4, s0, 16
5221; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5222; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5223; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5224; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5225; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5226; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5227; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5228; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5229; GFX7-NEXT:    buffer_wbinvl1_vol
5230; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5231; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5232; GFX7-NEXT:    flat_store_dword v[0:1], v2
5233; GFX7-NEXT:    s_endpgm
5234;
5235; GFX10-WGP-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
5236; GFX10-WGP:       ; %bb.0: ; %entry
5237; GFX10-WGP-NEXT:    s_clause 0x1
5238; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5239; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5240; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5241; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5242; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5243; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5244; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5245; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5246; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5247; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5248; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5249; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5250; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5251; GFX10-WGP-NEXT:    buffer_gl0_inv
5252; GFX10-WGP-NEXT:    buffer_gl1_inv
5253; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5254; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5255; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5256; GFX10-WGP-NEXT:    s_endpgm
5257;
5258; GFX10-CU-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
5259; GFX10-CU:       ; %bb.0: ; %entry
5260; GFX10-CU-NEXT:    s_clause 0x1
5261; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5262; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5263; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5264; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5265; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5266; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5267; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5268; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5269; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5270; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5271; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5272; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5273; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5274; GFX10-CU-NEXT:    buffer_gl0_inv
5275; GFX10-CU-NEXT:    buffer_gl1_inv
5276; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5277; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5278; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5279; GFX10-CU-NEXT:    s_endpgm
5280;
5281; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
5282; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5283; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5284; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5285; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5286; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5287; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5288; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5289; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5290; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5291; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5292; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5293; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5294; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5295; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5296; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5297; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5298; SKIP-CACHE-INV-NEXT:    s_endpgm
5299;
5300; GFX90A-NOTTGSPLIT-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
5301; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5302; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5303; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5304; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5305; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5306; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5307; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5308; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5309; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5310; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5311; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5312; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5313;
5314; GFX90A-TGSPLIT-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
5315; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5316; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5317; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5318; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5319; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5320; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5321; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5322; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5323; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5324; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5325; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5326; GFX90A-TGSPLIT-NEXT:    s_endpgm
5327;
5328; GFX940-NOTTGSPLIT-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
5329; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5330; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5331; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5332; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5333; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5334; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5335; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
5336; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5337; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5338; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5339; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
5340; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5341; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5342;
5343; GFX940-TGSPLIT-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
5344; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5345; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5346; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5347; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5348; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5349; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5350; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
5351; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5352; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5353; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5354; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
5355; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5356; GFX940-TGSPLIT-NEXT:    s_endpgm
5357    i32* %out, i32 %in, i32 %old) {
5358entry:
5359  %gep = getelementptr i32, i32* %out, i32 4
5360  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") monotonic seq_cst
5361  %val0 = extractvalue { i32, i1 } %val, 0
5362  store i32 %val0, i32* %out, align 4
5363  ret void
5364}
5365
5366define amdgpu_kernel void @flat_agent_acquire_seq_cst_ret_cmpxchg(
5367; GFX7-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
5368; GFX7:       ; %bb.0: ; %entry
5369; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5370; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5371; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5372; GFX7-NEXT:    s_add_u32 s4, s0, 16
5373; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5374; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5375; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5376; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5377; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5378; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5379; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5380; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5381; GFX7-NEXT:    buffer_wbinvl1_vol
5382; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5383; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5384; GFX7-NEXT:    flat_store_dword v[0:1], v2
5385; GFX7-NEXT:    s_endpgm
5386;
5387; GFX10-WGP-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
5388; GFX10-WGP:       ; %bb.0: ; %entry
5389; GFX10-WGP-NEXT:    s_clause 0x1
5390; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5391; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5392; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5393; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5394; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5395; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5396; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5397; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5398; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5399; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5400; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5401; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5402; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5403; GFX10-WGP-NEXT:    buffer_gl0_inv
5404; GFX10-WGP-NEXT:    buffer_gl1_inv
5405; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5406; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5407; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5408; GFX10-WGP-NEXT:    s_endpgm
5409;
5410; GFX10-CU-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
5411; GFX10-CU:       ; %bb.0: ; %entry
5412; GFX10-CU-NEXT:    s_clause 0x1
5413; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5414; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5415; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5416; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5417; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5418; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5419; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5420; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5421; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5422; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5423; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5424; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5425; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5426; GFX10-CU-NEXT:    buffer_gl0_inv
5427; GFX10-CU-NEXT:    buffer_gl1_inv
5428; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5429; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5430; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5431; GFX10-CU-NEXT:    s_endpgm
5432;
5433; SKIP-CACHE-INV-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
5434; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5435; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5436; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5437; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5438; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5439; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5440; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5441; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5442; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5443; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5444; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5445; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5446; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5447; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5448; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5449; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5450; SKIP-CACHE-INV-NEXT:    s_endpgm
5451;
5452; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
5453; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5454; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5455; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5456; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5457; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5458; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5459; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5460; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5461; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5462; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5463; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5464; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5465;
5466; GFX90A-TGSPLIT-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
5467; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5468; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5469; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5470; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5471; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5472; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5473; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5474; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5475; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5476; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5477; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5478; GFX90A-TGSPLIT-NEXT:    s_endpgm
5479;
5480; GFX940-NOTTGSPLIT-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
5481; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5482; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5483; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5484; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5485; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5486; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5487; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
5488; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5489; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5490; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5491; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
5492; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5493; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5494;
5495; GFX940-TGSPLIT-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
5496; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5497; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5498; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5499; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5500; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5501; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5502; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
5503; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5504; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5505; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5506; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
5507; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5508; GFX940-TGSPLIT-NEXT:    s_endpgm
5509    i32* %out, i32 %in, i32 %old) {
5510entry:
5511  %gep = getelementptr i32, i32* %out, i32 4
5512  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire seq_cst
5513  %val0 = extractvalue { i32, i1 } %val, 0
5514  store i32 %val0, i32* %out, align 4
5515  ret void
5516}
5517
5518define amdgpu_kernel void @flat_agent_release_seq_cst_ret_cmpxchg(
5519; GFX7-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
5520; GFX7:       ; %bb.0: ; %entry
5521; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5522; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5523; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5524; GFX7-NEXT:    s_add_u32 s4, s0, 16
5525; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5526; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5527; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5528; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5529; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5530; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5531; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5532; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5533; GFX7-NEXT:    buffer_wbinvl1_vol
5534; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5535; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5536; GFX7-NEXT:    flat_store_dword v[0:1], v2
5537; GFX7-NEXT:    s_endpgm
5538;
5539; GFX10-WGP-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
5540; GFX10-WGP:       ; %bb.0: ; %entry
5541; GFX10-WGP-NEXT:    s_clause 0x1
5542; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5543; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5544; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5545; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5546; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5547; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5548; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5549; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5550; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5551; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5552; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5553; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5554; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5555; GFX10-WGP-NEXT:    buffer_gl0_inv
5556; GFX10-WGP-NEXT:    buffer_gl1_inv
5557; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5558; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5559; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5560; GFX10-WGP-NEXT:    s_endpgm
5561;
5562; GFX10-CU-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
5563; GFX10-CU:       ; %bb.0: ; %entry
5564; GFX10-CU-NEXT:    s_clause 0x1
5565; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5566; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5567; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5568; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5569; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5570; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5571; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5572; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5573; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5574; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5575; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5576; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5577; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5578; GFX10-CU-NEXT:    buffer_gl0_inv
5579; GFX10-CU-NEXT:    buffer_gl1_inv
5580; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5581; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5582; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5583; GFX10-CU-NEXT:    s_endpgm
5584;
5585; SKIP-CACHE-INV-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
5586; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5587; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5588; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5589; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5590; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5591; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5592; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5593; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5594; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5595; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5596; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5597; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5598; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5599; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5600; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5601; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5602; SKIP-CACHE-INV-NEXT:    s_endpgm
5603;
5604; GFX90A-NOTTGSPLIT-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
5605; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5606; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5607; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5608; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5609; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5610; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5611; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5612; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5613; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5614; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5615; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5616; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5617;
5618; GFX90A-TGSPLIT-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
5619; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5620; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5621; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5622; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5623; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5624; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5625; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5626; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5627; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5628; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5629; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5630; GFX90A-TGSPLIT-NEXT:    s_endpgm
5631;
5632; GFX940-NOTTGSPLIT-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
5633; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5634; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5635; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5636; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5637; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5638; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5639; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
5640; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5641; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5642; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5643; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
5644; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5645; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5646;
5647; GFX940-TGSPLIT-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
5648; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5649; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5650; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5651; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5652; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5653; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5654; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
5655; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5656; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5657; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5658; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
5659; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5660; GFX940-TGSPLIT-NEXT:    s_endpgm
5661    i32* %out, i32 %in, i32 %old) {
5662entry:
5663  %gep = getelementptr i32, i32* %out, i32 4
5664  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release seq_cst
5665  %val0 = extractvalue { i32, i1 } %val, 0
5666  store i32 %val0, i32* %out, align 4
5667  ret void
5668}
5669
5670define amdgpu_kernel void @flat_agent_acq_rel_seq_cst_ret_cmpxchg(
5671; GFX7-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
5672; GFX7:       ; %bb.0: ; %entry
5673; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5674; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5675; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5676; GFX7-NEXT:    s_add_u32 s4, s0, 16
5677; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5678; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5679; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5680; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5681; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5682; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5683; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5684; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5685; GFX7-NEXT:    buffer_wbinvl1_vol
5686; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5687; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5688; GFX7-NEXT:    flat_store_dword v[0:1], v2
5689; GFX7-NEXT:    s_endpgm
5690;
5691; GFX10-WGP-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
5692; GFX10-WGP:       ; %bb.0: ; %entry
5693; GFX10-WGP-NEXT:    s_clause 0x1
5694; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5695; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5696; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5697; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5698; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5699; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5700; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5701; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5702; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5703; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5704; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5705; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5706; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5707; GFX10-WGP-NEXT:    buffer_gl0_inv
5708; GFX10-WGP-NEXT:    buffer_gl1_inv
5709; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5710; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5711; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5712; GFX10-WGP-NEXT:    s_endpgm
5713;
5714; GFX10-CU-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
5715; GFX10-CU:       ; %bb.0: ; %entry
5716; GFX10-CU-NEXT:    s_clause 0x1
5717; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5718; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5719; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5720; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5721; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5722; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5723; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5724; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5725; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5726; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5727; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5728; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5729; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5730; GFX10-CU-NEXT:    buffer_gl0_inv
5731; GFX10-CU-NEXT:    buffer_gl1_inv
5732; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5733; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5734; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5735; GFX10-CU-NEXT:    s_endpgm
5736;
5737; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
5738; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5739; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5740; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5741; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5742; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5743; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5744; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5745; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5746; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5747; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5748; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5749; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5750; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5751; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5752; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5753; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5754; SKIP-CACHE-INV-NEXT:    s_endpgm
5755;
5756; GFX90A-NOTTGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
5757; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5758; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5759; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5760; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5761; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5762; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5763; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5764; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5765; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5766; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5767; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5768; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5769;
5770; GFX90A-TGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
5771; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5772; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5773; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5774; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5775; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5776; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5777; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5778; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5779; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5780; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5781; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5782; GFX90A-TGSPLIT-NEXT:    s_endpgm
5783;
5784; GFX940-NOTTGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
5785; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5786; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5787; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5788; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5789; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5790; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5791; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
5792; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5793; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5794; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5795; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
5796; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5797; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5798;
5799; GFX940-TGSPLIT-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
5800; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5801; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5802; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5803; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5804; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5805; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5806; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
5807; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5808; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5809; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5810; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
5811; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5812; GFX940-TGSPLIT-NEXT:    s_endpgm
5813    i32* %out, i32 %in, i32 %old) {
5814entry:
5815  %gep = getelementptr i32, i32* %out, i32 4
5816  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel seq_cst
5817  %val0 = extractvalue { i32, i1 } %val, 0
5818  store i32 %val0, i32* %out, align 4
5819  ret void
5820}
5821
5822define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_ret_cmpxchg(
5823; GFX7-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
5824; GFX7:       ; %bb.0: ; %entry
5825; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5826; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5827; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5828; GFX7-NEXT:    s_add_u32 s4, s0, 16
5829; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5830; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5831; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5832; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5833; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5834; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5835; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5836; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5837; GFX7-NEXT:    buffer_wbinvl1_vol
5838; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5839; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5840; GFX7-NEXT:    flat_store_dword v[0:1], v2
5841; GFX7-NEXT:    s_endpgm
5842;
5843; GFX10-WGP-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
5844; GFX10-WGP:       ; %bb.0: ; %entry
5845; GFX10-WGP-NEXT:    s_clause 0x1
5846; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5847; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5848; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5849; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5850; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5851; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5852; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5853; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5854; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5855; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5856; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5857; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5858; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5859; GFX10-WGP-NEXT:    buffer_gl0_inv
5860; GFX10-WGP-NEXT:    buffer_gl1_inv
5861; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5862; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5863; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5864; GFX10-WGP-NEXT:    s_endpgm
5865;
5866; GFX10-CU-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
5867; GFX10-CU:       ; %bb.0: ; %entry
5868; GFX10-CU-NEXT:    s_clause 0x1
5869; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5870; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5871; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5872; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5873; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5874; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5875; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5876; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5877; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5878; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5879; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5880; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5881; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5882; GFX10-CU-NEXT:    buffer_gl0_inv
5883; GFX10-CU-NEXT:    buffer_gl1_inv
5884; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5885; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5886; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5887; GFX10-CU-NEXT:    s_endpgm
5888;
5889; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
5890; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5891; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5892; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5893; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5894; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5895; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5896; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5897; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5898; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5899; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5900; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5901; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5902; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5903; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5904; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5905; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5906; SKIP-CACHE-INV-NEXT:    s_endpgm
5907;
5908; GFX90A-NOTTGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
5909; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5910; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5911; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5912; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5913; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5914; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5915; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5916; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5917; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5918; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5919; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5920; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5921;
5922; GFX90A-TGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
5923; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5924; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5925; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5926; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5927; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5928; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5929; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5930; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5931; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5932; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5933; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5934; GFX90A-TGSPLIT-NEXT:    s_endpgm
5935;
5936; GFX940-NOTTGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
5937; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5938; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5939; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5940; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5941; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5942; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5943; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
5944; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5945; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5946; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5947; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
5948; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5949; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5950;
5951; GFX940-TGSPLIT-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
5952; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5953; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5954; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5955; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5956; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5957; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5958; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
5959; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5960; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5961; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5962; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
5963; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5964; GFX940-TGSPLIT-NEXT:    s_endpgm
5965    i32* %out, i32 %in, i32 %old) {
5966entry:
5967  %gep = getelementptr i32, i32* %out, i32 4
5968  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
5969  %val0 = extractvalue { i32, i1 } %val, 0
5970  store i32 %val0, i32* %out, align 4
5971  ret void
5972}
5973
5974define amdgpu_kernel void @flat_agent_one_as_unordered_load(
5975; GFX7-LABEL: flat_agent_one_as_unordered_load:
5976; GFX7:       ; %bb.0: ; %entry
5977; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5978; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5979; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5980; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5981; GFX7-NEXT:    flat_load_dword v2, v[0:1]
5982; GFX7-NEXT:    v_mov_b32_e32 v0, s2
5983; GFX7-NEXT:    v_mov_b32_e32 v1, s3
5984; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5985; GFX7-NEXT:    flat_store_dword v[0:1], v2
5986; GFX7-NEXT:    s_endpgm
5987;
5988; GFX10-WGP-LABEL: flat_agent_one_as_unordered_load:
5989; GFX10-WGP:       ; %bb.0: ; %entry
5990; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
5991; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5992; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5993; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5994; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
5995; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
5996; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
5997; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5998; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5999; GFX10-WGP-NEXT:    s_endpgm
6000;
6001; GFX10-CU-LABEL: flat_agent_one_as_unordered_load:
6002; GFX10-CU:       ; %bb.0: ; %entry
6003; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6004; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6005; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6006; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6007; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
6008; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
6009; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
6010; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6011; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6012; GFX10-CU-NEXT:    s_endpgm
6013;
6014; SKIP-CACHE-INV-LABEL: flat_agent_one_as_unordered_load:
6015; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6016; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6017; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6018; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6019; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6020; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
6021; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6022; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6023; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6024; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6025; SKIP-CACHE-INV-NEXT:    s_endpgm
6026;
6027; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_unordered_load:
6028; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6029; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6030; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6031; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6032; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6033; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6034; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6035; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6036; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6037; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6038; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6039;
6040; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_unordered_load:
6041; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6042; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6043; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6044; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6045; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6046; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6047; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6048; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6049; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6050; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6051; GFX90A-TGSPLIT-NEXT:    s_endpgm
6052;
6053; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_unordered_load:
6054; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6055; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6056; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6057; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6058; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6059; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6060; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6061; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6062; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6063; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6064; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6065;
6066; GFX940-TGSPLIT-LABEL: flat_agent_one_as_unordered_load:
6067; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6068; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6069; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6070; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6071; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6072; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6073; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6074; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6075; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6076; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6077; GFX940-TGSPLIT-NEXT:    s_endpgm
6078    i32* %in, i32* %out) {
6079entry:
6080  %val = load atomic i32, i32* %in syncscope("agent-one-as") unordered, align 4
6081  store i32 %val, i32* %out
6082  ret void
6083}
6084
6085define amdgpu_kernel void @flat_agent_one_as_monotonic_load(
6086; GFX7-LABEL: flat_agent_one_as_monotonic_load:
6087; GFX7:       ; %bb.0: ; %entry
6088; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6089; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6090; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6091; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6092; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
6093; GFX7-NEXT:    v_mov_b32_e32 v0, s2
6094; GFX7-NEXT:    v_mov_b32_e32 v1, s3
6095; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6096; GFX7-NEXT:    flat_store_dword v[0:1], v2
6097; GFX7-NEXT:    s_endpgm
6098;
6099; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_load:
6100; GFX10-WGP:       ; %bb.0: ; %entry
6101; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6102; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6103; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6104; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6105; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
6106; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
6107; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
6108; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6109; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6110; GFX10-WGP-NEXT:    s_endpgm
6111;
6112; GFX10-CU-LABEL: flat_agent_one_as_monotonic_load:
6113; GFX10-CU:       ; %bb.0: ; %entry
6114; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6115; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6116; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6117; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6118; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
6119; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
6120; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
6121; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6122; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6123; GFX10-CU-NEXT:    s_endpgm
6124;
6125; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_load:
6126; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6127; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6128; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6129; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6130; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6131; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
6132; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6133; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6134; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6135; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6136; SKIP-CACHE-INV-NEXT:    s_endpgm
6137;
6138; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_load:
6139; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6140; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6141; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6142; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6143; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6144; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
6145; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6146; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6147; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6148; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6149; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6150;
6151; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_load:
6152; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6153; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6154; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6155; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6156; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6157; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
6158; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6159; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6160; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6161; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6162; GFX90A-TGSPLIT-NEXT:    s_endpgm
6163;
6164; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_load:
6165; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6166; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6167; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6168; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6169; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6170; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc1
6171; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6172; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6173; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6174; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6175; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6176;
6177; GFX940-TGSPLIT-LABEL: flat_agent_one_as_monotonic_load:
6178; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6179; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6180; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6181; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6182; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6183; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc1
6184; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6185; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6186; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6187; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6188; GFX940-TGSPLIT-NEXT:    s_endpgm
6189    i32* %in, i32* %out) {
6190entry:
6191  %val = load atomic i32, i32* %in syncscope("agent-one-as") monotonic, align 4
6192  store i32 %val, i32* %out
6193  ret void
6194}
6195
6196define amdgpu_kernel void @flat_agent_one_as_acquire_load(
6197; GFX7-LABEL: flat_agent_one_as_acquire_load:
6198; GFX7:       ; %bb.0: ; %entry
6199; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6200; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6201; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6202; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6203; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
6204; GFX7-NEXT:    s_waitcnt vmcnt(0)
6205; GFX7-NEXT:    buffer_wbinvl1_vol
6206; GFX7-NEXT:    v_mov_b32_e32 v0, s2
6207; GFX7-NEXT:    v_mov_b32_e32 v1, s3
6208; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6209; GFX7-NEXT:    flat_store_dword v[0:1], v2
6210; GFX7-NEXT:    s_endpgm
6211;
6212; GFX10-WGP-LABEL: flat_agent_one_as_acquire_load:
6213; GFX10-WGP:       ; %bb.0: ; %entry
6214; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6215; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6216; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6217; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6218; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
6219; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6220; GFX10-WGP-NEXT:    buffer_gl0_inv
6221; GFX10-WGP-NEXT:    buffer_gl1_inv
6222; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
6223; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
6224; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6225; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6226; GFX10-WGP-NEXT:    s_endpgm
6227;
6228; GFX10-CU-LABEL: flat_agent_one_as_acquire_load:
6229; GFX10-CU:       ; %bb.0: ; %entry
6230; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6231; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6232; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6233; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6234; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
6235; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6236; GFX10-CU-NEXT:    buffer_gl0_inv
6237; GFX10-CU-NEXT:    buffer_gl1_inv
6238; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
6239; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
6240; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6241; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6242; GFX10-CU-NEXT:    s_endpgm
6243;
6244; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_load:
6245; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6246; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6247; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6248; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6249; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6250; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
6251; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6252; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6253; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6254; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6255; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6256; SKIP-CACHE-INV-NEXT:    s_endpgm
6257;
6258; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_load:
6259; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6260; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6261; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6262; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6263; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6264; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
6265; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6266; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6267; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6268; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6269; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6270; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6271; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6272;
6273; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_load:
6274; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6275; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6276; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6277; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6278; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6279; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
6280; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6281; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6282; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6283; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6284; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6285; GFX90A-TGSPLIT-NEXT:    s_endpgm
6286;
6287; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_load:
6288; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6289; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6290; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6291; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6292; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6293; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc1
6294; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6295; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
6296; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6297; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6298; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6299; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6300; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6301;
6302; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acquire_load:
6303; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6304; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6305; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6306; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6307; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6308; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc1
6309; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6310; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
6311; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6312; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6313; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6314; GFX940-TGSPLIT-NEXT:    s_endpgm
6315    i32* %in, i32* %out) {
6316entry:
6317  %val = load atomic i32, i32* %in syncscope("agent-one-as") acquire, align 4
6318  store i32 %val, i32* %out
6319  ret void
6320}
6321
6322define amdgpu_kernel void @flat_agent_one_as_seq_cst_load(
6323; GFX7-LABEL: flat_agent_one_as_seq_cst_load:
6324; GFX7:       ; %bb.0: ; %entry
6325; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6326; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6327; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6328; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6329; GFX7-NEXT:    s_waitcnt vmcnt(0)
6330; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
6331; GFX7-NEXT:    s_waitcnt vmcnt(0)
6332; GFX7-NEXT:    buffer_wbinvl1_vol
6333; GFX7-NEXT:    v_mov_b32_e32 v0, s2
6334; GFX7-NEXT:    v_mov_b32_e32 v1, s3
6335; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6336; GFX7-NEXT:    flat_store_dword v[0:1], v2
6337; GFX7-NEXT:    s_endpgm
6338;
6339; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_load:
6340; GFX10-WGP:       ; %bb.0: ; %entry
6341; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6342; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6343; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6344; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6345; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6346; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6347; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
6348; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6349; GFX10-WGP-NEXT:    buffer_gl0_inv
6350; GFX10-WGP-NEXT:    buffer_gl1_inv
6351; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
6352; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
6353; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6354; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6355; GFX10-WGP-NEXT:    s_endpgm
6356;
6357; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_load:
6358; GFX10-CU:       ; %bb.0: ; %entry
6359; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6360; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6361; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6362; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6363; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6364; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6365; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
6366; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6367; GFX10-CU-NEXT:    buffer_gl0_inv
6368; GFX10-CU-NEXT:    buffer_gl1_inv
6369; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
6370; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
6371; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6372; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6373; GFX10-CU-NEXT:    s_endpgm
6374;
6375; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_load:
6376; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6377; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6378; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6379; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6380; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6381; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6382; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
6383; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6384; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6385; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6386; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6387; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6388; SKIP-CACHE-INV-NEXT:    s_endpgm
6389;
6390; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_load:
6391; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6392; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6393; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6394; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6395; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6396; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6397; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
6398; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6399; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6400; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6401; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6402; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6403; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6404; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6405;
6406; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_load:
6407; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6408; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6409; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6410; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6411; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6412; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6413; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
6414; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6415; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6416; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6417; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6418; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6419; GFX90A-TGSPLIT-NEXT:    s_endpgm
6420;
6421; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_load:
6422; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6423; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6424; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6425; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6426; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6427; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6428; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc1
6429; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6430; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
6431; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6432; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6433; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6434; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6435; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6436;
6437; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_load:
6438; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6439; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6440; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6441; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6442; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6443; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6444; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc1
6445; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6446; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
6447; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6448; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6449; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6450; GFX940-TGSPLIT-NEXT:    s_endpgm
6451    i32* %in, i32* %out) {
6452entry:
6453  %val = load atomic i32, i32* %in syncscope("agent-one-as") seq_cst, align 4
6454  store i32 %val, i32* %out
6455  ret void
6456}
6457
6458define amdgpu_kernel void @flat_agent_one_as_unordered_store(
6459; GFX7-LABEL: flat_agent_one_as_unordered_store:
6460; GFX7:       ; %bb.0: ; %entry
6461; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
6462; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
6463; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6464; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6465; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6466; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6467; GFX7-NEXT:    flat_store_dword v[0:1], v2
6468; GFX7-NEXT:    s_endpgm
6469;
6470; GFX10-WGP-LABEL: flat_agent_one_as_unordered_store:
6471; GFX10-WGP:       ; %bb.0: ; %entry
6472; GFX10-WGP-NEXT:    s_clause 0x1
6473; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6474; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
6475; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6476; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6477; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6478; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6479; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6480; GFX10-WGP-NEXT:    s_endpgm
6481;
6482; GFX10-CU-LABEL: flat_agent_one_as_unordered_store:
6483; GFX10-CU:       ; %bb.0: ; %entry
6484; GFX10-CU-NEXT:    s_clause 0x1
6485; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6486; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
6487; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6488; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6489; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6490; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6491; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6492; GFX10-CU-NEXT:    s_endpgm
6493;
6494; SKIP-CACHE-INV-LABEL: flat_agent_one_as_unordered_store:
6495; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6496; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
6497; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
6498; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6499; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6500; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6501; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6502; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6503; SKIP-CACHE-INV-NEXT:    s_endpgm
6504;
6505; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_unordered_store:
6506; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6507; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6508; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
6509; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6510; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6511; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6512; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6513; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6514;
6515; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_unordered_store:
6516; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6517; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6518; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
6519; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6520; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6521; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6522; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6523; GFX90A-TGSPLIT-NEXT:    s_endpgm
6524;
6525; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_unordered_store:
6526; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6527; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
6528; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
6529; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6530; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6531; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
6532; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6533; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6534;
6535; GFX940-TGSPLIT-LABEL: flat_agent_one_as_unordered_store:
6536; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6537; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
6538; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
6539; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6540; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6541; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
6542; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6543; GFX940-TGSPLIT-NEXT:    s_endpgm
6544    i32 %in, i32* %out) {
6545entry:
6546  store atomic i32 %in, i32* %out syncscope("agent-one-as") unordered, align 4
6547  ret void
6548}
6549
6550define amdgpu_kernel void @flat_agent_one_as_monotonic_store(
6551; GFX7-LABEL: flat_agent_one_as_monotonic_store:
6552; GFX7:       ; %bb.0: ; %entry
6553; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
6554; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
6555; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6556; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6557; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6558; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6559; GFX7-NEXT:    flat_store_dword v[0:1], v2
6560; GFX7-NEXT:    s_endpgm
6561;
6562; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_store:
6563; GFX10-WGP:       ; %bb.0: ; %entry
6564; GFX10-WGP-NEXT:    s_clause 0x1
6565; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6566; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
6567; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6568; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6569; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6570; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6571; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6572; GFX10-WGP-NEXT:    s_endpgm
6573;
6574; GFX10-CU-LABEL: flat_agent_one_as_monotonic_store:
6575; GFX10-CU:       ; %bb.0: ; %entry
6576; GFX10-CU-NEXT:    s_clause 0x1
6577; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6578; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
6579; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6580; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6581; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6582; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6583; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6584; GFX10-CU-NEXT:    s_endpgm
6585;
6586; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_store:
6587; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6588; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
6589; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
6590; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6591; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6592; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6593; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6594; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6595; SKIP-CACHE-INV-NEXT:    s_endpgm
6596;
6597; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_store:
6598; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6599; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6600; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
6601; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6602; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6603; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6604; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6605; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6606;
6607; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_store:
6608; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6609; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6610; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
6611; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6612; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6613; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6614; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6615; GFX90A-TGSPLIT-NEXT:    s_endpgm
6616;
6617; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_store:
6618; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6619; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
6620; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
6621; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6622; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6623; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
6624; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc1
6625; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6626;
6627; GFX940-TGSPLIT-LABEL: flat_agent_one_as_monotonic_store:
6628; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6629; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
6630; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
6631; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6632; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6633; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
6634; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc1
6635; GFX940-TGSPLIT-NEXT:    s_endpgm
6636    i32 %in, i32* %out) {
6637entry:
6638  store atomic i32 %in, i32* %out syncscope("agent-one-as") monotonic, align 4
6639  ret void
6640}
6641
6642define amdgpu_kernel void @flat_agent_one_as_release_store(
6643; GFX7-LABEL: flat_agent_one_as_release_store:
6644; GFX7:       ; %bb.0: ; %entry
6645; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
6646; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
6647; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6648; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6649; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6650; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6651; GFX7-NEXT:    s_waitcnt vmcnt(0)
6652; GFX7-NEXT:    flat_store_dword v[0:1], v2
6653; GFX7-NEXT:    s_endpgm
6654;
6655; GFX10-WGP-LABEL: flat_agent_one_as_release_store:
6656; GFX10-WGP:       ; %bb.0: ; %entry
6657; GFX10-WGP-NEXT:    s_clause 0x1
6658; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6659; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
6660; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6661; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6662; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6663; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6664; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6665; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6666; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6667; GFX10-WGP-NEXT:    s_endpgm
6668;
6669; GFX10-CU-LABEL: flat_agent_one_as_release_store:
6670; GFX10-CU:       ; %bb.0: ; %entry
6671; GFX10-CU-NEXT:    s_clause 0x1
6672; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6673; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
6674; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6675; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6676; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6677; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6678; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6679; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6680; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6681; GFX10-CU-NEXT:    s_endpgm
6682;
6683; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_store:
6684; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6685; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
6686; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
6687; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6688; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6689; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6690; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6691; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6692; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6693; SKIP-CACHE-INV-NEXT:    s_endpgm
6694;
6695; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_store:
6696; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6697; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6698; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
6699; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6700; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6701; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6702; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6703; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6704; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6705;
6706; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_store:
6707; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6708; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6709; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
6710; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6711; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6712; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6713; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6714; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6715; GFX90A-TGSPLIT-NEXT:    s_endpgm
6716;
6717; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_release_store:
6718; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6719; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
6720; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
6721; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6722; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6723; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
6724; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
6725; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6726; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc1
6727; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6728;
6729; GFX940-TGSPLIT-LABEL: flat_agent_one_as_release_store:
6730; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6731; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
6732; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
6733; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6734; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6735; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
6736; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
6737; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6738; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc1
6739; GFX940-TGSPLIT-NEXT:    s_endpgm
6740    i32 %in, i32* %out) {
6741entry:
6742  store atomic i32 %in, i32* %out syncscope("agent-one-as") release, align 4
6743  ret void
6744}
6745
6746define amdgpu_kernel void @flat_agent_one_as_seq_cst_store(
6747; GFX7-LABEL: flat_agent_one_as_seq_cst_store:
6748; GFX7:       ; %bb.0: ; %entry
6749; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
6750; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
6751; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6752; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6753; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6754; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6755; GFX7-NEXT:    s_waitcnt vmcnt(0)
6756; GFX7-NEXT:    flat_store_dword v[0:1], v2
6757; GFX7-NEXT:    s_endpgm
6758;
6759; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_store:
6760; GFX10-WGP:       ; %bb.0: ; %entry
6761; GFX10-WGP-NEXT:    s_clause 0x1
6762; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6763; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
6764; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6765; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6766; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6767; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6768; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6769; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6770; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6771; GFX10-WGP-NEXT:    s_endpgm
6772;
6773; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_store:
6774; GFX10-CU:       ; %bb.0: ; %entry
6775; GFX10-CU-NEXT:    s_clause 0x1
6776; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6777; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
6778; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6779; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6780; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6781; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6782; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6783; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6784; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6785; GFX10-CU-NEXT:    s_endpgm
6786;
6787; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_store:
6788; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6789; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
6790; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
6791; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6792; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6793; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6794; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6795; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6796; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6797; SKIP-CACHE-INV-NEXT:    s_endpgm
6798;
6799; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_store:
6800; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6801; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6802; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
6803; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6804; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6805; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6806; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6807; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6808; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6809;
6810; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_store:
6811; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6812; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6813; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
6814; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6815; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6816; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6817; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6818; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6819; GFX90A-TGSPLIT-NEXT:    s_endpgm
6820;
6821; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_store:
6822; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6823; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
6824; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
6825; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6826; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6827; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
6828; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
6829; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6830; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc1
6831; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6832;
6833; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_store:
6834; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6835; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
6836; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
6837; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6838; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6839; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
6840; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
6841; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6842; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc1
6843; GFX940-TGSPLIT-NEXT:    s_endpgm
6844    i32 %in, i32* %out) {
6845entry:
6846  store atomic i32 %in, i32* %out syncscope("agent-one-as") seq_cst, align 4
6847  ret void
6848}
6849
6850define amdgpu_kernel void @flat_agent_one_as_monotonic_atomicrmw(
6851; GFX7-LABEL: flat_agent_one_as_monotonic_atomicrmw:
6852; GFX7:       ; %bb.0: ; %entry
6853; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6854; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
6855; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6856; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6857; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6858; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6859; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
6860; GFX7-NEXT:    s_endpgm
6861;
6862; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_atomicrmw:
6863; GFX10-WGP:       ; %bb.0: ; %entry
6864; GFX10-WGP-NEXT:    s_clause 0x1
6865; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6866; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
6867; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6868; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6869; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6870; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6871; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
6872; GFX10-WGP-NEXT:    s_endpgm
6873;
6874; GFX10-CU-LABEL: flat_agent_one_as_monotonic_atomicrmw:
6875; GFX10-CU:       ; %bb.0: ; %entry
6876; GFX10-CU-NEXT:    s_clause 0x1
6877; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6878; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
6879; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6880; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6881; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6882; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6883; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
6884; GFX10-CU-NEXT:    s_endpgm
6885;
6886; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_atomicrmw:
6887; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6888; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6889; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
6890; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6891; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6892; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6893; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6894; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
6895; SKIP-CACHE-INV-NEXT:    s_endpgm
6896;
6897; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_atomicrmw:
6898; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6899; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6900; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6901; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6902; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6903; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6904; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
6905; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6906;
6907; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_atomicrmw:
6908; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6909; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6910; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6911; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6912; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6913; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6914; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
6915; GFX90A-TGSPLIT-NEXT:    s_endpgm
6916;
6917; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_atomicrmw:
6918; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6919; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6920; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
6921; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6922; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6923; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
6924; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
6925; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6926;
6927; GFX940-TGSPLIT-LABEL: flat_agent_one_as_monotonic_atomicrmw:
6928; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6929; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6930; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
6931; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6932; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6933; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
6934; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
6935; GFX940-TGSPLIT-NEXT:    s_endpgm
6936    i32* %out, i32 %in) {
6937entry:
6938  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") monotonic
6939  ret void
6940}
6941
6942define amdgpu_kernel void @flat_agent_one_as_acquire_atomicrmw(
6943; GFX7-LABEL: flat_agent_one_as_acquire_atomicrmw:
6944; GFX7:       ; %bb.0: ; %entry
6945; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6946; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
6947; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6948; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6949; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6950; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6951; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
6952; GFX7-NEXT:    s_waitcnt vmcnt(0)
6953; GFX7-NEXT:    buffer_wbinvl1_vol
6954; GFX7-NEXT:    s_endpgm
6955;
6956; GFX10-WGP-LABEL: flat_agent_one_as_acquire_atomicrmw:
6957; GFX10-WGP:       ; %bb.0: ; %entry
6958; GFX10-WGP-NEXT:    s_clause 0x1
6959; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6960; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
6961; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6962; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6963; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6964; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6965; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
6966; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6967; GFX10-WGP-NEXT:    buffer_gl0_inv
6968; GFX10-WGP-NEXT:    buffer_gl1_inv
6969; GFX10-WGP-NEXT:    s_endpgm
6970;
6971; GFX10-CU-LABEL: flat_agent_one_as_acquire_atomicrmw:
6972; GFX10-CU:       ; %bb.0: ; %entry
6973; GFX10-CU-NEXT:    s_clause 0x1
6974; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6975; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
6976; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6977; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6978; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6979; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6980; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
6981; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6982; GFX10-CU-NEXT:    buffer_gl0_inv
6983; GFX10-CU-NEXT:    buffer_gl1_inv
6984; GFX10-CU-NEXT:    s_endpgm
6985;
6986; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_atomicrmw:
6987; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6988; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6989; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
6990; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6991; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6992; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6993; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6994; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
6995; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6996; SKIP-CACHE-INV-NEXT:    s_endpgm
6997;
6998; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_atomicrmw:
6999; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7000; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7001; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7002; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7003; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7004; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7005; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7006; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7007; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7008; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7009;
7010; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_atomicrmw:
7011; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7012; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7013; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7014; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7015; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7016; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7017; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7018; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7019; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7020; GFX90A-TGSPLIT-NEXT:    s_endpgm
7021;
7022; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_atomicrmw:
7023; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7024; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7025; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7026; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7027; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7028; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7029; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7030; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7031; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
7032; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7033;
7034; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acquire_atomicrmw:
7035; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7036; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7037; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7038; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7039; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7040; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7041; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7042; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7043; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
7044; GFX940-TGSPLIT-NEXT:    s_endpgm
7045    i32* %out, i32 %in) {
7046entry:
7047  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") acquire
7048  ret void
7049}
7050
7051define amdgpu_kernel void @flat_agent_one_as_release_atomicrmw(
7052; GFX7-LABEL: flat_agent_one_as_release_atomicrmw:
7053; GFX7:       ; %bb.0: ; %entry
7054; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7055; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7056; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7057; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7058; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7059; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7060; GFX7-NEXT:    s_waitcnt vmcnt(0)
7061; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
7062; GFX7-NEXT:    s_endpgm
7063;
7064; GFX10-WGP-LABEL: flat_agent_one_as_release_atomicrmw:
7065; GFX10-WGP:       ; %bb.0: ; %entry
7066; GFX10-WGP-NEXT:    s_clause 0x1
7067; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7068; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7069; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7070; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7071; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7072; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7073; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7074; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7075; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
7076; GFX10-WGP-NEXT:    s_endpgm
7077;
7078; GFX10-CU-LABEL: flat_agent_one_as_release_atomicrmw:
7079; GFX10-CU:       ; %bb.0: ; %entry
7080; GFX10-CU-NEXT:    s_clause 0x1
7081; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7082; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7083; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7084; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7085; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7086; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7087; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7088; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7089; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
7090; GFX10-CU-NEXT:    s_endpgm
7091;
7092; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_atomicrmw:
7093; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7094; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7095; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
7096; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7097; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7098; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7099; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7100; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7101; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
7102; SKIP-CACHE-INV-NEXT:    s_endpgm
7103;
7104; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_atomicrmw:
7105; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7106; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7107; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7108; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7109; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7110; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7111; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7112; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7113; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7114;
7115; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_atomicrmw:
7116; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7117; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7118; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7119; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7120; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7121; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7122; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7123; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7124; GFX90A-TGSPLIT-NEXT:    s_endpgm
7125;
7126; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_release_atomicrmw:
7127; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7128; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7129; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7130; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7131; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7132; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7133; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
7134; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7135; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7136; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7137;
7138; GFX940-TGSPLIT-LABEL: flat_agent_one_as_release_atomicrmw:
7139; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7140; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7141; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7142; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7143; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7144; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7145; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
7146; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7147; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7148; GFX940-TGSPLIT-NEXT:    s_endpgm
7149    i32* %out, i32 %in) {
7150entry:
7151  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") release
7152  ret void
7153}
7154
7155define amdgpu_kernel void @flat_agent_one_as_acq_rel_atomicrmw(
7156; GFX7-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
7157; GFX7:       ; %bb.0: ; %entry
7158; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7159; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7160; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7161; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7162; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7163; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7164; GFX7-NEXT:    s_waitcnt vmcnt(0)
7165; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
7166; GFX7-NEXT:    s_waitcnt vmcnt(0)
7167; GFX7-NEXT:    buffer_wbinvl1_vol
7168; GFX7-NEXT:    s_endpgm
7169;
7170; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
7171; GFX10-WGP:       ; %bb.0: ; %entry
7172; GFX10-WGP-NEXT:    s_clause 0x1
7173; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7174; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7175; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7176; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7177; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7178; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7179; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7180; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7181; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
7182; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7183; GFX10-WGP-NEXT:    buffer_gl0_inv
7184; GFX10-WGP-NEXT:    buffer_gl1_inv
7185; GFX10-WGP-NEXT:    s_endpgm
7186;
7187; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
7188; GFX10-CU:       ; %bb.0: ; %entry
7189; GFX10-CU-NEXT:    s_clause 0x1
7190; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7191; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7192; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7193; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7194; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7195; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7196; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7197; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7198; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
7199; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7200; GFX10-CU-NEXT:    buffer_gl0_inv
7201; GFX10-CU-NEXT:    buffer_gl1_inv
7202; GFX10-CU-NEXT:    s_endpgm
7203;
7204; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
7205; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7206; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7207; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
7208; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7209; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7210; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7211; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7212; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7213; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
7214; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7215; SKIP-CACHE-INV-NEXT:    s_endpgm
7216;
7217; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
7218; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7219; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7220; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7221; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7222; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7223; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7224; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7225; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7226; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7227; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7228; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7229;
7230; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
7231; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7232; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7233; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7234; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7235; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7236; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7237; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7238; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7239; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7240; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7241; GFX90A-TGSPLIT-NEXT:    s_endpgm
7242;
7243; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
7244; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7245; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7246; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7247; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7248; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7249; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7250; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
7251; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7252; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7253; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7254; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
7255; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7256;
7257; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
7258; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7259; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7260; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7261; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7262; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7263; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7264; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
7265; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7266; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7267; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7268; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
7269; GFX940-TGSPLIT-NEXT:    s_endpgm
7270    i32* %out, i32 %in) {
7271entry:
7272  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") acq_rel
7273  ret void
7274}
7275
7276define amdgpu_kernel void @flat_agent_one_as_seq_cst_atomicrmw(
7277; GFX7-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
7278; GFX7:       ; %bb.0: ; %entry
7279; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7280; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7281; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7282; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7283; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7284; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7285; GFX7-NEXT:    s_waitcnt vmcnt(0)
7286; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
7287; GFX7-NEXT:    s_waitcnt vmcnt(0)
7288; GFX7-NEXT:    buffer_wbinvl1_vol
7289; GFX7-NEXT:    s_endpgm
7290;
7291; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
7292; GFX10-WGP:       ; %bb.0: ; %entry
7293; GFX10-WGP-NEXT:    s_clause 0x1
7294; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7295; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7296; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7297; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7298; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7299; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7300; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7301; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7302; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
7303; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7304; GFX10-WGP-NEXT:    buffer_gl0_inv
7305; GFX10-WGP-NEXT:    buffer_gl1_inv
7306; GFX10-WGP-NEXT:    s_endpgm
7307;
7308; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
7309; GFX10-CU:       ; %bb.0: ; %entry
7310; GFX10-CU-NEXT:    s_clause 0x1
7311; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7312; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7313; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7314; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7315; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7316; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7317; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7318; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7319; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
7320; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7321; GFX10-CU-NEXT:    buffer_gl0_inv
7322; GFX10-CU-NEXT:    buffer_gl1_inv
7323; GFX10-CU-NEXT:    s_endpgm
7324;
7325; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
7326; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7327; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7328; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
7329; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7330; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7331; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7332; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7333; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7334; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
7335; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7336; SKIP-CACHE-INV-NEXT:    s_endpgm
7337;
7338; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
7339; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7340; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7341; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7342; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7343; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7344; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7345; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7346; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7347; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7348; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7349; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7350;
7351; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
7352; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7353; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7354; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7355; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7356; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7357; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7358; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7359; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7360; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7361; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7362; GFX90A-TGSPLIT-NEXT:    s_endpgm
7363;
7364; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
7365; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7366; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7367; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7368; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7369; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7370; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7371; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
7372; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7373; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7374; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7375; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
7376; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7377;
7378; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
7379; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7380; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7381; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7382; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7383; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7384; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7385; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
7386; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7387; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7388; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7389; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
7390; GFX940-TGSPLIT-NEXT:    s_endpgm
7391    i32* %out, i32 %in) {
7392entry:
7393  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") seq_cst
7394  ret void
7395}
7396
7397define amdgpu_kernel void @flat_agent_one_as_acquire_ret_atomicrmw(
7398; GFX7-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
7399; GFX7:       ; %bb.0: ; %entry
7400; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7401; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7402; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7403; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7404; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7405; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7406; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7407; GFX7-NEXT:    s_waitcnt vmcnt(0)
7408; GFX7-NEXT:    buffer_wbinvl1_vol
7409; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7410; GFX7-NEXT:    flat_store_dword v[0:1], v2
7411; GFX7-NEXT:    s_endpgm
7412;
7413; GFX10-WGP-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
7414; GFX10-WGP:       ; %bb.0: ; %entry
7415; GFX10-WGP-NEXT:    s_clause 0x1
7416; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7417; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7418; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7419; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7420; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7421; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7422; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7423; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7424; GFX10-WGP-NEXT:    buffer_gl0_inv
7425; GFX10-WGP-NEXT:    buffer_gl1_inv
7426; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7427; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7428; GFX10-WGP-NEXT:    s_endpgm
7429;
7430; GFX10-CU-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
7431; GFX10-CU:       ; %bb.0: ; %entry
7432; GFX10-CU-NEXT:    s_clause 0x1
7433; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7434; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7435; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7436; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7437; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7438; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7439; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7440; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7441; GFX10-CU-NEXT:    buffer_gl0_inv
7442; GFX10-CU-NEXT:    buffer_gl1_inv
7443; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7444; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7445; GFX10-CU-NEXT:    s_endpgm
7446;
7447; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
7448; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7449; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7450; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
7451; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7452; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7453; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7454; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7455; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7456; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7457; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7458; SKIP-CACHE-INV-NEXT:    s_endpgm
7459;
7460; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
7461; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7462; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7463; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7464; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7465; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7466; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7467; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7468; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7469; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7470; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7471; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7472; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7473;
7474; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
7475; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7476; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7477; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7478; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7479; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7480; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7481; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7482; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7483; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7484; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7485; GFX90A-TGSPLIT-NEXT:    s_endpgm
7486;
7487; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
7488; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7489; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7490; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7491; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7492; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7493; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7494; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
7495; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7496; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
7497; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7498; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7499; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7500;
7501; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
7502; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7503; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7504; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7505; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7506; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7507; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7508; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
7509; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7510; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
7511; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7512; GFX940-TGSPLIT-NEXT:    s_endpgm
7513    i32* %out, i32 %in) {
7514entry:
7515  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") acquire
7516  store i32 %val, i32* %out, align 4
7517  ret void
7518}
7519
7520define amdgpu_kernel void @flat_agent_one_as_acq_rel_ret_atomicrmw(
7521; GFX7-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
7522; GFX7:       ; %bb.0: ; %entry
7523; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7524; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7525; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7526; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7527; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7528; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7529; GFX7-NEXT:    s_waitcnt vmcnt(0)
7530; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7531; GFX7-NEXT:    s_waitcnt vmcnt(0)
7532; GFX7-NEXT:    buffer_wbinvl1_vol
7533; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7534; GFX7-NEXT:    flat_store_dword v[0:1], v2
7535; GFX7-NEXT:    s_endpgm
7536;
7537; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
7538; GFX10-WGP:       ; %bb.0: ; %entry
7539; GFX10-WGP-NEXT:    s_clause 0x1
7540; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7541; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7542; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7543; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7544; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7545; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7546; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7547; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7548; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7549; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7550; GFX10-WGP-NEXT:    buffer_gl0_inv
7551; GFX10-WGP-NEXT:    buffer_gl1_inv
7552; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7553; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7554; GFX10-WGP-NEXT:    s_endpgm
7555;
7556; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
7557; GFX10-CU:       ; %bb.0: ; %entry
7558; GFX10-CU-NEXT:    s_clause 0x1
7559; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7560; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7561; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7562; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7563; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7564; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7565; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7566; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7567; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7568; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7569; GFX10-CU-NEXT:    buffer_gl0_inv
7570; GFX10-CU-NEXT:    buffer_gl1_inv
7571; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7572; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7573; GFX10-CU-NEXT:    s_endpgm
7574;
7575; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
7576; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7577; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7578; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
7579; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7580; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7581; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7582; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7583; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7584; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7585; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7586; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7587; SKIP-CACHE-INV-NEXT:    s_endpgm
7588;
7589; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
7590; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7591; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7592; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7593; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7594; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7595; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7596; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7597; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7598; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7599; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7600; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7601; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7602; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7603;
7604; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
7605; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7606; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7607; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7608; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7609; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7610; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7611; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7612; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7613; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7614; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7615; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7616; GFX90A-TGSPLIT-NEXT:    s_endpgm
7617;
7618; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
7619; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7620; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7621; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7622; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7623; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7624; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7625; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
7626; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7627; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
7628; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7629; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
7630; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7631; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7632; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7633;
7634; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
7635; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7636; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7637; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7638; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7639; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7640; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7641; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
7642; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7643; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
7644; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7645; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
7646; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7647; GFX940-TGSPLIT-NEXT:    s_endpgm
7648    i32* %out, i32 %in) {
7649entry:
7650  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") acq_rel
7651  store i32 %val, i32* %out, align 4
7652  ret void
7653}
7654
7655define amdgpu_kernel void @flat_agent_one_as_seq_cst_ret_atomicrmw(
7656; GFX7-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
7657; GFX7:       ; %bb.0: ; %entry
7658; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7659; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7660; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7661; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7662; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7663; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7664; GFX7-NEXT:    s_waitcnt vmcnt(0)
7665; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7666; GFX7-NEXT:    s_waitcnt vmcnt(0)
7667; GFX7-NEXT:    buffer_wbinvl1_vol
7668; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7669; GFX7-NEXT:    flat_store_dword v[0:1], v2
7670; GFX7-NEXT:    s_endpgm
7671;
7672; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
7673; GFX10-WGP:       ; %bb.0: ; %entry
7674; GFX10-WGP-NEXT:    s_clause 0x1
7675; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7676; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7677; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7678; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7679; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7680; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7681; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7682; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7683; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7684; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7685; GFX10-WGP-NEXT:    buffer_gl0_inv
7686; GFX10-WGP-NEXT:    buffer_gl1_inv
7687; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7688; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7689; GFX10-WGP-NEXT:    s_endpgm
7690;
7691; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
7692; GFX10-CU:       ; %bb.0: ; %entry
7693; GFX10-CU-NEXT:    s_clause 0x1
7694; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7695; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7696; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7697; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7698; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7699; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7700; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7701; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7702; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7703; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7704; GFX10-CU-NEXT:    buffer_gl0_inv
7705; GFX10-CU-NEXT:    buffer_gl1_inv
7706; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7707; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7708; GFX10-CU-NEXT:    s_endpgm
7709;
7710; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
7711; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7712; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7713; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
7714; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7715; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7716; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7717; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7718; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7719; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7720; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7721; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7722; SKIP-CACHE-INV-NEXT:    s_endpgm
7723;
7724; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
7725; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7726; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7727; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7728; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7729; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7730; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7731; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7732; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7733; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7734; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7735; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7736; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7737; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7738;
7739; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
7740; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7741; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7742; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7743; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7744; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7745; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7746; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7747; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7748; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7749; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7750; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7751; GFX90A-TGSPLIT-NEXT:    s_endpgm
7752;
7753; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
7754; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7755; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7756; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7757; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7758; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7759; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7760; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
7761; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7762; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
7763; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7764; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
7765; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7766; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7767; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7768;
7769; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
7770; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7771; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7772; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7773; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7774; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7775; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7776; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
7777; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7778; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0
7779; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7780; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
7781; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7782; GFX940-TGSPLIT-NEXT:    s_endpgm
7783    i32* %out, i32 %in) {
7784entry:
7785  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") seq_cst
7786  store i32 %val, i32* %out, align 4
7787  ret void
7788}
7789
7790define amdgpu_kernel void @flat_agent_one_as_monotonic_monotonic_cmpxchg(
7791; GFX7-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
7792; GFX7:       ; %bb.0: ; %entry
7793; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7794; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7795; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7796; GFX7-NEXT:    s_add_u32 s0, s0, 16
7797; GFX7-NEXT:    s_addc_u32 s1, s1, 0
7798; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7799; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7800; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7801; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7802; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7803; GFX7-NEXT:    s_endpgm
7804;
7805; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
7806; GFX10-WGP:       ; %bb.0: ; %entry
7807; GFX10-WGP-NEXT:    s_clause 0x1
7808; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7809; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7810; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7811; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
7812; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
7813; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7814; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7815; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7816; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7817; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7818; GFX10-WGP-NEXT:    s_endpgm
7819;
7820; GFX10-CU-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
7821; GFX10-CU:       ; %bb.0: ; %entry
7822; GFX10-CU-NEXT:    s_clause 0x1
7823; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7824; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7825; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7826; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
7827; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
7828; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7829; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7830; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7831; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7832; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7833; GFX10-CU-NEXT:    s_endpgm
7834;
7835; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
7836; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7837; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7838; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7839; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7840; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
7841; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
7842; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7843; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7844; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7845; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7846; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7847; SKIP-CACHE-INV-NEXT:    s_endpgm
7848;
7849; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
7850; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7851; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7852; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7853; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7854; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7855; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7856; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7857; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7858;
7859; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
7860; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7861; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7862; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7863; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7864; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7865; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7866; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7867; GFX90A-TGSPLIT-NEXT:    s_endpgm
7868;
7869; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
7870; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7871; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7872; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7873; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7874; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7875; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7876; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7877; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7878;
7879; GFX940-TGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
7880; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7881; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7882; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7883; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7884; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7885; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7886; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7887; GFX940-TGSPLIT-NEXT:    s_endpgm
7888    i32* %out, i32 %in, i32 %old) {
7889entry:
7890  %gep = getelementptr i32, i32* %out, i32 4
7891  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") monotonic monotonic
7892  ret void
7893}
7894
7895define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_cmpxchg(
7896; GFX7-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
7897; GFX7:       ; %bb.0: ; %entry
7898; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7899; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7900; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7901; GFX7-NEXT:    s_add_u32 s0, s0, 16
7902; GFX7-NEXT:    s_addc_u32 s1, s1, 0
7903; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7904; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7905; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7906; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7907; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7908; GFX7-NEXT:    s_waitcnt vmcnt(0)
7909; GFX7-NEXT:    buffer_wbinvl1_vol
7910; GFX7-NEXT:    s_endpgm
7911;
7912; GFX10-WGP-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
7913; GFX10-WGP:       ; %bb.0: ; %entry
7914; GFX10-WGP-NEXT:    s_clause 0x1
7915; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7916; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7917; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7918; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
7919; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
7920; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7921; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7922; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7923; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7924; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7925; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7926; GFX10-WGP-NEXT:    buffer_gl0_inv
7927; GFX10-WGP-NEXT:    buffer_gl1_inv
7928; GFX10-WGP-NEXT:    s_endpgm
7929;
7930; GFX10-CU-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
7931; GFX10-CU:       ; %bb.0: ; %entry
7932; GFX10-CU-NEXT:    s_clause 0x1
7933; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7934; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7935; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7936; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
7937; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
7938; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7939; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7940; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7941; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7942; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7943; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7944; GFX10-CU-NEXT:    buffer_gl0_inv
7945; GFX10-CU-NEXT:    buffer_gl1_inv
7946; GFX10-CU-NEXT:    s_endpgm
7947;
7948; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
7949; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7950; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7951; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7952; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7953; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
7954; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
7955; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7956; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7957; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7958; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7959; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7960; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7961; SKIP-CACHE-INV-NEXT:    s_endpgm
7962;
7963; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
7964; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7965; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7966; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7967; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7968; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7969; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7970; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7971; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7972; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7973; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7974;
7975; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
7976; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7977; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7978; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7979; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7980; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7981; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7982; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7983; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7984; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7985; GFX90A-TGSPLIT-NEXT:    s_endpgm
7986;
7987; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
7988; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7989; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7990; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
7991; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7992; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7993; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
7994; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7995; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7996; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
7997; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7998;
7999; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
8000; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8001; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8002; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8003; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8004; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8005; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8006; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8007; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8008; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
8009; GFX940-TGSPLIT-NEXT:    s_endpgm
8010    i32* %out, i32 %in, i32 %old) {
8011entry:
8012  %gep = getelementptr i32, i32* %out, i32 4
8013  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire monotonic
8014  ret void
8015}
8016
8017define amdgpu_kernel void @flat_agent_one_as_release_monotonic_cmpxchg(
8018; GFX7-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
8019; GFX7:       ; %bb.0: ; %entry
8020; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8021; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8022; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8023; GFX7-NEXT:    s_add_u32 s0, s0, 16
8024; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8025; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8026; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8027; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8028; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8029; GFX7-NEXT:    s_waitcnt vmcnt(0)
8030; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8031; GFX7-NEXT:    s_endpgm
8032;
8033; GFX10-WGP-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
8034; GFX10-WGP:       ; %bb.0: ; %entry
8035; GFX10-WGP-NEXT:    s_clause 0x1
8036; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8037; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8038; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8039; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
8040; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
8041; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8042; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8043; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8044; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8045; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8046; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8047; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8048; GFX10-WGP-NEXT:    s_endpgm
8049;
8050; GFX10-CU-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
8051; GFX10-CU:       ; %bb.0: ; %entry
8052; GFX10-CU-NEXT:    s_clause 0x1
8053; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8054; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8055; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8056; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8057; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8058; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8059; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8060; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8061; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8062; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8063; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8064; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8065; GFX10-CU-NEXT:    s_endpgm
8066;
8067; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
8068; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8069; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8070; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8071; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8072; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
8073; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
8074; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8075; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8076; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8077; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8078; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8079; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8080; SKIP-CACHE-INV-NEXT:    s_endpgm
8081;
8082; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
8083; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8084; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8085; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8086; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8087; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8088; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8089; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8090; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8091; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8092;
8093; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
8094; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8095; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8096; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8097; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8098; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8099; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8100; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8101; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8102; GFX90A-TGSPLIT-NEXT:    s_endpgm
8103;
8104; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
8105; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8106; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8107; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8108; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8109; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8110; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8111; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
8112; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8113; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8114; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8115;
8116; GFX940-TGSPLIT-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
8117; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8118; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8119; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8120; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8121; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8122; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8123; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
8124; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8125; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8126; GFX940-TGSPLIT-NEXT:    s_endpgm
8127    i32* %out, i32 %in, i32 %old) {
8128entry:
8129  %gep = getelementptr i32, i32* %out, i32 4
8130  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release monotonic
8131  ret void
8132}
8133
8134define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_cmpxchg(
8135; GFX7-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
8136; GFX7:       ; %bb.0: ; %entry
8137; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8138; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8139; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8140; GFX7-NEXT:    s_add_u32 s0, s0, 16
8141; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8142; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8143; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8144; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8145; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8146; GFX7-NEXT:    s_waitcnt vmcnt(0)
8147; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8148; GFX7-NEXT:    s_waitcnt vmcnt(0)
8149; GFX7-NEXT:    buffer_wbinvl1_vol
8150; GFX7-NEXT:    s_endpgm
8151;
8152; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
8153; GFX10-WGP:       ; %bb.0: ; %entry
8154; GFX10-WGP-NEXT:    s_clause 0x1
8155; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8156; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8157; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8158; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
8159; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
8160; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8161; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8162; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8163; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8164; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8165; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8166; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8167; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8168; GFX10-WGP-NEXT:    buffer_gl0_inv
8169; GFX10-WGP-NEXT:    buffer_gl1_inv
8170; GFX10-WGP-NEXT:    s_endpgm
8171;
8172; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
8173; GFX10-CU:       ; %bb.0: ; %entry
8174; GFX10-CU-NEXT:    s_clause 0x1
8175; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8176; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8177; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8178; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8179; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8180; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8181; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8182; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8183; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8184; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8185; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8186; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8187; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8188; GFX10-CU-NEXT:    buffer_gl0_inv
8189; GFX10-CU-NEXT:    buffer_gl1_inv
8190; GFX10-CU-NEXT:    s_endpgm
8191;
8192; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
8193; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8194; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8195; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8196; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8197; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
8198; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
8199; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8200; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8201; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8202; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8203; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8204; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8205; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8206; SKIP-CACHE-INV-NEXT:    s_endpgm
8207;
8208; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
8209; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8210; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8211; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8212; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8213; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8214; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8215; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8216; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8217; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8218; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
8219; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8220;
8221; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
8222; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8223; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8224; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8225; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8226; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8227; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8228; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8229; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8230; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8231; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8232; GFX90A-TGSPLIT-NEXT:    s_endpgm
8233;
8234; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
8235; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8236; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8237; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8238; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8239; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8240; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8241; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
8242; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8243; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8244; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8245; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
8246; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8247;
8248; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
8249; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8250; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8251; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8252; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8253; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8254; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8255; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
8256; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8257; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8258; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8259; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
8260; GFX940-TGSPLIT-NEXT:    s_endpgm
8261    i32* %out, i32 %in, i32 %old) {
8262entry:
8263  %gep = getelementptr i32, i32* %out, i32 4
8264  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel monotonic
8265  ret void
8266}
8267
8268define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_cmpxchg(
8269; GFX7-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
8270; GFX7:       ; %bb.0: ; %entry
8271; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8272; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8273; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8274; GFX7-NEXT:    s_add_u32 s0, s0, 16
8275; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8276; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8277; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8278; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8279; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8280; GFX7-NEXT:    s_waitcnt vmcnt(0)
8281; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8282; GFX7-NEXT:    s_waitcnt vmcnt(0)
8283; GFX7-NEXT:    buffer_wbinvl1_vol
8284; GFX7-NEXT:    s_endpgm
8285;
8286; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
8287; GFX10-WGP:       ; %bb.0: ; %entry
8288; GFX10-WGP-NEXT:    s_clause 0x1
8289; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8290; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8291; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8292; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
8293; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
8294; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8295; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8296; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8297; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8298; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8299; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8300; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8301; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8302; GFX10-WGP-NEXT:    buffer_gl0_inv
8303; GFX10-WGP-NEXT:    buffer_gl1_inv
8304; GFX10-WGP-NEXT:    s_endpgm
8305;
8306; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
8307; GFX10-CU:       ; %bb.0: ; %entry
8308; GFX10-CU-NEXT:    s_clause 0x1
8309; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8310; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8311; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8312; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8313; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8314; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8315; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8316; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8317; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8318; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8319; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8320; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8321; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8322; GFX10-CU-NEXT:    buffer_gl0_inv
8323; GFX10-CU-NEXT:    buffer_gl1_inv
8324; GFX10-CU-NEXT:    s_endpgm
8325;
8326; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
8327; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8328; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8329; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8330; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8331; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
8332; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
8333; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8334; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8335; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8336; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8337; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8338; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8339; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8340; SKIP-CACHE-INV-NEXT:    s_endpgm
8341;
8342; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
8343; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8344; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8345; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8346; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8347; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8348; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8349; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8350; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8351; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8352; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
8353; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8354;
8355; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
8356; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8357; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8358; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8359; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8360; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8361; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8362; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8363; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8364; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8365; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8366; GFX90A-TGSPLIT-NEXT:    s_endpgm
8367;
8368; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
8369; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8370; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8371; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8372; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8373; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8374; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8375; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
8376; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8377; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8378; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8379; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
8380; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8381;
8382; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
8383; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8384; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8385; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8386; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8387; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8388; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8389; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
8390; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8391; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8392; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8393; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
8394; GFX940-TGSPLIT-NEXT:    s_endpgm
8395    i32* %out, i32 %in, i32 %old) {
8396entry:
8397  %gep = getelementptr i32, i32* %out, i32 4
8398  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst monotonic
8399  ret void
8400}
8401
8402define amdgpu_kernel void @flat_agent_one_as_monotonic_acquire_cmpxchg(
8403; GFX7-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
8404; GFX7:       ; %bb.0: ; %entry
8405; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8406; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8407; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8408; GFX7-NEXT:    s_add_u32 s0, s0, 16
8409; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8410; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8411; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8412; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8413; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8414; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8415; GFX7-NEXT:    s_waitcnt vmcnt(0)
8416; GFX7-NEXT:    buffer_wbinvl1_vol
8417; GFX7-NEXT:    s_endpgm
8418;
8419; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
8420; GFX10-WGP:       ; %bb.0: ; %entry
8421; GFX10-WGP-NEXT:    s_clause 0x1
8422; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8423; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8424; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8425; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
8426; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
8427; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8428; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8429; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8430; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8431; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8432; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8433; GFX10-WGP-NEXT:    buffer_gl0_inv
8434; GFX10-WGP-NEXT:    buffer_gl1_inv
8435; GFX10-WGP-NEXT:    s_endpgm
8436;
8437; GFX10-CU-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
8438; GFX10-CU:       ; %bb.0: ; %entry
8439; GFX10-CU-NEXT:    s_clause 0x1
8440; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8441; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8442; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8443; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8444; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8445; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8446; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8447; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8448; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8449; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8450; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8451; GFX10-CU-NEXT:    buffer_gl0_inv
8452; GFX10-CU-NEXT:    buffer_gl1_inv
8453; GFX10-CU-NEXT:    s_endpgm
8454;
8455; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
8456; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8457; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8458; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8459; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8460; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
8461; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
8462; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8463; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8464; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8465; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8466; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8467; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8468; SKIP-CACHE-INV-NEXT:    s_endpgm
8469;
8470; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
8471; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8472; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8473; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8474; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8475; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8476; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8477; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8478; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8479; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
8480; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8481;
8482; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
8483; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8484; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8485; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8486; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8487; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8488; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8489; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8490; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8491; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8492; GFX90A-TGSPLIT-NEXT:    s_endpgm
8493;
8494; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
8495; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8496; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8497; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8498; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8499; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8500; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8501; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8502; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8503; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
8504; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8505;
8506; GFX940-TGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
8507; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8508; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8509; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8510; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8511; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8512; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8513; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8514; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8515; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
8516; GFX940-TGSPLIT-NEXT:    s_endpgm
8517    i32* %out, i32 %in, i32 %old) {
8518entry:
8519  %gep = getelementptr i32, i32* %out, i32 4
8520  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") monotonic acquire
8521  ret void
8522}
8523
8524define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_cmpxchg(
8525; GFX7-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
8526; GFX7:       ; %bb.0: ; %entry
8527; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8528; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8529; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8530; GFX7-NEXT:    s_add_u32 s0, s0, 16
8531; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8532; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8533; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8534; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8535; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8536; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8537; GFX7-NEXT:    s_waitcnt vmcnt(0)
8538; GFX7-NEXT:    buffer_wbinvl1_vol
8539; GFX7-NEXT:    s_endpgm
8540;
8541; GFX10-WGP-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
8542; GFX10-WGP:       ; %bb.0: ; %entry
8543; GFX10-WGP-NEXT:    s_clause 0x1
8544; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8545; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8546; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8547; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
8548; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
8549; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8550; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8551; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8552; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8553; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8554; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8555; GFX10-WGP-NEXT:    buffer_gl0_inv
8556; GFX10-WGP-NEXT:    buffer_gl1_inv
8557; GFX10-WGP-NEXT:    s_endpgm
8558;
8559; GFX10-CU-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
8560; GFX10-CU:       ; %bb.0: ; %entry
8561; GFX10-CU-NEXT:    s_clause 0x1
8562; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8563; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8564; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8565; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8566; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8567; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8568; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8569; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8570; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8571; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8572; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8573; GFX10-CU-NEXT:    buffer_gl0_inv
8574; GFX10-CU-NEXT:    buffer_gl1_inv
8575; GFX10-CU-NEXT:    s_endpgm
8576;
8577; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
8578; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8579; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8580; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8581; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8582; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
8583; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
8584; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8585; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8586; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8587; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8588; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8589; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8590; SKIP-CACHE-INV-NEXT:    s_endpgm
8591;
8592; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
8593; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8594; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8595; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8596; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8597; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8598; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8599; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8600; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8601; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
8602; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8603;
8604; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
8605; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8606; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8607; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8608; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8609; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8610; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8611; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8612; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8613; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8614; GFX90A-TGSPLIT-NEXT:    s_endpgm
8615;
8616; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
8617; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8618; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8619; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8620; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8621; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8622; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8623; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8624; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8625; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
8626; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8627;
8628; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
8629; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8630; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8631; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8632; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8633; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8634; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8635; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8636; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8637; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
8638; GFX940-TGSPLIT-NEXT:    s_endpgm
8639    i32* %out, i32 %in, i32 %old) {
8640entry:
8641  %gep = getelementptr i32, i32* %out, i32 4
8642  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire acquire
8643  ret void
8644}
8645
8646define amdgpu_kernel void @flat_agent_one_as_release_acquire_cmpxchg(
8647; GFX7-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
8648; GFX7:       ; %bb.0: ; %entry
8649; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8650; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8651; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8652; GFX7-NEXT:    s_add_u32 s0, s0, 16
8653; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8654; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8655; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8656; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8657; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8658; GFX7-NEXT:    s_waitcnt vmcnt(0)
8659; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8660; GFX7-NEXT:    s_waitcnt vmcnt(0)
8661; GFX7-NEXT:    buffer_wbinvl1_vol
8662; GFX7-NEXT:    s_endpgm
8663;
8664; GFX10-WGP-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
8665; GFX10-WGP:       ; %bb.0: ; %entry
8666; GFX10-WGP-NEXT:    s_clause 0x1
8667; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8668; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8669; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8670; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
8671; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
8672; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8673; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8674; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8675; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8676; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8677; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8678; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8679; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8680; GFX10-WGP-NEXT:    buffer_gl0_inv
8681; GFX10-WGP-NEXT:    buffer_gl1_inv
8682; GFX10-WGP-NEXT:    s_endpgm
8683;
8684; GFX10-CU-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
8685; GFX10-CU:       ; %bb.0: ; %entry
8686; GFX10-CU-NEXT:    s_clause 0x1
8687; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8688; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8689; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8690; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8691; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8692; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8693; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8694; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8695; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8696; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8697; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8698; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8699; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8700; GFX10-CU-NEXT:    buffer_gl0_inv
8701; GFX10-CU-NEXT:    buffer_gl1_inv
8702; GFX10-CU-NEXT:    s_endpgm
8703;
8704; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
8705; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8706; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8707; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8708; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8709; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
8710; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
8711; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8712; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8713; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8714; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8715; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8716; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8717; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8718; SKIP-CACHE-INV-NEXT:    s_endpgm
8719;
8720; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
8721; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8722; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8723; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8724; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8725; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8726; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8727; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8728; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8729; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8730; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
8731; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8732;
8733; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
8734; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8735; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8736; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8737; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8738; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8739; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8740; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8741; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8742; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8743; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8744; GFX90A-TGSPLIT-NEXT:    s_endpgm
8745;
8746; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
8747; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8748; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8749; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8750; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8751; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8752; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8753; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
8754; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8755; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8756; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8757; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
8758; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8759;
8760; GFX940-TGSPLIT-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
8761; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8762; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8763; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8764; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8765; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8766; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8767; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
8768; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8769; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8770; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8771; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
8772; GFX940-TGSPLIT-NEXT:    s_endpgm
8773    i32* %out, i32 %in, i32 %old) {
8774entry:
8775  %gep = getelementptr i32, i32* %out, i32 4
8776  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release acquire
8777  ret void
8778}
8779
8780define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_cmpxchg(
8781; GFX7-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
8782; GFX7:       ; %bb.0: ; %entry
8783; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8784; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8785; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8786; GFX7-NEXT:    s_add_u32 s0, s0, 16
8787; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8788; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8789; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8790; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8791; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8792; GFX7-NEXT:    s_waitcnt vmcnt(0)
8793; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8794; GFX7-NEXT:    s_waitcnt vmcnt(0)
8795; GFX7-NEXT:    buffer_wbinvl1_vol
8796; GFX7-NEXT:    s_endpgm
8797;
8798; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
8799; GFX10-WGP:       ; %bb.0: ; %entry
8800; GFX10-WGP-NEXT:    s_clause 0x1
8801; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8802; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8803; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8804; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
8805; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
8806; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8807; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8808; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8809; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8810; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8811; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8812; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8813; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8814; GFX10-WGP-NEXT:    buffer_gl0_inv
8815; GFX10-WGP-NEXT:    buffer_gl1_inv
8816; GFX10-WGP-NEXT:    s_endpgm
8817;
8818; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
8819; GFX10-CU:       ; %bb.0: ; %entry
8820; GFX10-CU-NEXT:    s_clause 0x1
8821; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8822; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8823; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8824; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8825; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8826; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8827; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8828; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8829; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8830; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8831; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8832; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8833; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8834; GFX10-CU-NEXT:    buffer_gl0_inv
8835; GFX10-CU-NEXT:    buffer_gl1_inv
8836; GFX10-CU-NEXT:    s_endpgm
8837;
8838; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
8839; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8840; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8841; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8842; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8843; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
8844; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
8845; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8846; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8847; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8848; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8849; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8850; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8851; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8852; SKIP-CACHE-INV-NEXT:    s_endpgm
8853;
8854; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
8855; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8856; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8857; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8858; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8859; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8860; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8861; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8862; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8863; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8864; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
8865; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8866;
8867; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
8868; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8869; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8870; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8871; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8872; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8873; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8874; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8875; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8876; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8877; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8878; GFX90A-TGSPLIT-NEXT:    s_endpgm
8879;
8880; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
8881; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8882; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8883; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8884; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8885; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8886; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8887; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
8888; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8889; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8890; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8891; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
8892; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8893;
8894; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
8895; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8896; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8897; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8898; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8899; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8900; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8901; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
8902; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8903; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8904; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8905; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
8906; GFX940-TGSPLIT-NEXT:    s_endpgm
8907    i32* %out, i32 %in, i32 %old) {
8908entry:
8909  %gep = getelementptr i32, i32* %out, i32 4
8910  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel acquire
8911  ret void
8912}
8913
8914define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_cmpxchg(
8915; GFX7-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
8916; GFX7:       ; %bb.0: ; %entry
8917; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8918; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8919; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8920; GFX7-NEXT:    s_add_u32 s0, s0, 16
8921; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8922; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8923; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8924; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8925; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8926; GFX7-NEXT:    s_waitcnt vmcnt(0)
8927; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8928; GFX7-NEXT:    s_waitcnt vmcnt(0)
8929; GFX7-NEXT:    buffer_wbinvl1_vol
8930; GFX7-NEXT:    s_endpgm
8931;
8932; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
8933; GFX10-WGP:       ; %bb.0: ; %entry
8934; GFX10-WGP-NEXT:    s_clause 0x1
8935; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8936; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8937; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8938; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
8939; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
8940; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8941; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8942; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8943; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8944; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8945; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8946; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8947; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8948; GFX10-WGP-NEXT:    buffer_gl0_inv
8949; GFX10-WGP-NEXT:    buffer_gl1_inv
8950; GFX10-WGP-NEXT:    s_endpgm
8951;
8952; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
8953; GFX10-CU:       ; %bb.0: ; %entry
8954; GFX10-CU-NEXT:    s_clause 0x1
8955; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8956; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8957; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8958; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8959; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8960; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8961; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8962; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8963; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8964; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8965; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8966; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8967; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8968; GFX10-CU-NEXT:    buffer_gl0_inv
8969; GFX10-CU-NEXT:    buffer_gl1_inv
8970; GFX10-CU-NEXT:    s_endpgm
8971;
8972; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
8973; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8974; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8975; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8976; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8977; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
8978; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
8979; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8980; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8981; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8982; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8983; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8984; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8985; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8986; SKIP-CACHE-INV-NEXT:    s_endpgm
8987;
8988; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
8989; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8990; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8991; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8992; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8993; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8994; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8995; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8996; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8997; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8998; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
8999; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9000;
9001; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
9002; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9003; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9004; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9005; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9006; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9007; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9008; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9009; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9010; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9011; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9012; GFX90A-TGSPLIT-NEXT:    s_endpgm
9013;
9014; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
9015; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9016; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9017; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9018; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9019; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9020; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9021; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
9022; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9023; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9024; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9025; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
9026; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9027;
9028; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
9029; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9030; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9031; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9032; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9033; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9034; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9035; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
9036; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9037; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9038; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9039; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
9040; GFX940-TGSPLIT-NEXT:    s_endpgm
9041    i32* %out, i32 %in, i32 %old) {
9042entry:
9043  %gep = getelementptr i32, i32* %out, i32 4
9044  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst acquire
9045  ret void
9046}
9047
9048define amdgpu_kernel void @flat_agent_one_as_monotonic_seq_cst_cmpxchg(
9049; GFX7-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
9050; GFX7:       ; %bb.0: ; %entry
9051; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9052; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9053; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9054; GFX7-NEXT:    s_add_u32 s0, s0, 16
9055; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9056; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9057; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9058; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9059; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9060; GFX7-NEXT:    s_waitcnt vmcnt(0)
9061; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9062; GFX7-NEXT:    s_waitcnt vmcnt(0)
9063; GFX7-NEXT:    buffer_wbinvl1_vol
9064; GFX7-NEXT:    s_endpgm
9065;
9066; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
9067; GFX10-WGP:       ; %bb.0: ; %entry
9068; GFX10-WGP-NEXT:    s_clause 0x1
9069; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9070; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9071; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9072; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9073; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9074; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9075; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9076; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9077; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9078; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9079; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9080; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9081; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9082; GFX10-WGP-NEXT:    buffer_gl0_inv
9083; GFX10-WGP-NEXT:    buffer_gl1_inv
9084; GFX10-WGP-NEXT:    s_endpgm
9085;
9086; GFX10-CU-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
9087; GFX10-CU:       ; %bb.0: ; %entry
9088; GFX10-CU-NEXT:    s_clause 0x1
9089; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9090; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9091; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9092; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9093; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9094; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9095; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9096; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9097; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9098; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9099; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9100; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9101; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9102; GFX10-CU-NEXT:    buffer_gl0_inv
9103; GFX10-CU-NEXT:    buffer_gl1_inv
9104; GFX10-CU-NEXT:    s_endpgm
9105;
9106; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
9107; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9108; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9109; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9110; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9111; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9112; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9113; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9114; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9115; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9116; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9117; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9118; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9119; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9120; SKIP-CACHE-INV-NEXT:    s_endpgm
9121;
9122; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
9123; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9124; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9125; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9126; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9127; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9128; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9129; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9130; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9131; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9132; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9133; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9134;
9135; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
9136; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9137; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9138; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9139; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9140; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9141; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9142; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9143; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9144; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9145; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9146; GFX90A-TGSPLIT-NEXT:    s_endpgm
9147;
9148; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
9149; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9150; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9151; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9152; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9153; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9154; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9155; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
9156; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9157; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9158; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9159; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
9160; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9161;
9162; GFX940-TGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
9163; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9164; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9165; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9166; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9167; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9168; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9169; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
9170; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9171; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9172; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9173; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
9174; GFX940-TGSPLIT-NEXT:    s_endpgm
9175    i32* %out, i32 %in, i32 %old) {
9176entry:
9177  %gep = getelementptr i32, i32* %out, i32 4
9178  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") monotonic seq_cst
9179  ret void
9180}
9181
9182define amdgpu_kernel void @flat_agent_one_as_acquire_seq_cst_cmpxchg(
9183; GFX7-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
9184; GFX7:       ; %bb.0: ; %entry
9185; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9186; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9187; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9188; GFX7-NEXT:    s_add_u32 s0, s0, 16
9189; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9190; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9191; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9192; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9193; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9194; GFX7-NEXT:    s_waitcnt vmcnt(0)
9195; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9196; GFX7-NEXT:    s_waitcnt vmcnt(0)
9197; GFX7-NEXT:    buffer_wbinvl1_vol
9198; GFX7-NEXT:    s_endpgm
9199;
9200; GFX10-WGP-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
9201; GFX10-WGP:       ; %bb.0: ; %entry
9202; GFX10-WGP-NEXT:    s_clause 0x1
9203; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9204; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9205; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9206; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9207; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9208; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9209; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9210; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9211; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9212; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9213; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9214; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9215; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9216; GFX10-WGP-NEXT:    buffer_gl0_inv
9217; GFX10-WGP-NEXT:    buffer_gl1_inv
9218; GFX10-WGP-NEXT:    s_endpgm
9219;
9220; GFX10-CU-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
9221; GFX10-CU:       ; %bb.0: ; %entry
9222; GFX10-CU-NEXT:    s_clause 0x1
9223; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9224; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9225; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9226; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9227; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9228; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9229; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9230; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9231; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9232; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9233; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9234; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9235; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9236; GFX10-CU-NEXT:    buffer_gl0_inv
9237; GFX10-CU-NEXT:    buffer_gl1_inv
9238; GFX10-CU-NEXT:    s_endpgm
9239;
9240; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
9241; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9242; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9243; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9244; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9245; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9246; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9247; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9248; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9249; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9250; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9251; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9252; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9253; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9254; SKIP-CACHE-INV-NEXT:    s_endpgm
9255;
9256; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
9257; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9258; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9259; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9260; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9261; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9262; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9263; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9264; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9265; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9266; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9267; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9268;
9269; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
9270; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9271; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9272; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9273; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9274; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9275; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9276; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9277; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9278; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9279; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9280; GFX90A-TGSPLIT-NEXT:    s_endpgm
9281;
9282; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
9283; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9284; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9285; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9286; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9287; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9288; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9289; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
9290; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9291; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9292; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9293; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
9294; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9295;
9296; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
9297; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9298; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9299; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9300; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9301; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9302; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9303; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
9304; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9305; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9306; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9307; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
9308; GFX940-TGSPLIT-NEXT:    s_endpgm
9309    i32* %out, i32 %in, i32 %old) {
9310entry:
9311  %gep = getelementptr i32, i32* %out, i32 4
9312  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire seq_cst
9313  ret void
9314}
9315
9316define amdgpu_kernel void @flat_agent_one_as_release_seq_cst_cmpxchg(
9317; GFX7-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
9318; GFX7:       ; %bb.0: ; %entry
9319; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9320; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9321; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9322; GFX7-NEXT:    s_add_u32 s0, s0, 16
9323; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9324; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9325; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9326; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9327; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9328; GFX7-NEXT:    s_waitcnt vmcnt(0)
9329; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9330; GFX7-NEXT:    s_waitcnt vmcnt(0)
9331; GFX7-NEXT:    buffer_wbinvl1_vol
9332; GFX7-NEXT:    s_endpgm
9333;
9334; GFX10-WGP-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
9335; GFX10-WGP:       ; %bb.0: ; %entry
9336; GFX10-WGP-NEXT:    s_clause 0x1
9337; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9338; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9339; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9340; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9341; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9342; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9343; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9344; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9345; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9346; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9347; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9348; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9349; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9350; GFX10-WGP-NEXT:    buffer_gl0_inv
9351; GFX10-WGP-NEXT:    buffer_gl1_inv
9352; GFX10-WGP-NEXT:    s_endpgm
9353;
9354; GFX10-CU-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
9355; GFX10-CU:       ; %bb.0: ; %entry
9356; GFX10-CU-NEXT:    s_clause 0x1
9357; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9358; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9359; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9360; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9361; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9362; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9363; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9364; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9365; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9366; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9367; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9368; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9369; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9370; GFX10-CU-NEXT:    buffer_gl0_inv
9371; GFX10-CU-NEXT:    buffer_gl1_inv
9372; GFX10-CU-NEXT:    s_endpgm
9373;
9374; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
9375; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9376; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9377; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9378; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9379; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9380; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9381; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9382; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9383; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9384; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9385; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9386; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9387; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9388; SKIP-CACHE-INV-NEXT:    s_endpgm
9389;
9390; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
9391; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9392; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9393; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9394; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9395; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9396; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9397; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9398; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9399; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9400; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9401; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9402;
9403; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
9404; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9405; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9406; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9407; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9408; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9409; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9410; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9411; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9412; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9413; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9414; GFX90A-TGSPLIT-NEXT:    s_endpgm
9415;
9416; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
9417; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9418; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9419; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9420; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9421; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9422; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9423; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
9424; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9425; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9426; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9427; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
9428; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9429;
9430; GFX940-TGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
9431; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9432; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9433; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9434; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9435; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9436; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9437; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
9438; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9439; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9440; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9441; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
9442; GFX940-TGSPLIT-NEXT:    s_endpgm
9443    i32* %out, i32 %in, i32 %old) {
9444entry:
9445  %gep = getelementptr i32, i32* %out, i32 4
9446  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release seq_cst
9447  ret void
9448}
9449
9450define amdgpu_kernel void @flat_agent_one_as_acq_rel_seq_cst_cmpxchg(
9451; GFX7-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
9452; GFX7:       ; %bb.0: ; %entry
9453; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9454; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9455; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9456; GFX7-NEXT:    s_add_u32 s0, s0, 16
9457; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9458; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9459; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9460; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9461; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9462; GFX7-NEXT:    s_waitcnt vmcnt(0)
9463; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9464; GFX7-NEXT:    s_waitcnt vmcnt(0)
9465; GFX7-NEXT:    buffer_wbinvl1_vol
9466; GFX7-NEXT:    s_endpgm
9467;
9468; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
9469; GFX10-WGP:       ; %bb.0: ; %entry
9470; GFX10-WGP-NEXT:    s_clause 0x1
9471; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9472; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9473; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9474; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9475; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9476; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9477; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9478; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9479; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9480; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9481; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9482; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9483; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9484; GFX10-WGP-NEXT:    buffer_gl0_inv
9485; GFX10-WGP-NEXT:    buffer_gl1_inv
9486; GFX10-WGP-NEXT:    s_endpgm
9487;
9488; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
9489; GFX10-CU:       ; %bb.0: ; %entry
9490; GFX10-CU-NEXT:    s_clause 0x1
9491; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9492; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9493; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9494; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9495; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9496; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9497; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9498; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9499; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9500; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9501; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9502; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9503; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9504; GFX10-CU-NEXT:    buffer_gl0_inv
9505; GFX10-CU-NEXT:    buffer_gl1_inv
9506; GFX10-CU-NEXT:    s_endpgm
9507;
9508; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
9509; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9510; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9511; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9512; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9513; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9514; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9515; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9516; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9517; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9518; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9519; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9520; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9521; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9522; SKIP-CACHE-INV-NEXT:    s_endpgm
9523;
9524; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
9525; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9526; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9527; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9528; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9529; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9530; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9531; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9532; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9533; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9534; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9535; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9536;
9537; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
9538; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9539; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9540; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9541; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9542; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9543; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9544; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9545; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9546; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9547; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9548; GFX90A-TGSPLIT-NEXT:    s_endpgm
9549;
9550; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
9551; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9552; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9553; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9554; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9555; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9556; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9557; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
9558; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9559; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9560; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9561; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
9562; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9563;
9564; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
9565; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9566; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9567; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9568; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9569; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9570; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9571; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
9572; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9573; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9574; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9575; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
9576; GFX940-TGSPLIT-NEXT:    s_endpgm
9577    i32* %out, i32 %in, i32 %old) {
9578entry:
9579  %gep = getelementptr i32, i32* %out, i32 4
9580  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel seq_cst
9581  ret void
9582}
9583
9584define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_cmpxchg(
9585; GFX7-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
9586; GFX7:       ; %bb.0: ; %entry
9587; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9588; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9589; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9590; GFX7-NEXT:    s_add_u32 s0, s0, 16
9591; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9592; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9593; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9594; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9595; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9596; GFX7-NEXT:    s_waitcnt vmcnt(0)
9597; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9598; GFX7-NEXT:    s_waitcnt vmcnt(0)
9599; GFX7-NEXT:    buffer_wbinvl1_vol
9600; GFX7-NEXT:    s_endpgm
9601;
9602; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
9603; GFX10-WGP:       ; %bb.0: ; %entry
9604; GFX10-WGP-NEXT:    s_clause 0x1
9605; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9606; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9607; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9608; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9609; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9610; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9611; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9612; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9613; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9614; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9615; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9616; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9617; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9618; GFX10-WGP-NEXT:    buffer_gl0_inv
9619; GFX10-WGP-NEXT:    buffer_gl1_inv
9620; GFX10-WGP-NEXT:    s_endpgm
9621;
9622; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
9623; GFX10-CU:       ; %bb.0: ; %entry
9624; GFX10-CU-NEXT:    s_clause 0x1
9625; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9626; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9627; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9628; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9629; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9630; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9631; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9632; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9633; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9634; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9635; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9636; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9637; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9638; GFX10-CU-NEXT:    buffer_gl0_inv
9639; GFX10-CU-NEXT:    buffer_gl1_inv
9640; GFX10-CU-NEXT:    s_endpgm
9641;
9642; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
9643; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9644; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9645; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9646; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9647; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9648; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9649; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9650; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9651; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9652; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9653; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9654; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9655; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9656; SKIP-CACHE-INV-NEXT:    s_endpgm
9657;
9658; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
9659; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9660; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9661; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9662; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9663; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9664; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9665; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9666; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9667; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9668; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9669; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9670;
9671; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
9672; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9673; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9674; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9675; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9676; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9677; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9678; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9679; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9680; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9681; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9682; GFX90A-TGSPLIT-NEXT:    s_endpgm
9683;
9684; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
9685; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9686; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9687; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9688; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9689; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9690; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9691; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
9692; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9693; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9694; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9695; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
9696; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9697;
9698; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
9699; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9700; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9701; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9702; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9703; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9704; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9705; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
9706; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9707; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9708; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9709; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
9710; GFX940-TGSPLIT-NEXT:    s_endpgm
9711    i32* %out, i32 %in, i32 %old) {
9712entry:
9713  %gep = getelementptr i32, i32* %out, i32 4
9714  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst seq_cst
9715  ret void
9716}
9717
9718define amdgpu_kernel void @flat_agent_one_as_monotonic_monotonic_ret_cmpxchg(
9719; GFX7-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
9720; GFX7:       ; %bb.0: ; %entry
9721; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9722; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9723; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9724; GFX7-NEXT:    s_add_u32 s4, s0, 16
9725; GFX7-NEXT:    s_addc_u32 s5, s1, 0
9726; GFX7-NEXT:    v_mov_b32_e32 v0, s4
9727; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9728; GFX7-NEXT:    v_mov_b32_e32 v1, s5
9729; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9730; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9731; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9732; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9733; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9734; GFX7-NEXT:    flat_store_dword v[0:1], v2
9735; GFX7-NEXT:    s_endpgm
9736;
9737; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
9738; GFX10-WGP:       ; %bb.0: ; %entry
9739; GFX10-WGP-NEXT:    s_clause 0x1
9740; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9741; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9742; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9743; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
9744; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
9745; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
9746; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9747; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
9748; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9749; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9750; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9751; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9752; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9753; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
9754; GFX10-WGP-NEXT:    s_endpgm
9755;
9756; GFX10-CU-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
9757; GFX10-CU:       ; %bb.0: ; %entry
9758; GFX10-CU-NEXT:    s_clause 0x1
9759; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9760; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9761; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9762; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
9763; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
9764; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
9765; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9766; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
9767; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9768; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9769; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9770; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9771; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9772; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
9773; GFX10-CU-NEXT:    s_endpgm
9774;
9775; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
9776; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9777; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9778; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9779; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9780; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
9781; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
9782; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
9783; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9784; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
9785; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9786; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9787; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9788; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9789; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9790; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
9791; SKIP-CACHE-INV-NEXT:    s_endpgm
9792;
9793; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
9794; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9795; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9796; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9797; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9798; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9799; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9800; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9801; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9802; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9803; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9804;
9805; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
9806; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9807; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9808; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9809; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9810; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9811; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9812; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9813; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9814; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9815; GFX90A-TGSPLIT-NEXT:    s_endpgm
9816;
9817; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
9818; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9819; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9820; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9821; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9822; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9823; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9824; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9825; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9826; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9827; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9828;
9829; GFX940-TGSPLIT-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
9830; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9831; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9832; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9833; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9834; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9835; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9836; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9837; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9838; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9839; GFX940-TGSPLIT-NEXT:    s_endpgm
9840    i32* %out, i32 %in, i32 %old) {
9841entry:
9842  %gep = getelementptr i32, i32* %out, i32 4
9843  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") monotonic monotonic
9844  %val0 = extractvalue { i32, i1 } %val, 0
9845  store i32 %val0, i32* %out, align 4
9846  ret void
9847}
9848
9849define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_ret_cmpxchg(
9850; GFX7-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
9851; GFX7:       ; %bb.0: ; %entry
9852; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9853; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9854; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9855; GFX7-NEXT:    s_add_u32 s4, s0, 16
9856; GFX7-NEXT:    s_addc_u32 s5, s1, 0
9857; GFX7-NEXT:    v_mov_b32_e32 v0, s4
9858; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9859; GFX7-NEXT:    v_mov_b32_e32 v1, s5
9860; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9861; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9862; GFX7-NEXT:    s_waitcnt vmcnt(0)
9863; GFX7-NEXT:    buffer_wbinvl1_vol
9864; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9865; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9866; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9867; GFX7-NEXT:    flat_store_dword v[0:1], v2
9868; GFX7-NEXT:    s_endpgm
9869;
9870; GFX10-WGP-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
9871; GFX10-WGP:       ; %bb.0: ; %entry
9872; GFX10-WGP-NEXT:    s_clause 0x1
9873; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9874; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9875; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9876; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
9877; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
9878; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
9879; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9880; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
9881; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9882; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9883; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9884; GFX10-WGP-NEXT:    buffer_gl0_inv
9885; GFX10-WGP-NEXT:    buffer_gl1_inv
9886; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9887; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9888; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9889; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
9890; GFX10-WGP-NEXT:    s_endpgm
9891;
9892; GFX10-CU-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
9893; GFX10-CU:       ; %bb.0: ; %entry
9894; GFX10-CU-NEXT:    s_clause 0x1
9895; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9896; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9897; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9898; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
9899; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
9900; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
9901; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9902; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
9903; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9904; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9905; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9906; GFX10-CU-NEXT:    buffer_gl0_inv
9907; GFX10-CU-NEXT:    buffer_gl1_inv
9908; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9909; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9910; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9911; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
9912; GFX10-CU-NEXT:    s_endpgm
9913;
9914; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
9915; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9916; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9917; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9918; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9919; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
9920; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
9921; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
9922; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9923; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
9924; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9925; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9926; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9927; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9928; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9929; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9930; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
9931; SKIP-CACHE-INV-NEXT:    s_endpgm
9932;
9933; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
9934; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9935; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9936; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9937; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9938; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9939; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9940; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9941; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9942; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9943; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9944; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9945; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9946;
9947; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
9948; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9949; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9950; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9951; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9952; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9953; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9954; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9955; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9956; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9957; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9958; GFX90A-TGSPLIT-NEXT:    s_endpgm
9959;
9960; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
9961; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9962; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9963; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9964; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9965; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9966; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9967; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9968; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9969; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
9970; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9971; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9972; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9973;
9974; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
9975; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9976; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9977; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9978; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9979; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9980; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9981; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9982; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9983; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
9984; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
9985; GFX940-TGSPLIT-NEXT:    s_endpgm
9986    i32* %out, i32 %in, i32 %old) {
9987entry:
9988  %gep = getelementptr i32, i32* %out, i32 4
9989  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire monotonic
9990  %val0 = extractvalue { i32, i1 } %val, 0
9991  store i32 %val0, i32* %out, align 4
9992  ret void
9993}
9994
9995define amdgpu_kernel void @flat_agent_one_as_release_monotonic_ret_cmpxchg(
9996; GFX7-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
9997; GFX7:       ; %bb.0: ; %entry
9998; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9999; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10000; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10001; GFX7-NEXT:    s_add_u32 s4, s0, 16
10002; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10003; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10004; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10005; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10006; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10007; GFX7-NEXT:    s_waitcnt vmcnt(0)
10008; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10009; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10010; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10011; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10012; GFX7-NEXT:    flat_store_dword v[0:1], v2
10013; GFX7-NEXT:    s_endpgm
10014;
10015; GFX10-WGP-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
10016; GFX10-WGP:       ; %bb.0: ; %entry
10017; GFX10-WGP-NEXT:    s_clause 0x1
10018; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10019; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10020; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10021; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
10022; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
10023; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
10024; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10025; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
10026; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10027; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10028; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10029; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10030; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10031; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10032; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10033; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
10034; GFX10-WGP-NEXT:    s_endpgm
10035;
10036; GFX10-CU-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
10037; GFX10-CU:       ; %bb.0: ; %entry
10038; GFX10-CU-NEXT:    s_clause 0x1
10039; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10040; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10041; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10042; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
10043; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
10044; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
10045; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10046; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
10047; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10048; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10049; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10050; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10051; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10052; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10053; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10054; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
10055; GFX10-CU-NEXT:    s_endpgm
10056;
10057; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
10058; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10059; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10060; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10061; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10062; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
10063; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
10064; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10065; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10066; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10067; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10068; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10069; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10070; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10071; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10072; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10073; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
10074; SKIP-CACHE-INV-NEXT:    s_endpgm
10075;
10076; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
10077; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10078; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10079; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10080; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10081; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10082; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10083; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10084; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10085; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10086; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10087; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10088;
10089; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
10090; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10091; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10092; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10093; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10094; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10095; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10096; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10097; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10098; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10099; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10100; GFX90A-TGSPLIT-NEXT:    s_endpgm
10101;
10102; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
10103; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10104; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10105; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10106; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10107; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10108; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10109; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
10110; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10111; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10112; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10113; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10114; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10115;
10116; GFX940-TGSPLIT-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
10117; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10118; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10119; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10120; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10121; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10122; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10123; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
10124; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10125; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10126; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10127; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10128; GFX940-TGSPLIT-NEXT:    s_endpgm
10129    i32* %out, i32 %in, i32 %old) {
10130entry:
10131  %gep = getelementptr i32, i32* %out, i32 4
10132  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release monotonic
10133  %val0 = extractvalue { i32, i1 } %val, 0
10134  store i32 %val0, i32* %out, align 4
10135  ret void
10136}
10137
10138define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg(
10139; GFX7-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
10140; GFX7:       ; %bb.0: ; %entry
10141; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10142; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10143; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10144; GFX7-NEXT:    s_add_u32 s4, s0, 16
10145; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10146; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10147; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10148; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10149; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10150; GFX7-NEXT:    s_waitcnt vmcnt(0)
10151; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10152; GFX7-NEXT:    s_waitcnt vmcnt(0)
10153; GFX7-NEXT:    buffer_wbinvl1_vol
10154; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10155; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10156; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10157; GFX7-NEXT:    flat_store_dword v[0:1], v2
10158; GFX7-NEXT:    s_endpgm
10159;
10160; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
10161; GFX10-WGP:       ; %bb.0: ; %entry
10162; GFX10-WGP-NEXT:    s_clause 0x1
10163; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10164; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10165; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10166; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
10167; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
10168; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
10169; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10170; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
10171; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10172; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10173; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10174; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10175; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10176; GFX10-WGP-NEXT:    buffer_gl0_inv
10177; GFX10-WGP-NEXT:    buffer_gl1_inv
10178; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10179; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10180; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10181; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
10182; GFX10-WGP-NEXT:    s_endpgm
10183;
10184; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
10185; GFX10-CU:       ; %bb.0: ; %entry
10186; GFX10-CU-NEXT:    s_clause 0x1
10187; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10188; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10189; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10190; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
10191; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
10192; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
10193; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10194; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
10195; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10196; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10197; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10198; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10199; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10200; GFX10-CU-NEXT:    buffer_gl0_inv
10201; GFX10-CU-NEXT:    buffer_gl1_inv
10202; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10203; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10204; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10205; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
10206; GFX10-CU-NEXT:    s_endpgm
10207;
10208; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
10209; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10210; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10211; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10212; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10213; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
10214; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
10215; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10216; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10217; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10218; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10219; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10220; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10221; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10222; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10223; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10224; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10225; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
10226; SKIP-CACHE-INV-NEXT:    s_endpgm
10227;
10228; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
10229; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10230; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10231; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10232; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10233; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10234; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10235; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10236; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10237; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10238; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
10239; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10240; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10241; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10242;
10243; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
10244; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10245; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10246; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10247; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10248; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10249; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10250; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10251; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10252; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10253; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10254; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10255; GFX90A-TGSPLIT-NEXT:    s_endpgm
10256;
10257; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
10258; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10259; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10260; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10261; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10262; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10263; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10264; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
10265; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10266; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10267; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10268; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
10269; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10270; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10271; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10272;
10273; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
10274; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10275; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10276; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10277; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10278; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10279; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10280; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
10281; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10282; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10283; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10284; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
10285; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10286; GFX940-TGSPLIT-NEXT:    s_endpgm
10287    i32* %out, i32 %in, i32 %old) {
10288entry:
10289  %gep = getelementptr i32, i32* %out, i32 4
10290  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel monotonic
10291  %val0 = extractvalue { i32, i1 } %val, 0
10292  store i32 %val0, i32* %out, align 4
10293  ret void
10294}
10295
10296define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg(
10297; GFX7-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
10298; GFX7:       ; %bb.0: ; %entry
10299; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10300; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10301; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10302; GFX7-NEXT:    s_add_u32 s4, s0, 16
10303; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10304; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10305; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10306; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10307; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10308; GFX7-NEXT:    s_waitcnt vmcnt(0)
10309; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10310; GFX7-NEXT:    s_waitcnt vmcnt(0)
10311; GFX7-NEXT:    buffer_wbinvl1_vol
10312; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10313; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10314; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10315; GFX7-NEXT:    flat_store_dword v[0:1], v2
10316; GFX7-NEXT:    s_endpgm
10317;
10318; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
10319; GFX10-WGP:       ; %bb.0: ; %entry
10320; GFX10-WGP-NEXT:    s_clause 0x1
10321; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10322; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10323; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10324; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
10325; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
10326; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
10327; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10328; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
10329; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10330; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10331; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10332; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10333; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10334; GFX10-WGP-NEXT:    buffer_gl0_inv
10335; GFX10-WGP-NEXT:    buffer_gl1_inv
10336; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10337; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10338; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10339; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
10340; GFX10-WGP-NEXT:    s_endpgm
10341;
10342; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
10343; GFX10-CU:       ; %bb.0: ; %entry
10344; GFX10-CU-NEXT:    s_clause 0x1
10345; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10346; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10347; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10348; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
10349; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
10350; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
10351; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10352; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
10353; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10354; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10355; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10356; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10357; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10358; GFX10-CU-NEXT:    buffer_gl0_inv
10359; GFX10-CU-NEXT:    buffer_gl1_inv
10360; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10361; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10362; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10363; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
10364; GFX10-CU-NEXT:    s_endpgm
10365;
10366; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
10367; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10368; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10369; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10370; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10371; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
10372; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
10373; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10374; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10375; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10376; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10377; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10378; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10379; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10380; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10381; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10382; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10383; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
10384; SKIP-CACHE-INV-NEXT:    s_endpgm
10385;
10386; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
10387; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10388; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10389; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10390; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10391; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10392; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10393; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10394; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10395; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10396; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
10397; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10398; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10399; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10400;
10401; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
10402; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10403; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10404; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10405; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10406; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10407; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10408; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10409; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10410; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10411; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10412; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10413; GFX90A-TGSPLIT-NEXT:    s_endpgm
10414;
10415; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
10416; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10417; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10418; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10419; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10420; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10421; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10422; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
10423; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10424; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10425; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10426; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
10427; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10428; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10429; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10430;
10431; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
10432; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10433; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10434; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10435; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10436; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10437; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10438; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
10439; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10440; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10441; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10442; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
10443; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10444; GFX940-TGSPLIT-NEXT:    s_endpgm
10445    i32* %out, i32 %in, i32 %old) {
10446entry:
10447  %gep = getelementptr i32, i32* %out, i32 4
10448  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst monotonic
10449  %val0 = extractvalue { i32, i1 } %val, 0
10450  store i32 %val0, i32* %out, align 4
10451  ret void
10452}
10453
10454define amdgpu_kernel void @flat_agent_one_as_monotonic_acquire_ret_cmpxchg(
10455; GFX7-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
10456; GFX7:       ; %bb.0: ; %entry
10457; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10458; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10459; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10460; GFX7-NEXT:    s_add_u32 s4, s0, 16
10461; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10462; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10463; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10464; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10465; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10466; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10467; GFX7-NEXT:    s_waitcnt vmcnt(0)
10468; GFX7-NEXT:    buffer_wbinvl1_vol
10469; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10470; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10471; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10472; GFX7-NEXT:    flat_store_dword v[0:1], v2
10473; GFX7-NEXT:    s_endpgm
10474;
10475; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
10476; GFX10-WGP:       ; %bb.0: ; %entry
10477; GFX10-WGP-NEXT:    s_clause 0x1
10478; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10479; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10480; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10481; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
10482; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
10483; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
10484; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10485; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
10486; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10487; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10488; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10489; GFX10-WGP-NEXT:    buffer_gl0_inv
10490; GFX10-WGP-NEXT:    buffer_gl1_inv
10491; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10492; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10493; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10494; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
10495; GFX10-WGP-NEXT:    s_endpgm
10496;
10497; GFX10-CU-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
10498; GFX10-CU:       ; %bb.0: ; %entry
10499; GFX10-CU-NEXT:    s_clause 0x1
10500; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10501; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10502; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10503; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
10504; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
10505; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
10506; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10507; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
10508; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10509; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10510; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10511; GFX10-CU-NEXT:    buffer_gl0_inv
10512; GFX10-CU-NEXT:    buffer_gl1_inv
10513; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10514; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10515; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10516; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
10517; GFX10-CU-NEXT:    s_endpgm
10518;
10519; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
10520; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10521; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10522; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10523; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10524; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
10525; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
10526; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10527; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10528; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10529; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10530; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10531; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10532; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10533; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10534; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10535; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
10536; SKIP-CACHE-INV-NEXT:    s_endpgm
10537;
10538; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
10539; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10540; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10541; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10542; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10543; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10544; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10545; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10546; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10547; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
10548; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10549; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10550; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10551;
10552; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
10553; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10554; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10555; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10556; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10557; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10558; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10559; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10560; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10561; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10562; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10563; GFX90A-TGSPLIT-NEXT:    s_endpgm
10564;
10565; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
10566; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10567; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10568; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10569; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10570; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10571; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10572; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10573; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10574; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
10575; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10576; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10577; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10578;
10579; GFX940-TGSPLIT-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
10580; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10581; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10582; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10583; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10584; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10585; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10586; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10587; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10588; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
10589; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10590; GFX940-TGSPLIT-NEXT:    s_endpgm
10591    i32* %out, i32 %in, i32 %old) {
10592entry:
10593  %gep = getelementptr i32, i32* %out, i32 4
10594  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") monotonic acquire
10595  %val0 = extractvalue { i32, i1 } %val, 0
10596  store i32 %val0, i32* %out, align 4
10597  ret void
10598}
10599
10600define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_ret_cmpxchg(
10601; GFX7-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
10602; GFX7:       ; %bb.0: ; %entry
10603; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10604; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10605; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10606; GFX7-NEXT:    s_add_u32 s4, s0, 16
10607; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10608; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10609; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10610; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10611; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10612; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10613; GFX7-NEXT:    s_waitcnt vmcnt(0)
10614; GFX7-NEXT:    buffer_wbinvl1_vol
10615; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10616; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10617; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10618; GFX7-NEXT:    flat_store_dword v[0:1], v2
10619; GFX7-NEXT:    s_endpgm
10620;
10621; GFX10-WGP-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
10622; GFX10-WGP:       ; %bb.0: ; %entry
10623; GFX10-WGP-NEXT:    s_clause 0x1
10624; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10625; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10626; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10627; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
10628; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
10629; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
10630; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10631; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
10632; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10633; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10634; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10635; GFX10-WGP-NEXT:    buffer_gl0_inv
10636; GFX10-WGP-NEXT:    buffer_gl1_inv
10637; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10638; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10639; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10640; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
10641; GFX10-WGP-NEXT:    s_endpgm
10642;
10643; GFX10-CU-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
10644; GFX10-CU:       ; %bb.0: ; %entry
10645; GFX10-CU-NEXT:    s_clause 0x1
10646; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10647; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10648; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10649; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
10650; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
10651; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
10652; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10653; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
10654; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10655; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10656; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10657; GFX10-CU-NEXT:    buffer_gl0_inv
10658; GFX10-CU-NEXT:    buffer_gl1_inv
10659; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10660; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10661; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10662; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
10663; GFX10-CU-NEXT:    s_endpgm
10664;
10665; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
10666; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10667; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10668; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10669; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10670; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
10671; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
10672; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10673; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10674; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10675; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10676; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10677; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10678; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10679; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10680; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10681; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
10682; SKIP-CACHE-INV-NEXT:    s_endpgm
10683;
10684; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
10685; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10686; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10687; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10688; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10689; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10690; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10691; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10692; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10693; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
10694; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10695; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10696; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10697;
10698; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
10699; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10700; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10701; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10702; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10703; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10704; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10705; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10706; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10707; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10708; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10709; GFX90A-TGSPLIT-NEXT:    s_endpgm
10710;
10711; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
10712; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10713; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10714; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10715; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10716; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10717; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10718; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10719; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10720; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
10721; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10722; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10723; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10724;
10725; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
10726; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10727; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10728; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10729; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10730; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10731; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10732; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10733; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10734; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
10735; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10736; GFX940-TGSPLIT-NEXT:    s_endpgm
10737    i32* %out, i32 %in, i32 %old) {
10738entry:
10739  %gep = getelementptr i32, i32* %out, i32 4
10740  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire acquire
10741  %val0 = extractvalue { i32, i1 } %val, 0
10742  store i32 %val0, i32* %out, align 4
10743  ret void
10744}
10745
10746define amdgpu_kernel void @flat_agent_one_as_release_acquire_ret_cmpxchg(
10747; GFX7-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
10748; GFX7:       ; %bb.0: ; %entry
10749; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10750; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10751; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10752; GFX7-NEXT:    s_add_u32 s4, s0, 16
10753; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10754; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10755; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10756; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10757; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10758; GFX7-NEXT:    s_waitcnt vmcnt(0)
10759; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10760; GFX7-NEXT:    s_waitcnt vmcnt(0)
10761; GFX7-NEXT:    buffer_wbinvl1_vol
10762; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10763; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10764; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10765; GFX7-NEXT:    flat_store_dword v[0:1], v2
10766; GFX7-NEXT:    s_endpgm
10767;
10768; GFX10-WGP-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
10769; GFX10-WGP:       ; %bb.0: ; %entry
10770; GFX10-WGP-NEXT:    s_clause 0x1
10771; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10772; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10773; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10774; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
10775; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
10776; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
10777; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10778; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
10779; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10780; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10781; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10782; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10783; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10784; GFX10-WGP-NEXT:    buffer_gl0_inv
10785; GFX10-WGP-NEXT:    buffer_gl1_inv
10786; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10787; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10788; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10789; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
10790; GFX10-WGP-NEXT:    s_endpgm
10791;
10792; GFX10-CU-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
10793; GFX10-CU:       ; %bb.0: ; %entry
10794; GFX10-CU-NEXT:    s_clause 0x1
10795; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10796; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10797; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10798; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
10799; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
10800; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
10801; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10802; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
10803; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10804; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10805; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10806; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10807; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10808; GFX10-CU-NEXT:    buffer_gl0_inv
10809; GFX10-CU-NEXT:    buffer_gl1_inv
10810; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10811; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10812; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10813; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
10814; GFX10-CU-NEXT:    s_endpgm
10815;
10816; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
10817; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10818; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10819; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10820; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10821; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
10822; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
10823; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10824; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10825; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10826; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10827; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10828; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10829; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10830; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10831; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10832; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10833; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
10834; SKIP-CACHE-INV-NEXT:    s_endpgm
10835;
10836; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
10837; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10838; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10839; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10840; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10841; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10842; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10843; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10844; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10845; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10846; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
10847; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10848; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10849; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10850;
10851; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
10852; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10853; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10854; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10855; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10856; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10857; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10858; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10859; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10860; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10861; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10862; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10863; GFX90A-TGSPLIT-NEXT:    s_endpgm
10864;
10865; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
10866; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10867; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10868; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10869; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10870; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10871; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10872; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
10873; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10874; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10875; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10876; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
10877; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10878; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10879; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10880;
10881; GFX940-TGSPLIT-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
10882; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10883; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10884; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10885; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10886; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10887; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10888; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
10889; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10890; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10891; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10892; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
10893; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10894; GFX940-TGSPLIT-NEXT:    s_endpgm
10895    i32* %out, i32 %in, i32 %old) {
10896entry:
10897  %gep = getelementptr i32, i32* %out, i32 4
10898  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release acquire
10899  %val0 = extractvalue { i32, i1 } %val, 0
10900  store i32 %val0, i32* %out, align 4
10901  ret void
10902}
10903
10904define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_ret_cmpxchg(
10905; GFX7-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
10906; GFX7:       ; %bb.0: ; %entry
10907; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10908; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10909; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10910; GFX7-NEXT:    s_add_u32 s4, s0, 16
10911; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10912; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10913; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10914; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10915; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10916; GFX7-NEXT:    s_waitcnt vmcnt(0)
10917; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10918; GFX7-NEXT:    s_waitcnt vmcnt(0)
10919; GFX7-NEXT:    buffer_wbinvl1_vol
10920; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10921; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10922; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10923; GFX7-NEXT:    flat_store_dword v[0:1], v2
10924; GFX7-NEXT:    s_endpgm
10925;
10926; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
10927; GFX10-WGP:       ; %bb.0: ; %entry
10928; GFX10-WGP-NEXT:    s_clause 0x1
10929; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10930; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10931; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10932; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
10933; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
10934; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
10935; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10936; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
10937; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10938; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10939; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10940; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10941; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10942; GFX10-WGP-NEXT:    buffer_gl0_inv
10943; GFX10-WGP-NEXT:    buffer_gl1_inv
10944; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10945; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10946; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10947; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
10948; GFX10-WGP-NEXT:    s_endpgm
10949;
10950; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
10951; GFX10-CU:       ; %bb.0: ; %entry
10952; GFX10-CU-NEXT:    s_clause 0x1
10953; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10954; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10955; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10956; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
10957; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
10958; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
10959; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10960; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
10961; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10962; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10963; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10964; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10965; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10966; GFX10-CU-NEXT:    buffer_gl0_inv
10967; GFX10-CU-NEXT:    buffer_gl1_inv
10968; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10969; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10970; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10971; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
10972; GFX10-CU-NEXT:    s_endpgm
10973;
10974; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
10975; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10976; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10977; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10978; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10979; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
10980; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
10981; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10982; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10983; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10984; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10985; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10986; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10987; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10988; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10989; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10990; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10991; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
10992; SKIP-CACHE-INV-NEXT:    s_endpgm
10993;
10994; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
10995; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10996; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10997; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10998; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10999; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11000; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11001; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11002; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11003; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11004; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
11005; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11006; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11007; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11008;
11009; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
11010; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11011; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11012; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11013; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11014; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11015; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11016; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11017; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11018; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11019; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11020; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11021; GFX90A-TGSPLIT-NEXT:    s_endpgm
11022;
11023; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
11024; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11025; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11026; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11027; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11028; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11029; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11030; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
11031; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11032; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11033; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11034; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
11035; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11036; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11037; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11038;
11039; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
11040; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11041; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11042; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11043; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11044; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11045; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11046; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
11047; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11048; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11049; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11050; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
11051; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11052; GFX940-TGSPLIT-NEXT:    s_endpgm
11053    i32* %out, i32 %in, i32 %old) {
11054entry:
11055  %gep = getelementptr i32, i32* %out, i32 4
11056  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel acquire
11057  %val0 = extractvalue { i32, i1 } %val, 0
11058  store i32 %val0, i32* %out, align 4
11059  ret void
11060}
11061
11062define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_ret_cmpxchg(
11063; GFX7-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
11064; GFX7:       ; %bb.0: ; %entry
11065; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11066; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11067; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11068; GFX7-NEXT:    s_add_u32 s4, s0, 16
11069; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11070; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11071; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11072; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11073; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11074; GFX7-NEXT:    s_waitcnt vmcnt(0)
11075; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11076; GFX7-NEXT:    s_waitcnt vmcnt(0)
11077; GFX7-NEXT:    buffer_wbinvl1_vol
11078; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11079; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11080; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11081; GFX7-NEXT:    flat_store_dword v[0:1], v2
11082; GFX7-NEXT:    s_endpgm
11083;
11084; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
11085; GFX10-WGP:       ; %bb.0: ; %entry
11086; GFX10-WGP-NEXT:    s_clause 0x1
11087; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11088; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11089; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11090; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
11091; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
11092; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
11093; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11094; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
11095; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11096; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11097; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11098; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11099; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11100; GFX10-WGP-NEXT:    buffer_gl0_inv
11101; GFX10-WGP-NEXT:    buffer_gl1_inv
11102; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11103; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11104; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11105; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
11106; GFX10-WGP-NEXT:    s_endpgm
11107;
11108; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
11109; GFX10-CU:       ; %bb.0: ; %entry
11110; GFX10-CU-NEXT:    s_clause 0x1
11111; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11112; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11113; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11114; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11115; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11116; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11117; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11118; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11119; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11120; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11121; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11122; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11123; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11124; GFX10-CU-NEXT:    buffer_gl0_inv
11125; GFX10-CU-NEXT:    buffer_gl1_inv
11126; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11127; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11128; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11129; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11130; GFX10-CU-NEXT:    s_endpgm
11131;
11132; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
11133; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11134; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11135; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11136; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11137; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
11138; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
11139; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11140; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11141; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11142; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11143; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11144; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11145; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11146; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11147; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11148; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11149; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
11150; SKIP-CACHE-INV-NEXT:    s_endpgm
11151;
11152; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
11153; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11154; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11155; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11156; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11157; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11158; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11159; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11160; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11161; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11162; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
11163; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11164; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11165; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11166;
11167; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
11168; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11169; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11170; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11171; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11172; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11173; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11174; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11175; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11176; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11177; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11178; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11179; GFX90A-TGSPLIT-NEXT:    s_endpgm
11180;
11181; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
11182; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11183; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11184; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11185; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11186; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11187; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11188; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
11189; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11190; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11191; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11192; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
11193; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11194; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11195; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11196;
11197; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
11198; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11199; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11200; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11201; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11202; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11203; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11204; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
11205; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11206; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11207; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11208; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
11209; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11210; GFX940-TGSPLIT-NEXT:    s_endpgm
11211    i32* %out, i32 %in, i32 %old) {
11212entry:
11213  %gep = getelementptr i32, i32* %out, i32 4
11214  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst acquire
11215  %val0 = extractvalue { i32, i1 } %val, 0
11216  store i32 %val0, i32* %out, align 4
11217  ret void
11218}
11219
11220define amdgpu_kernel void @flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg(
11221; GFX7-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
11222; GFX7:       ; %bb.0: ; %entry
11223; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11224; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11225; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11226; GFX7-NEXT:    s_add_u32 s4, s0, 16
11227; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11228; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11229; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11230; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11231; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11232; GFX7-NEXT:    s_waitcnt vmcnt(0)
11233; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11234; GFX7-NEXT:    s_waitcnt vmcnt(0)
11235; GFX7-NEXT:    buffer_wbinvl1_vol
11236; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11237; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11238; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11239; GFX7-NEXT:    flat_store_dword v[0:1], v2
11240; GFX7-NEXT:    s_endpgm
11241;
11242; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
11243; GFX10-WGP:       ; %bb.0: ; %entry
11244; GFX10-WGP-NEXT:    s_clause 0x1
11245; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11246; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11247; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11248; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
11249; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
11250; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
11251; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11252; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
11253; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11254; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11255; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11256; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11257; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11258; GFX10-WGP-NEXT:    buffer_gl0_inv
11259; GFX10-WGP-NEXT:    buffer_gl1_inv
11260; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11261; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11262; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11263; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
11264; GFX10-WGP-NEXT:    s_endpgm
11265;
11266; GFX10-CU-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
11267; GFX10-CU:       ; %bb.0: ; %entry
11268; GFX10-CU-NEXT:    s_clause 0x1
11269; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11270; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11271; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11272; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11273; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11274; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11275; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11276; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11277; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11278; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11279; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11280; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11281; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11282; GFX10-CU-NEXT:    buffer_gl0_inv
11283; GFX10-CU-NEXT:    buffer_gl1_inv
11284; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11285; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11286; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11287; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11288; GFX10-CU-NEXT:    s_endpgm
11289;
11290; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
11291; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11292; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11293; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11294; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11295; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
11296; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
11297; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11298; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11299; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11300; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11301; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11302; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11303; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11304; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11305; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11306; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11307; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
11308; SKIP-CACHE-INV-NEXT:    s_endpgm
11309;
11310; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
11311; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11312; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11313; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11314; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11315; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11316; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11317; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11318; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11319; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11320; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
11321; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11322; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11323; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11324;
11325; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
11326; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11327; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11328; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11329; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11330; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11331; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11332; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11333; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11334; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11335; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11336; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11337; GFX90A-TGSPLIT-NEXT:    s_endpgm
11338;
11339; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
11340; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11341; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11342; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11343; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11344; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11345; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11346; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
11347; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11348; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11349; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11350; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
11351; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11352; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11353; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11354;
11355; GFX940-TGSPLIT-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
11356; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11357; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11358; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11359; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11360; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11361; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11362; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
11363; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11364; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11365; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11366; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
11367; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11368; GFX940-TGSPLIT-NEXT:    s_endpgm
11369    i32* %out, i32 %in, i32 %old) {
11370entry:
11371  %gep = getelementptr i32, i32* %out, i32 4
11372  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") monotonic seq_cst
11373  %val0 = extractvalue { i32, i1 } %val, 0
11374  store i32 %val0, i32* %out, align 4
11375  ret void
11376}
11377
11378define amdgpu_kernel void @flat_agent_one_as_acquire_seq_cst_ret_cmpxchg(
11379; GFX7-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
11380; GFX7:       ; %bb.0: ; %entry
11381; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11382; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11383; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11384; GFX7-NEXT:    s_add_u32 s4, s0, 16
11385; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11386; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11387; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11388; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11389; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11390; GFX7-NEXT:    s_waitcnt vmcnt(0)
11391; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11392; GFX7-NEXT:    s_waitcnt vmcnt(0)
11393; GFX7-NEXT:    buffer_wbinvl1_vol
11394; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11395; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11396; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11397; GFX7-NEXT:    flat_store_dword v[0:1], v2
11398; GFX7-NEXT:    s_endpgm
11399;
11400; GFX10-WGP-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
11401; GFX10-WGP:       ; %bb.0: ; %entry
11402; GFX10-WGP-NEXT:    s_clause 0x1
11403; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11404; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11405; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11406; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
11407; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
11408; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
11409; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11410; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
11411; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11412; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11413; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11414; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11415; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11416; GFX10-WGP-NEXT:    buffer_gl0_inv
11417; GFX10-WGP-NEXT:    buffer_gl1_inv
11418; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11419; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11420; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11421; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
11422; GFX10-WGP-NEXT:    s_endpgm
11423;
11424; GFX10-CU-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
11425; GFX10-CU:       ; %bb.0: ; %entry
11426; GFX10-CU-NEXT:    s_clause 0x1
11427; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11428; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11429; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11430; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11431; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11432; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11433; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11434; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11435; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11436; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11437; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11438; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11439; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11440; GFX10-CU-NEXT:    buffer_gl0_inv
11441; GFX10-CU-NEXT:    buffer_gl1_inv
11442; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11443; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11444; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11445; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11446; GFX10-CU-NEXT:    s_endpgm
11447;
11448; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
11449; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11450; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11451; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11452; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11453; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
11454; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
11455; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11456; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11457; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11458; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11459; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11460; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11461; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11462; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11463; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11464; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11465; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
11466; SKIP-CACHE-INV-NEXT:    s_endpgm
11467;
11468; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
11469; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11470; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11471; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11472; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11473; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11474; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11475; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11476; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11477; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11478; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
11479; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11480; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11481; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11482;
11483; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
11484; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11485; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11486; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11487; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11488; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11489; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11490; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11491; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11492; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11493; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11494; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11495; GFX90A-TGSPLIT-NEXT:    s_endpgm
11496;
11497; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
11498; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11499; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11500; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11501; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11502; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11503; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11504; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
11505; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11506; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11507; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11508; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
11509; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11510; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11511; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11512;
11513; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
11514; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11515; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11516; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11517; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11518; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11519; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11520; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
11521; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11522; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11523; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11524; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
11525; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11526; GFX940-TGSPLIT-NEXT:    s_endpgm
11527    i32* %out, i32 %in, i32 %old) {
11528entry:
11529  %gep = getelementptr i32, i32* %out, i32 4
11530  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire seq_cst
11531  %val0 = extractvalue { i32, i1 } %val, 0
11532  store i32 %val0, i32* %out, align 4
11533  ret void
11534}
11535
11536define amdgpu_kernel void @flat_agent_one_as_release_seq_cst_ret_cmpxchg(
11537; GFX7-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
11538; GFX7:       ; %bb.0: ; %entry
11539; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11540; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11541; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11542; GFX7-NEXT:    s_add_u32 s4, s0, 16
11543; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11544; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11545; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11546; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11547; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11548; GFX7-NEXT:    s_waitcnt vmcnt(0)
11549; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11550; GFX7-NEXT:    s_waitcnt vmcnt(0)
11551; GFX7-NEXT:    buffer_wbinvl1_vol
11552; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11553; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11554; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11555; GFX7-NEXT:    flat_store_dword v[0:1], v2
11556; GFX7-NEXT:    s_endpgm
11557;
11558; GFX10-WGP-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
11559; GFX10-WGP:       ; %bb.0: ; %entry
11560; GFX10-WGP-NEXT:    s_clause 0x1
11561; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11562; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11563; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11564; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
11565; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
11566; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
11567; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11568; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
11569; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11570; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11571; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11572; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11573; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11574; GFX10-WGP-NEXT:    buffer_gl0_inv
11575; GFX10-WGP-NEXT:    buffer_gl1_inv
11576; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11577; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11578; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11579; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
11580; GFX10-WGP-NEXT:    s_endpgm
11581;
11582; GFX10-CU-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
11583; GFX10-CU:       ; %bb.0: ; %entry
11584; GFX10-CU-NEXT:    s_clause 0x1
11585; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11586; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11587; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11588; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11589; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11590; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11591; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11592; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11593; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11594; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11595; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11596; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11597; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11598; GFX10-CU-NEXT:    buffer_gl0_inv
11599; GFX10-CU-NEXT:    buffer_gl1_inv
11600; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11601; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11602; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11603; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11604; GFX10-CU-NEXT:    s_endpgm
11605;
11606; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
11607; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11608; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11609; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11610; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11611; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
11612; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
11613; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11614; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11615; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11616; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11617; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11618; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11619; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11620; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11621; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11622; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11623; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
11624; SKIP-CACHE-INV-NEXT:    s_endpgm
11625;
11626; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
11627; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11628; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11629; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11630; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11631; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11632; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11633; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11634; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11635; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11636; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
11637; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11638; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11639; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11640;
11641; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
11642; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11643; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11644; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11645; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11646; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11647; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11648; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11649; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11650; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11651; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11652; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11653; GFX90A-TGSPLIT-NEXT:    s_endpgm
11654;
11655; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
11656; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11657; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11658; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11659; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11660; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11661; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11662; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
11663; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11664; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11665; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11666; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
11667; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11668; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11669; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11670;
11671; GFX940-TGSPLIT-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
11672; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11673; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11674; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11675; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11676; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11677; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11678; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
11679; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11680; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11681; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11682; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
11683; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11684; GFX940-TGSPLIT-NEXT:    s_endpgm
11685    i32* %out, i32 %in, i32 %old) {
11686entry:
11687  %gep = getelementptr i32, i32* %out, i32 4
11688  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release seq_cst
11689  %val0 = extractvalue { i32, i1 } %val, 0
11690  store i32 %val0, i32* %out, align 4
11691  ret void
11692}
11693
11694define amdgpu_kernel void @flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg(
11695; GFX7-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
11696; GFX7:       ; %bb.0: ; %entry
11697; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11698; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11699; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11700; GFX7-NEXT:    s_add_u32 s4, s0, 16
11701; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11702; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11703; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11704; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11705; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11706; GFX7-NEXT:    s_waitcnt vmcnt(0)
11707; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11708; GFX7-NEXT:    s_waitcnt vmcnt(0)
11709; GFX7-NEXT:    buffer_wbinvl1_vol
11710; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11711; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11712; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11713; GFX7-NEXT:    flat_store_dword v[0:1], v2
11714; GFX7-NEXT:    s_endpgm
11715;
11716; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
11717; GFX10-WGP:       ; %bb.0: ; %entry
11718; GFX10-WGP-NEXT:    s_clause 0x1
11719; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11720; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11721; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11722; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
11723; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
11724; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
11725; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11726; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
11727; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11728; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11729; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11730; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11731; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11732; GFX10-WGP-NEXT:    buffer_gl0_inv
11733; GFX10-WGP-NEXT:    buffer_gl1_inv
11734; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11735; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11736; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11737; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
11738; GFX10-WGP-NEXT:    s_endpgm
11739;
11740; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
11741; GFX10-CU:       ; %bb.0: ; %entry
11742; GFX10-CU-NEXT:    s_clause 0x1
11743; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11744; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11745; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11746; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11747; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11748; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11749; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11750; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11751; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11752; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11753; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11754; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11755; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11756; GFX10-CU-NEXT:    buffer_gl0_inv
11757; GFX10-CU-NEXT:    buffer_gl1_inv
11758; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11759; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11760; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11761; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11762; GFX10-CU-NEXT:    s_endpgm
11763;
11764; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
11765; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11766; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11767; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11768; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11769; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
11770; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
11771; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11772; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11773; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11774; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11775; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11776; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11777; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11778; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11779; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11780; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11781; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
11782; SKIP-CACHE-INV-NEXT:    s_endpgm
11783;
11784; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
11785; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11786; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11787; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11788; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11789; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11790; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11791; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11792; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11793; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11794; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
11795; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11796; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11797; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11798;
11799; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
11800; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11801; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11802; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11803; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11804; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11805; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11806; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11807; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11808; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11809; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11810; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11811; GFX90A-TGSPLIT-NEXT:    s_endpgm
11812;
11813; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
11814; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11815; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11816; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11817; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11818; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11819; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11820; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
11821; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11822; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11823; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11824; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
11825; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11826; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11827; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11828;
11829; GFX940-TGSPLIT-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
11830; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11831; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11832; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11833; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11834; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11835; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11836; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
11837; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11838; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11839; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11840; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
11841; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11842; GFX940-TGSPLIT-NEXT:    s_endpgm
11843    i32* %out, i32 %in, i32 %old) {
11844entry:
11845  %gep = getelementptr i32, i32* %out, i32 4
11846  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel seq_cst
11847  %val0 = extractvalue { i32, i1 } %val, 0
11848  store i32 %val0, i32* %out, align 4
11849  ret void
11850}
11851
11852define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg(
11853; GFX7-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
11854; GFX7:       ; %bb.0: ; %entry
11855; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11856; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11857; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11858; GFX7-NEXT:    s_add_u32 s4, s0, 16
11859; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11860; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11861; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11862; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11863; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11864; GFX7-NEXT:    s_waitcnt vmcnt(0)
11865; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11866; GFX7-NEXT:    s_waitcnt vmcnt(0)
11867; GFX7-NEXT:    buffer_wbinvl1_vol
11868; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11869; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11870; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11871; GFX7-NEXT:    flat_store_dword v[0:1], v2
11872; GFX7-NEXT:    s_endpgm
11873;
11874; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
11875; GFX10-WGP:       ; %bb.0: ; %entry
11876; GFX10-WGP-NEXT:    s_clause 0x1
11877; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11878; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11879; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11880; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
11881; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
11882; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
11883; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11884; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
11885; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11886; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11887; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11888; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11889; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11890; GFX10-WGP-NEXT:    buffer_gl0_inv
11891; GFX10-WGP-NEXT:    buffer_gl1_inv
11892; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11893; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11894; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11895; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
11896; GFX10-WGP-NEXT:    s_endpgm
11897;
11898; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
11899; GFX10-CU:       ; %bb.0: ; %entry
11900; GFX10-CU-NEXT:    s_clause 0x1
11901; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11902; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11903; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11904; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11905; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11906; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11907; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11908; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11909; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11910; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11911; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11912; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11913; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11914; GFX10-CU-NEXT:    buffer_gl0_inv
11915; GFX10-CU-NEXT:    buffer_gl1_inv
11916; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11917; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11918; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11919; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11920; GFX10-CU-NEXT:    s_endpgm
11921;
11922; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
11923; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11924; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11925; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11926; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11927; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
11928; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
11929; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11930; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11931; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11932; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11933; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11934; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11935; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11936; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11937; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11938; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11939; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
11940; SKIP-CACHE-INV-NEXT:    s_endpgm
11941;
11942; GFX90A-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
11943; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11944; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11945; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11946; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11947; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11948; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11949; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11950; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11951; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11952; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
11953; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11954; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11955; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11956;
11957; GFX90A-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
11958; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11959; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11960; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11961; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11962; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11963; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11964; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11965; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11966; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11967; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11968; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11969; GFX90A-TGSPLIT-NEXT:    s_endpgm
11970;
11971; GFX940-NOTTGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
11972; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11973; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11974; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11975; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11976; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11977; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11978; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc1
11979; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11980; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11981; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11982; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc1
11983; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11984; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11985; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11986;
11987; GFX940-TGSPLIT-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
11988; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11989; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11990; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11991; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11992; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11993; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11994; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc1
11995; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11996; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11997; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11998; GFX940-TGSPLIT-NEXT:    buffer_inv sc1
11999; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12000; GFX940-TGSPLIT-NEXT:    s_endpgm
12001    i32* %out, i32 %in, i32 %old) {
12002entry:
12003  %gep = getelementptr i32, i32* %out, i32 4
12004  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst seq_cst
12005  %val0 = extractvalue { i32, i1 } %val, 0
12006  store i32 %val0, i32* %out, align 4
12007  ret void
12008}
12009
12010