1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
3; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
4; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
5; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
6; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-NOTTGSPLIT %s
7; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-TGSPLIT %s
8
9define amdgpu_kernel void @flat_system_unordered_load(
10; GFX7-LABEL: flat_system_unordered_load:
11; GFX7:       ; %bb.0: ; %entry
12; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
13; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14; GFX7-NEXT:    v_mov_b32_e32 v0, s0
15; GFX7-NEXT:    v_mov_b32_e32 v1, s1
16; GFX7-NEXT:    flat_load_dword v0, v[0:1]
17; GFX7-NEXT:    v_mov_b32_e32 v2, s2
18; GFX7-NEXT:    v_mov_b32_e32 v3, s3
19; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
20; GFX7-NEXT:    flat_store_dword v[2:3], v0
21; GFX7-NEXT:    s_endpgm
22;
23; GFX10-WGP-LABEL: flat_system_unordered_load:
24; GFX10-WGP:       ; %bb.0: ; %entry
25; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
26; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
27; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
28; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
29; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
30; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
31; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
32; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
33; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
34; GFX10-WGP-NEXT:    s_endpgm
35;
36; GFX10-CU-LABEL: flat_system_unordered_load:
37; GFX10-CU:       ; %bb.0: ; %entry
38; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
39; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
40; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
41; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
42; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
43; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
44; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
45; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
46; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
47; GFX10-CU-NEXT:    s_endpgm
48;
49; SKIP-CACHE-INV-LABEL: flat_system_unordered_load:
50; SKIP-CACHE-INV:       ; %bb.0: ; %entry
51; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
52; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
53; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
54; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
55; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
56; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
57; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
58; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
59; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
60; SKIP-CACHE-INV-NEXT:    s_endpgm
61;
62; GFX90A-NOTTGSPLIT-LABEL: flat_system_unordered_load:
63; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
64; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
65; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
66; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
67; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
68; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v0, v[0:1]
69; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
70; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
71; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
72; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[2:3], v0
73; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
74;
75; GFX90A-TGSPLIT-LABEL: flat_system_unordered_load:
76; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
77; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
78; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
79; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
80; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
81; GFX90A-TGSPLIT-NEXT:    flat_load_dword v0, v[0:1]
82; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
83; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
84; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
85; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[2:3], v0
86; GFX90A-TGSPLIT-NEXT:    s_endpgm
87    i32* %in, i32* %out) {
88entry:
89  %val = load atomic i32, i32* %in unordered, align 4
90  store i32 %val, i32* %out
91  ret void
92}
93
94define amdgpu_kernel void @flat_system_monotonic_load(
95; GFX7-LABEL: flat_system_monotonic_load:
96; GFX7:       ; %bb.0: ; %entry
97; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
98; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
99; GFX7-NEXT:    v_mov_b32_e32 v0, s0
100; GFX7-NEXT:    v_mov_b32_e32 v1, s1
101; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
102; GFX7-NEXT:    v_mov_b32_e32 v2, s2
103; GFX7-NEXT:    v_mov_b32_e32 v3, s3
104; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
105; GFX7-NEXT:    flat_store_dword v[2:3], v0
106; GFX7-NEXT:    s_endpgm
107;
108; GFX10-WGP-LABEL: flat_system_monotonic_load:
109; GFX10-WGP:       ; %bb.0: ; %entry
110; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
111; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
112; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
113; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
114; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
115; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
116; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
117; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
118; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
119; GFX10-WGP-NEXT:    s_endpgm
120;
121; GFX10-CU-LABEL: flat_system_monotonic_load:
122; GFX10-CU:       ; %bb.0: ; %entry
123; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
124; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
125; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
126; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
127; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
128; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
129; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
130; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
131; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
132; GFX10-CU-NEXT:    s_endpgm
133;
134; SKIP-CACHE-INV-LABEL: flat_system_monotonic_load:
135; SKIP-CACHE-INV:       ; %bb.0: ; %entry
136; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
137; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
138; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
139; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
140; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1] glc
141; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
142; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
143; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
144; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
145; SKIP-CACHE-INV-NEXT:    s_endpgm
146;
147; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_load:
148; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
149; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
150; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
151; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
152; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
153; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v0, v[0:1] glc scc
154; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
155; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
156; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
157; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[2:3], v0
158; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
159;
160; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_load:
161; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
162; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
163; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
164; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
165; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
166; GFX90A-TGSPLIT-NEXT:    flat_load_dword v0, v[0:1] glc scc
167; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
168; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
169; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
170; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[2:3], v0
171; GFX90A-TGSPLIT-NEXT:    s_endpgm
172    i32* %in, i32* %out) {
173entry:
174  %val = load atomic i32, i32* %in monotonic, align 4
175  store i32 %val, i32* %out
176  ret void
177}
178
179define amdgpu_kernel void @flat_system_acquire_load(
180; GFX7-LABEL: flat_system_acquire_load:
181; GFX7:       ; %bb.0: ; %entry
182; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
183; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
184; GFX7-NEXT:    v_mov_b32_e32 v0, s0
185; GFX7-NEXT:    v_mov_b32_e32 v1, s1
186; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
187; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
188; GFX7-NEXT:    buffer_wbinvl1_vol
189; GFX7-NEXT:    v_mov_b32_e32 v2, s2
190; GFX7-NEXT:    v_mov_b32_e32 v3, s3
191; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
192; GFX7-NEXT:    flat_store_dword v[2:3], v0
193; GFX7-NEXT:    s_endpgm
194;
195; GFX10-WGP-LABEL: flat_system_acquire_load:
196; GFX10-WGP:       ; %bb.0: ; %entry
197; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
198; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
199; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
200; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
201; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
202; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
203; GFX10-WGP-NEXT:    buffer_gl0_inv
204; GFX10-WGP-NEXT:    buffer_gl1_inv
205; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
206; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
207; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
208; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
209; GFX10-WGP-NEXT:    s_endpgm
210;
211; GFX10-CU-LABEL: flat_system_acquire_load:
212; GFX10-CU:       ; %bb.0: ; %entry
213; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
214; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
215; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
216; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
217; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
218; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
219; GFX10-CU-NEXT:    buffer_gl0_inv
220; GFX10-CU-NEXT:    buffer_gl1_inv
221; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
222; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
223; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
224; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
225; GFX10-CU-NEXT:    s_endpgm
226;
227; SKIP-CACHE-INV-LABEL: flat_system_acquire_load:
228; SKIP-CACHE-INV:       ; %bb.0: ; %entry
229; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
230; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
231; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
232; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
233; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1] glc
234; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
235; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
236; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
237; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
238; SKIP-CACHE-INV-NEXT:    s_endpgm
239;
240; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_load:
241; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
242; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
243; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
244; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
245; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
246; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v0, v[0:1] glc scc
247; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
248; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
249; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
250; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
251; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
252; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
253; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[2:3], v0
254; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
255;
256; GFX90A-TGSPLIT-LABEL: flat_system_acquire_load:
257; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
258; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
259; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
260; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
261; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
262; GFX90A-TGSPLIT-NEXT:    flat_load_dword v0, v[0:1] glc scc
263; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
264; GFX90A-TGSPLIT-NEXT:    buffer_invl2
265; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
266; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
267; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
268; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
269; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[2:3], v0
270; GFX90A-TGSPLIT-NEXT:    s_endpgm
271    i32* %in, i32* %out) {
272entry:
273  %val = load atomic i32, i32* %in acquire, align 4
274  store i32 %val, i32* %out
275  ret void
276}
277
278define amdgpu_kernel void @flat_system_seq_cst_load(
279; GFX7-LABEL: flat_system_seq_cst_load:
280; GFX7:       ; %bb.0: ; %entry
281; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
282; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
283; GFX7-NEXT:    v_mov_b32_e32 v0, s0
284; GFX7-NEXT:    v_mov_b32_e32 v1, s1
285; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
286; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
287; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
288; GFX7-NEXT:    buffer_wbinvl1_vol
289; GFX7-NEXT:    v_mov_b32_e32 v2, s2
290; GFX7-NEXT:    v_mov_b32_e32 v3, s3
291; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
292; GFX7-NEXT:    flat_store_dword v[2:3], v0
293; GFX7-NEXT:    s_endpgm
294;
295; GFX10-WGP-LABEL: flat_system_seq_cst_load:
296; GFX10-WGP:       ; %bb.0: ; %entry
297; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
298; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
299; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
300; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
301; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
302; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
303; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
304; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
305; GFX10-WGP-NEXT:    buffer_gl0_inv
306; GFX10-WGP-NEXT:    buffer_gl1_inv
307; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
308; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
309; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
310; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
311; GFX10-WGP-NEXT:    s_endpgm
312;
313; GFX10-CU-LABEL: flat_system_seq_cst_load:
314; GFX10-CU:       ; %bb.0: ; %entry
315; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
316; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
317; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
318; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
319; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
320; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
321; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
322; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
323; GFX10-CU-NEXT:    buffer_gl0_inv
324; GFX10-CU-NEXT:    buffer_gl1_inv
325; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
326; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
327; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
328; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
329; GFX10-CU-NEXT:    s_endpgm
330;
331; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_load:
332; SKIP-CACHE-INV:       ; %bb.0: ; %entry
333; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
334; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
335; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
336; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
337; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
338; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1] glc
339; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
340; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
341; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
342; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
343; SKIP-CACHE-INV-NEXT:    s_endpgm
344;
345; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_load:
346; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
347; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
348; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
349; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
350; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
351; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
352; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v0, v[0:1] glc scc
353; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
354; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
355; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
356; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
357; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
358; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
359; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[2:3], v0
360; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
361;
362; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_load:
363; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
364; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
365; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
366; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
367; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
368; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
369; GFX90A-TGSPLIT-NEXT:    flat_load_dword v0, v[0:1] glc scc
370; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
371; GFX90A-TGSPLIT-NEXT:    buffer_invl2
372; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
373; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
374; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
375; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
376; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[2:3], v0
377; GFX90A-TGSPLIT-NEXT:    s_endpgm
378    i32* %in, i32* %out) {
379entry:
380  %val = load atomic i32, i32* %in seq_cst, align 4
381  store i32 %val, i32* %out
382  ret void
383}
384
385define amdgpu_kernel void @flat_system_unordered_store(
386; GFX7-LABEL: flat_system_unordered_store:
387; GFX7:       ; %bb.0: ; %entry
388; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
389; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
390; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
391; GFX7-NEXT:    v_mov_b32_e32 v2, s2
392; GFX7-NEXT:    v_mov_b32_e32 v0, s0
393; GFX7-NEXT:    v_mov_b32_e32 v1, s1
394; GFX7-NEXT:    flat_store_dword v[0:1], v2
395; GFX7-NEXT:    s_endpgm
396;
397; GFX10-WGP-LABEL: flat_system_unordered_store:
398; GFX10-WGP:       ; %bb.0: ; %entry
399; GFX10-WGP-NEXT:    s_clause 0x1
400; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
401; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
402; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
403; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
404; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
405; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
406; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
407; GFX10-WGP-NEXT:    s_endpgm
408;
409; GFX10-CU-LABEL: flat_system_unordered_store:
410; GFX10-CU:       ; %bb.0: ; %entry
411; GFX10-CU-NEXT:    s_clause 0x1
412; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
413; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
414; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
415; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
416; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
417; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
418; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
419; GFX10-CU-NEXT:    s_endpgm
420;
421; SKIP-CACHE-INV-LABEL: flat_system_unordered_store:
422; SKIP-CACHE-INV:       ; %bb.0: ; %entry
423; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
424; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
425; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
426; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
427; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
428; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
429; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
430; SKIP-CACHE-INV-NEXT:    s_endpgm
431;
432; GFX90A-NOTTGSPLIT-LABEL: flat_system_unordered_store:
433; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
434; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
435; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
436; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
437; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
438; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
439; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
440; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
441;
442; GFX90A-TGSPLIT-LABEL: flat_system_unordered_store:
443; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
444; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
445; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
446; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
447; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
448; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
449; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
450; GFX90A-TGSPLIT-NEXT:    s_endpgm
451    i32 %in, i32* %out) {
452entry:
453  store atomic i32 %in, i32* %out unordered, align 4
454  ret void
455}
456
457define amdgpu_kernel void @flat_system_monotonic_store(
458; GFX7-LABEL: flat_system_monotonic_store:
459; GFX7:       ; %bb.0: ; %entry
460; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
461; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
462; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
463; GFX7-NEXT:    v_mov_b32_e32 v2, s2
464; GFX7-NEXT:    v_mov_b32_e32 v0, s0
465; GFX7-NEXT:    v_mov_b32_e32 v1, s1
466; GFX7-NEXT:    flat_store_dword v[0:1], v2
467; GFX7-NEXT:    s_endpgm
468;
469; GFX10-WGP-LABEL: flat_system_monotonic_store:
470; GFX10-WGP:       ; %bb.0: ; %entry
471; GFX10-WGP-NEXT:    s_clause 0x1
472; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
473; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
474; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
475; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
476; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
477; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
478; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
479; GFX10-WGP-NEXT:    s_endpgm
480;
481; GFX10-CU-LABEL: flat_system_monotonic_store:
482; GFX10-CU:       ; %bb.0: ; %entry
483; GFX10-CU-NEXT:    s_clause 0x1
484; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
485; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
486; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
487; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
488; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
489; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
490; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
491; GFX10-CU-NEXT:    s_endpgm
492;
493; SKIP-CACHE-INV-LABEL: flat_system_monotonic_store:
494; SKIP-CACHE-INV:       ; %bb.0: ; %entry
495; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
496; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
497; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
498; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
499; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
500; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
501; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
502; SKIP-CACHE-INV-NEXT:    s_endpgm
503;
504; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_store:
505; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
506; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
507; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
508; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
509; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
510; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
511; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 scc
512; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
513;
514; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_store:
515; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
516; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
517; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
518; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
519; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
520; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
521; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 scc
522; GFX90A-TGSPLIT-NEXT:    s_endpgm
523    i32 %in, i32* %out) {
524entry:
525  store atomic i32 %in, i32* %out monotonic, align 4
526  ret void
527}
528
529define amdgpu_kernel void @flat_system_release_store(
530; GFX7-LABEL: flat_system_release_store:
531; GFX7:       ; %bb.0: ; %entry
532; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
533; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
534; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
535; GFX7-NEXT:    v_mov_b32_e32 v2, s2
536; GFX7-NEXT:    v_mov_b32_e32 v0, s0
537; GFX7-NEXT:    v_mov_b32_e32 v1, s1
538; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
539; GFX7-NEXT:    flat_store_dword v[0:1], v2
540; GFX7-NEXT:    s_endpgm
541;
542; GFX10-WGP-LABEL: flat_system_release_store:
543; GFX10-WGP:       ; %bb.0: ; %entry
544; GFX10-WGP-NEXT:    s_clause 0x1
545; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
546; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
547; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
548; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
549; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
550; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
551; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
552; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
553; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
554; GFX10-WGP-NEXT:    s_endpgm
555;
556; GFX10-CU-LABEL: flat_system_release_store:
557; GFX10-CU:       ; %bb.0: ; %entry
558; GFX10-CU-NEXT:    s_clause 0x1
559; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
560; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
561; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
562; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
563; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
564; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
565; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
566; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
567; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
568; GFX10-CU-NEXT:    s_endpgm
569;
570; SKIP-CACHE-INV-LABEL: flat_system_release_store:
571; SKIP-CACHE-INV:       ; %bb.0: ; %entry
572; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
573; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
574; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
575; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
576; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
577; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
578; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
579; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
580; SKIP-CACHE-INV-NEXT:    s_endpgm
581;
582; GFX90A-NOTTGSPLIT-LABEL: flat_system_release_store:
583; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
584; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
585; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
586; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
587; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
588; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
589; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
590; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
591; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 scc
592; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
593;
594; GFX90A-TGSPLIT-LABEL: flat_system_release_store:
595; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
596; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
597; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
598; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
599; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
600; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
601; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
602; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
603; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 scc
604; GFX90A-TGSPLIT-NEXT:    s_endpgm
605    i32 %in, i32* %out) {
606entry:
607  store atomic i32 %in, i32* %out release, align 4
608  ret void
609}
610
611define amdgpu_kernel void @flat_system_seq_cst_store(
612; GFX7-LABEL: flat_system_seq_cst_store:
613; GFX7:       ; %bb.0: ; %entry
614; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
615; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
616; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
617; GFX7-NEXT:    v_mov_b32_e32 v2, s2
618; GFX7-NEXT:    v_mov_b32_e32 v0, s0
619; GFX7-NEXT:    v_mov_b32_e32 v1, s1
620; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
621; GFX7-NEXT:    flat_store_dword v[0:1], v2
622; GFX7-NEXT:    s_endpgm
623;
624; GFX10-WGP-LABEL: flat_system_seq_cst_store:
625; GFX10-WGP:       ; %bb.0: ; %entry
626; GFX10-WGP-NEXT:    s_clause 0x1
627; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
628; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
629; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
630; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
631; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
632; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
633; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
634; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
635; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
636; GFX10-WGP-NEXT:    s_endpgm
637;
638; GFX10-CU-LABEL: flat_system_seq_cst_store:
639; GFX10-CU:       ; %bb.0: ; %entry
640; GFX10-CU-NEXT:    s_clause 0x1
641; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
642; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
643; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
644; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
645; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
646; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
647; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
648; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
649; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
650; GFX10-CU-NEXT:    s_endpgm
651;
652; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_store:
653; SKIP-CACHE-INV:       ; %bb.0: ; %entry
654; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
655; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
656; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
657; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
658; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
659; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
660; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
661; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
662; SKIP-CACHE-INV-NEXT:    s_endpgm
663;
664; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_store:
665; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
666; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
667; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
668; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
669; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
670; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
671; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
672; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
673; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 scc
674; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
675;
676; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_store:
677; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
678; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
679; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
680; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
681; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
682; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
683; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
684; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
685; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 scc
686; GFX90A-TGSPLIT-NEXT:    s_endpgm
687    i32 %in, i32* %out) {
688entry:
689  store atomic i32 %in, i32* %out seq_cst, align 4
690  ret void
691}
692
693define amdgpu_kernel void @flat_system_monotonic_atomicrmw(
694; GFX7-LABEL: flat_system_monotonic_atomicrmw:
695; GFX7:       ; %bb.0: ; %entry
696; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
697; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
698; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
699; GFX7-NEXT:    v_mov_b32_e32 v0, s0
700; GFX7-NEXT:    v_mov_b32_e32 v1, s1
701; GFX7-NEXT:    v_mov_b32_e32 v2, s2
702; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
703; GFX7-NEXT:    s_endpgm
704;
705; GFX10-WGP-LABEL: flat_system_monotonic_atomicrmw:
706; GFX10-WGP:       ; %bb.0: ; %entry
707; GFX10-WGP-NEXT:    s_clause 0x1
708; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
709; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
710; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
711; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
712; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
713; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
714; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
715; GFX10-WGP-NEXT:    s_endpgm
716;
717; GFX10-CU-LABEL: flat_system_monotonic_atomicrmw:
718; GFX10-CU:       ; %bb.0: ; %entry
719; GFX10-CU-NEXT:    s_clause 0x1
720; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
721; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
722; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
723; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
724; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
725; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
726; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
727; GFX10-CU-NEXT:    s_endpgm
728;
729; SKIP-CACHE-INV-LABEL: flat_system_monotonic_atomicrmw:
730; SKIP-CACHE-INV:       ; %bb.0: ; %entry
731; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
732; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
733; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
734; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
735; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
736; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
737; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
738; SKIP-CACHE-INV-NEXT:    s_endpgm
739;
740; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_atomicrmw:
741; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
742; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
743; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
744; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
745; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
746; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
747; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 scc
748; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
749;
750; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_atomicrmw:
751; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
752; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
753; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
754; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
755; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
756; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
757; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 scc
758; GFX90A-TGSPLIT-NEXT:    s_endpgm
759    i32* %out, i32 %in) {
760entry:
761  %val = atomicrmw volatile xchg i32* %out, i32 %in monotonic
762  ret void
763}
764
765define amdgpu_kernel void @flat_system_acquire_atomicrmw(
766; GFX7-LABEL: flat_system_acquire_atomicrmw:
767; GFX7:       ; %bb.0: ; %entry
768; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
769; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
770; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
771; GFX7-NEXT:    v_mov_b32_e32 v0, s0
772; GFX7-NEXT:    v_mov_b32_e32 v1, s1
773; GFX7-NEXT:    v_mov_b32_e32 v2, s2
774; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
775; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
776; GFX7-NEXT:    buffer_wbinvl1_vol
777; GFX7-NEXT:    s_endpgm
778;
779; GFX10-WGP-LABEL: flat_system_acquire_atomicrmw:
780; GFX10-WGP:       ; %bb.0: ; %entry
781; GFX10-WGP-NEXT:    s_clause 0x1
782; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
783; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
784; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
785; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
786; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
787; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
788; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
789; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
790; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
791; GFX10-WGP-NEXT:    buffer_gl0_inv
792; GFX10-WGP-NEXT:    buffer_gl1_inv
793; GFX10-WGP-NEXT:    s_endpgm
794;
795; GFX10-CU-LABEL: flat_system_acquire_atomicrmw:
796; GFX10-CU:       ; %bb.0: ; %entry
797; GFX10-CU-NEXT:    s_clause 0x1
798; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
799; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
800; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
801; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
802; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
803; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
804; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
805; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
806; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
807; GFX10-CU-NEXT:    buffer_gl0_inv
808; GFX10-CU-NEXT:    buffer_gl1_inv
809; GFX10-CU-NEXT:    s_endpgm
810;
811; SKIP-CACHE-INV-LABEL: flat_system_acquire_atomicrmw:
812; SKIP-CACHE-INV:       ; %bb.0: ; %entry
813; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
814; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
815; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
816; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
817; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
818; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
819; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
820; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
821; SKIP-CACHE-INV-NEXT:    s_endpgm
822;
823; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_atomicrmw:
824; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
825; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
826; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
827; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
828; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
829; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
830; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 scc
831; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
832; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
833; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
834; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
835; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
836;
837; GFX90A-TGSPLIT-LABEL: flat_system_acquire_atomicrmw:
838; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
839; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
840; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
841; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
842; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
843; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
844; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 scc
845; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
846; GFX90A-TGSPLIT-NEXT:    buffer_invl2
847; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
848; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
849; GFX90A-TGSPLIT-NEXT:    s_endpgm
850    i32* %out, i32 %in) {
851entry:
852  %val = atomicrmw volatile xchg i32* %out, i32 %in acquire
853  ret void
854}
855
856define amdgpu_kernel void @flat_system_release_atomicrmw(
857; GFX7-LABEL: flat_system_release_atomicrmw:
858; GFX7:       ; %bb.0: ; %entry
859; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
860; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
861; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
862; GFX7-NEXT:    v_mov_b32_e32 v0, s0
863; GFX7-NEXT:    v_mov_b32_e32 v1, s1
864; GFX7-NEXT:    v_mov_b32_e32 v2, s2
865; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
866; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
867; GFX7-NEXT:    s_endpgm
868;
869; GFX10-WGP-LABEL: flat_system_release_atomicrmw:
870; GFX10-WGP:       ; %bb.0: ; %entry
871; GFX10-WGP-NEXT:    s_clause 0x1
872; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
873; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
874; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
875; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
876; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
877; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
878; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
879; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
880; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
881; GFX10-WGP-NEXT:    s_endpgm
882;
883; GFX10-CU-LABEL: flat_system_release_atomicrmw:
884; GFX10-CU:       ; %bb.0: ; %entry
885; GFX10-CU-NEXT:    s_clause 0x1
886; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
887; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
888; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
889; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
890; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
891; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
892; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
893; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
894; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
895; GFX10-CU-NEXT:    s_endpgm
896;
897; SKIP-CACHE-INV-LABEL: flat_system_release_atomicrmw:
898; SKIP-CACHE-INV:       ; %bb.0: ; %entry
899; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
900; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
901; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
902; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
903; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
904; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
905; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
906; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
907; SKIP-CACHE-INV-NEXT:    s_endpgm
908;
909; GFX90A-NOTTGSPLIT-LABEL: flat_system_release_atomicrmw:
910; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
911; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
912; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
913; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
914; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
915; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
916; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
917; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
918; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 scc
919; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
920;
921; GFX90A-TGSPLIT-LABEL: flat_system_release_atomicrmw:
922; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
923; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
924; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
925; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
926; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
927; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
928; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
929; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
930; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 scc
931; GFX90A-TGSPLIT-NEXT:    s_endpgm
932    i32* %out, i32 %in) {
933entry:
934  %val = atomicrmw volatile xchg i32* %out, i32 %in release
935  ret void
936}
937
938define amdgpu_kernel void @flat_system_acq_rel_atomicrmw(
939; GFX7-LABEL: flat_system_acq_rel_atomicrmw:
940; GFX7:       ; %bb.0: ; %entry
941; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
942; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
943; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
944; GFX7-NEXT:    v_mov_b32_e32 v0, s0
945; GFX7-NEXT:    v_mov_b32_e32 v1, s1
946; GFX7-NEXT:    v_mov_b32_e32 v2, s2
947; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
948; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
949; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
950; GFX7-NEXT:    buffer_wbinvl1_vol
951; GFX7-NEXT:    s_endpgm
952;
953; GFX10-WGP-LABEL: flat_system_acq_rel_atomicrmw:
954; GFX10-WGP:       ; %bb.0: ; %entry
955; GFX10-WGP-NEXT:    s_clause 0x1
956; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
957; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
958; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
959; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
960; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
961; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
962; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
963; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
964; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
965; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
966; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
967; GFX10-WGP-NEXT:    buffer_gl0_inv
968; GFX10-WGP-NEXT:    buffer_gl1_inv
969; GFX10-WGP-NEXT:    s_endpgm
970;
971; GFX10-CU-LABEL: flat_system_acq_rel_atomicrmw:
972; GFX10-CU:       ; %bb.0: ; %entry
973; GFX10-CU-NEXT:    s_clause 0x1
974; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
975; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
976; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
977; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
978; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
979; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
980; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
981; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
982; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
983; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
984; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
985; GFX10-CU-NEXT:    buffer_gl0_inv
986; GFX10-CU-NEXT:    buffer_gl1_inv
987; GFX10-CU-NEXT:    s_endpgm
988;
989; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_atomicrmw:
990; SKIP-CACHE-INV:       ; %bb.0: ; %entry
991; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
992; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
993; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
994; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
995; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
996; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
997; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
998; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
999; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1000; SKIP-CACHE-INV-NEXT:    s_endpgm
1001;
1002; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_atomicrmw:
1003; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1004; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1005; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1006; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1007; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1008; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1009; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
1010; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1011; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 scc
1012; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1013; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
1014; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1015; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1016; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1017;
1018; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_atomicrmw:
1019; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1020; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1021; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1022; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1023; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1024; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1025; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
1026; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1027; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 scc
1028; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1029; GFX90A-TGSPLIT-NEXT:    buffer_invl2
1030; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1031; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1032; GFX90A-TGSPLIT-NEXT:    s_endpgm
1033    i32* %out, i32 %in) {
1034entry:
1035  %val = atomicrmw volatile xchg i32* %out, i32 %in acq_rel
1036  ret void
1037}
1038
1039define amdgpu_kernel void @flat_system_seq_cst_atomicrmw(
1040; GFX7-LABEL: flat_system_seq_cst_atomicrmw:
1041; GFX7:       ; %bb.0: ; %entry
1042; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1043; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1044; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1045; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1046; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1047; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1048; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1049; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1050; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1051; GFX7-NEXT:    buffer_wbinvl1_vol
1052; GFX7-NEXT:    s_endpgm
1053;
1054; GFX10-WGP-LABEL: flat_system_seq_cst_atomicrmw:
1055; GFX10-WGP:       ; %bb.0: ; %entry
1056; GFX10-WGP-NEXT:    s_clause 0x1
1057; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1058; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1059; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1060; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1061; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1062; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1063; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1064; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1065; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1066; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1067; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1068; GFX10-WGP-NEXT:    buffer_gl0_inv
1069; GFX10-WGP-NEXT:    buffer_gl1_inv
1070; GFX10-WGP-NEXT:    s_endpgm
1071;
1072; GFX10-CU-LABEL: flat_system_seq_cst_atomicrmw:
1073; GFX10-CU:       ; %bb.0: ; %entry
1074; GFX10-CU-NEXT:    s_clause 0x1
1075; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1076; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1077; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1078; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1079; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1080; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1081; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1082; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1083; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1084; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1085; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1086; GFX10-CU-NEXT:    buffer_gl0_inv
1087; GFX10-CU-NEXT:    buffer_gl1_inv
1088; GFX10-CU-NEXT:    s_endpgm
1089;
1090; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_atomicrmw:
1091; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1092; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
1093; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
1094; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1095; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1096; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1097; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1098; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1099; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1100; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1101; SKIP-CACHE-INV-NEXT:    s_endpgm
1102;
1103; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_atomicrmw:
1104; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1105; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1106; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1107; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1108; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1109; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1110; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
1111; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1112; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 scc
1113; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1114; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
1115; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1116; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1117; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1118;
1119; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_atomicrmw:
1120; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1121; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1122; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1123; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1124; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1125; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1126; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
1127; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1128; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 scc
1129; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1130; GFX90A-TGSPLIT-NEXT:    buffer_invl2
1131; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1132; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1133; GFX90A-TGSPLIT-NEXT:    s_endpgm
1134    i32* %out, i32 %in) {
1135entry:
1136  %val = atomicrmw volatile xchg i32* %out, i32 %in seq_cst
1137  ret void
1138}
1139
1140define amdgpu_kernel void @flat_system_acquire_ret_atomicrmw(
1141; GFX7-LABEL: flat_system_acquire_ret_atomicrmw:
1142; GFX7:       ; %bb.0: ; %entry
1143; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1144; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1145; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1146; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1147; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1148; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1149; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1150; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1151; GFX7-NEXT:    buffer_wbinvl1_vol
1152; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1153; GFX7-NEXT:    flat_store_dword v[0:1], v2
1154; GFX7-NEXT:    s_endpgm
1155;
1156; GFX10-WGP-LABEL: flat_system_acquire_ret_atomicrmw:
1157; GFX10-WGP:       ; %bb.0: ; %entry
1158; GFX10-WGP-NEXT:    s_clause 0x1
1159; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1160; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1161; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1162; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1163; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1164; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1165; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1166; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1167; GFX10-WGP-NEXT:    buffer_gl0_inv
1168; GFX10-WGP-NEXT:    buffer_gl1_inv
1169; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1170; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1171; GFX10-WGP-NEXT:    s_endpgm
1172;
1173; GFX10-CU-LABEL: flat_system_acquire_ret_atomicrmw:
1174; GFX10-CU:       ; %bb.0: ; %entry
1175; GFX10-CU-NEXT:    s_clause 0x1
1176; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1177; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1178; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1179; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1180; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1181; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1182; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1183; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1184; GFX10-CU-NEXT:    buffer_gl0_inv
1185; GFX10-CU-NEXT:    buffer_gl1_inv
1186; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1187; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1188; GFX10-CU-NEXT:    s_endpgm
1189;
1190; SKIP-CACHE-INV-LABEL: flat_system_acquire_ret_atomicrmw:
1191; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1192; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
1193; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
1194; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1195; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1196; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1197; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1198; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1199; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1200; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1201; SKIP-CACHE-INV-NEXT:    s_endpgm
1202;
1203; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_ret_atomicrmw:
1204; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1205; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1206; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1207; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1208; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1209; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1210; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc scc
1211; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1212; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
1213; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1214; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1215; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1216; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1217;
1218; GFX90A-TGSPLIT-LABEL: flat_system_acquire_ret_atomicrmw:
1219; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1220; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1221; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1222; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1223; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1224; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1225; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc scc
1226; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1227; GFX90A-TGSPLIT-NEXT:    buffer_invl2
1228; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1229; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1230; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1231; GFX90A-TGSPLIT-NEXT:    s_endpgm
1232    i32* %out, i32 %in) {
1233entry:
1234  %val = atomicrmw volatile xchg i32* %out, i32 %in acquire
1235  store i32 %val, i32* %out, align 4
1236  ret void
1237}
1238
1239define amdgpu_kernel void @flat_system_acq_rel_ret_atomicrmw(
1240; GFX7-LABEL: flat_system_acq_rel_ret_atomicrmw:
1241; GFX7:       ; %bb.0: ; %entry
1242; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1243; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1244; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1245; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1246; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1247; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1248; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1249; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1250; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1251; GFX7-NEXT:    buffer_wbinvl1_vol
1252; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1253; GFX7-NEXT:    flat_store_dword v[0:1], v2
1254; GFX7-NEXT:    s_endpgm
1255;
1256; GFX10-WGP-LABEL: flat_system_acq_rel_ret_atomicrmw:
1257; GFX10-WGP:       ; %bb.0: ; %entry
1258; GFX10-WGP-NEXT:    s_clause 0x1
1259; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1260; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1261; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1262; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1263; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1264; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1265; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1266; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1267; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1268; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1269; GFX10-WGP-NEXT:    buffer_gl0_inv
1270; GFX10-WGP-NEXT:    buffer_gl1_inv
1271; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1272; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1273; GFX10-WGP-NEXT:    s_endpgm
1274;
1275; GFX10-CU-LABEL: flat_system_acq_rel_ret_atomicrmw:
1276; GFX10-CU:       ; %bb.0: ; %entry
1277; GFX10-CU-NEXT:    s_clause 0x1
1278; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1279; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1280; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1281; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1282; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1283; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1284; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1285; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1286; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1287; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1288; GFX10-CU-NEXT:    buffer_gl0_inv
1289; GFX10-CU-NEXT:    buffer_gl1_inv
1290; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1291; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1292; GFX10-CU-NEXT:    s_endpgm
1293;
1294; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_ret_atomicrmw:
1295; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1296; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
1297; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
1298; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1299; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1300; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1301; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1302; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1303; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1304; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1305; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1306; SKIP-CACHE-INV-NEXT:    s_endpgm
1307;
1308; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_ret_atomicrmw:
1309; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1310; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1311; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1312; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1313; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1314; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1315; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
1316; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1317; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc scc
1318; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1319; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
1320; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1321; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1322; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1323; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1324;
1325; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_ret_atomicrmw:
1326; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1327; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1328; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1329; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1330; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1331; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1332; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
1333; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1334; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc scc
1335; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1336; GFX90A-TGSPLIT-NEXT:    buffer_invl2
1337; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1338; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1339; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1340; GFX90A-TGSPLIT-NEXT:    s_endpgm
1341    i32* %out, i32 %in) {
1342entry:
1343  %val = atomicrmw volatile xchg i32* %out, i32 %in acq_rel
1344  store i32 %val, i32* %out, align 4
1345  ret void
1346}
1347
1348define amdgpu_kernel void @flat_system_seq_cst_ret_atomicrmw(
1349; GFX7-LABEL: flat_system_seq_cst_ret_atomicrmw:
1350; GFX7:       ; %bb.0: ; %entry
1351; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1352; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1353; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1354; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1355; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1356; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1357; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1358; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1359; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1360; GFX7-NEXT:    buffer_wbinvl1_vol
1361; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1362; GFX7-NEXT:    flat_store_dword v[0:1], v2
1363; GFX7-NEXT:    s_endpgm
1364;
1365; GFX10-WGP-LABEL: flat_system_seq_cst_ret_atomicrmw:
1366; GFX10-WGP:       ; %bb.0: ; %entry
1367; GFX10-WGP-NEXT:    s_clause 0x1
1368; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1369; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1370; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1371; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1372; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1373; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1374; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1375; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1376; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1377; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1378; GFX10-WGP-NEXT:    buffer_gl0_inv
1379; GFX10-WGP-NEXT:    buffer_gl1_inv
1380; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1381; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1382; GFX10-WGP-NEXT:    s_endpgm
1383;
1384; GFX10-CU-LABEL: flat_system_seq_cst_ret_atomicrmw:
1385; GFX10-CU:       ; %bb.0: ; %entry
1386; GFX10-CU-NEXT:    s_clause 0x1
1387; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1388; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1389; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1390; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1391; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1392; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1393; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1394; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1395; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1396; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1397; GFX10-CU-NEXT:    buffer_gl0_inv
1398; GFX10-CU-NEXT:    buffer_gl1_inv
1399; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1400; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1401; GFX10-CU-NEXT:    s_endpgm
1402;
1403; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_ret_atomicrmw:
1404; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1405; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
1406; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
1407; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1408; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1409; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1410; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1411; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1412; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1413; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1414; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1415; SKIP-CACHE-INV-NEXT:    s_endpgm
1416;
1417; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_ret_atomicrmw:
1418; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1419; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1420; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1421; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1422; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1423; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1424; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
1425; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1426; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc scc
1427; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1428; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
1429; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1430; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1431; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1432; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1433;
1434; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_ret_atomicrmw:
1435; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1436; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1437; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1438; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1439; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1440; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1441; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
1442; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1443; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc scc
1444; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1445; GFX90A-TGSPLIT-NEXT:    buffer_invl2
1446; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1447; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1448; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1449; GFX90A-TGSPLIT-NEXT:    s_endpgm
1450    i32* %out, i32 %in) {
1451entry:
1452  %val = atomicrmw volatile xchg i32* %out, i32 %in seq_cst
1453  store i32 %val, i32* %out, align 4
1454  ret void
1455}
1456
1457define amdgpu_kernel void @flat_system_monotonic_monotonic_cmpxchg(
1458; GFX7-LABEL: flat_system_monotonic_monotonic_cmpxchg:
1459; GFX7:       ; %bb.0: ; %entry
1460; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1461; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1462; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1463; GFX7-NEXT:    s_add_u32 s0, s0, 16
1464; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1465; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1466; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1467; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1468; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1469; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1470; GFX7-NEXT:    s_endpgm
1471;
1472; GFX10-WGP-LABEL: flat_system_monotonic_monotonic_cmpxchg:
1473; GFX10-WGP:       ; %bb.0: ; %entry
1474; GFX10-WGP-NEXT:    s_clause 0x1
1475; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1476; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1477; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1478; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1479; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1480; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1481; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1482; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1483; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1484; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1485; GFX10-WGP-NEXT:    s_endpgm
1486;
1487; GFX10-CU-LABEL: flat_system_monotonic_monotonic_cmpxchg:
1488; GFX10-CU:       ; %bb.0: ; %entry
1489; GFX10-CU-NEXT:    s_clause 0x1
1490; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1491; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1492; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1493; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1494; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1495; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1496; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1497; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1498; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1499; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1500; GFX10-CU-NEXT:    s_endpgm
1501;
1502; SKIP-CACHE-INV-LABEL: flat_system_monotonic_monotonic_cmpxchg:
1503; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1504; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
1505; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
1506; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1507; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1508; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1509; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1510; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1511; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1512; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1513; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1514; SKIP-CACHE-INV-NEXT:    s_endpgm
1515;
1516; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_monotonic_cmpxchg:
1517; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1518; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1519; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1520; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1521; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1522; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1523; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
1524; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1525;
1526; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_monotonic_cmpxchg:
1527; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1528; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1529; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1530; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1531; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1532; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1533; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
1534; GFX90A-TGSPLIT-NEXT:    s_endpgm
1535    i32* %out, i32 %in, i32 %old) {
1536entry:
1537  %gep = getelementptr i32, i32* %out, i32 4
1538  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in monotonic monotonic
1539  ret void
1540}
1541
1542define amdgpu_kernel void @flat_system_acquire_monotonic_cmpxchg(
1543; GFX7-LABEL: flat_system_acquire_monotonic_cmpxchg:
1544; GFX7:       ; %bb.0: ; %entry
1545; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1546; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1547; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1548; GFX7-NEXT:    s_add_u32 s0, s0, 16
1549; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1550; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1551; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1552; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1553; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1554; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1555; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1556; GFX7-NEXT:    buffer_wbinvl1_vol
1557; GFX7-NEXT:    s_endpgm
1558;
1559; GFX10-WGP-LABEL: flat_system_acquire_monotonic_cmpxchg:
1560; GFX10-WGP:       ; %bb.0: ; %entry
1561; GFX10-WGP-NEXT:    s_clause 0x1
1562; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1563; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1564; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1565; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1566; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1567; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1568; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1569; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1570; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1571; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1572; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1573; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1574; GFX10-WGP-NEXT:    buffer_gl0_inv
1575; GFX10-WGP-NEXT:    buffer_gl1_inv
1576; GFX10-WGP-NEXT:    s_endpgm
1577;
1578; GFX10-CU-LABEL: flat_system_acquire_monotonic_cmpxchg:
1579; GFX10-CU:       ; %bb.0: ; %entry
1580; GFX10-CU-NEXT:    s_clause 0x1
1581; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1582; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1583; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1584; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1585; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1586; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1587; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1588; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1589; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1590; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1591; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1592; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1593; GFX10-CU-NEXT:    buffer_gl0_inv
1594; GFX10-CU-NEXT:    buffer_gl1_inv
1595; GFX10-CU-NEXT:    s_endpgm
1596;
1597; SKIP-CACHE-INV-LABEL: flat_system_acquire_monotonic_cmpxchg:
1598; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1599; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
1600; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
1601; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1602; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1603; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1604; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1605; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1606; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1607; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1608; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1609; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1610; SKIP-CACHE-INV-NEXT:    s_endpgm
1611;
1612; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_monotonic_cmpxchg:
1613; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1614; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1615; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1616; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1617; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1618; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1619; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
1620; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1621; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
1622; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1623; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1624; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1625;
1626; GFX90A-TGSPLIT-LABEL: flat_system_acquire_monotonic_cmpxchg:
1627; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1628; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1629; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1630; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1631; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1632; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1633; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
1634; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1635; GFX90A-TGSPLIT-NEXT:    buffer_invl2
1636; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1637; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1638; GFX90A-TGSPLIT-NEXT:    s_endpgm
1639    i32* %out, i32 %in, i32 %old) {
1640entry:
1641  %gep = getelementptr i32, i32* %out, i32 4
1642  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acquire monotonic
1643  ret void
1644}
1645
1646define amdgpu_kernel void @flat_system_release_monotonic_cmpxchg(
1647; GFX7-LABEL: flat_system_release_monotonic_cmpxchg:
1648; GFX7:       ; %bb.0: ; %entry
1649; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1650; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1651; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1652; GFX7-NEXT:    s_add_u32 s0, s0, 16
1653; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1654; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1655; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1656; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1657; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1658; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1659; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1660; GFX7-NEXT:    s_endpgm
1661;
1662; GFX10-WGP-LABEL: flat_system_release_monotonic_cmpxchg:
1663; GFX10-WGP:       ; %bb.0: ; %entry
1664; GFX10-WGP-NEXT:    s_clause 0x1
1665; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1666; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1667; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1668; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1669; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1670; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1671; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1672; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1673; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1674; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1675; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1676; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1677; GFX10-WGP-NEXT:    s_endpgm
1678;
1679; GFX10-CU-LABEL: flat_system_release_monotonic_cmpxchg:
1680; GFX10-CU:       ; %bb.0: ; %entry
1681; GFX10-CU-NEXT:    s_clause 0x1
1682; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1683; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1684; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1685; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1686; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1687; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1688; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1689; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1690; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1691; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1692; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1693; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1694; GFX10-CU-NEXT:    s_endpgm
1695;
1696; SKIP-CACHE-INV-LABEL: flat_system_release_monotonic_cmpxchg:
1697; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1698; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
1699; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
1700; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1701; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1702; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1703; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1704; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1705; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1706; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1707; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1708; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1709; SKIP-CACHE-INV-NEXT:    s_endpgm
1710;
1711; GFX90A-NOTTGSPLIT-LABEL: flat_system_release_monotonic_cmpxchg:
1712; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1713; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1714; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1715; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1716; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1717; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1718; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
1719; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1720; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
1721; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1722;
1723; GFX90A-TGSPLIT-LABEL: flat_system_release_monotonic_cmpxchg:
1724; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1725; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1726; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1727; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1728; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1729; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1730; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
1731; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1732; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
1733; GFX90A-TGSPLIT-NEXT:    s_endpgm
1734    i32* %out, i32 %in, i32 %old) {
1735entry:
1736  %gep = getelementptr i32, i32* %out, i32 4
1737  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in release monotonic
1738  ret void
1739}
1740
1741define amdgpu_kernel void @flat_system_acq_rel_monotonic_cmpxchg(
1742; GFX7-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
1743; GFX7:       ; %bb.0: ; %entry
1744; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1745; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1746; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1747; GFX7-NEXT:    s_add_u32 s0, s0, 16
1748; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1749; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1750; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1751; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1752; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1753; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1754; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1755; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1756; GFX7-NEXT:    buffer_wbinvl1_vol
1757; GFX7-NEXT:    s_endpgm
1758;
1759; GFX10-WGP-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
1760; GFX10-WGP:       ; %bb.0: ; %entry
1761; GFX10-WGP-NEXT:    s_clause 0x1
1762; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1763; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1764; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1765; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1766; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1767; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1768; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1769; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1770; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1771; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1772; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1773; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1774; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1775; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1776; GFX10-WGP-NEXT:    buffer_gl0_inv
1777; GFX10-WGP-NEXT:    buffer_gl1_inv
1778; GFX10-WGP-NEXT:    s_endpgm
1779;
1780; GFX10-CU-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
1781; GFX10-CU:       ; %bb.0: ; %entry
1782; GFX10-CU-NEXT:    s_clause 0x1
1783; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1784; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1785; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1786; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1787; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1788; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1789; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1790; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1791; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1792; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1793; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1794; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1795; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1796; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1797; GFX10-CU-NEXT:    buffer_gl0_inv
1798; GFX10-CU-NEXT:    buffer_gl1_inv
1799; GFX10-CU-NEXT:    s_endpgm
1800;
1801; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
1802; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1803; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
1804; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
1805; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1806; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1807; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1808; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1809; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1810; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1811; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1812; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1813; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1814; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1815; SKIP-CACHE-INV-NEXT:    s_endpgm
1816;
1817; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
1818; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1819; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1820; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1821; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1822; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1823; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1824; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
1825; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1826; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
1827; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1828; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
1829; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1830; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1831; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1832;
1833; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
1834; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1835; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1836; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1837; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1838; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1839; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1840; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
1841; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1842; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
1843; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1844; GFX90A-TGSPLIT-NEXT:    buffer_invl2
1845; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1846; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1847; GFX90A-TGSPLIT-NEXT:    s_endpgm
1848    i32* %out, i32 %in, i32 %old) {
1849entry:
1850  %gep = getelementptr i32, i32* %out, i32 4
1851  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acq_rel monotonic
1852  ret void
1853}
1854
1855define amdgpu_kernel void @flat_system_seq_cst_monotonic_cmpxchg(
1856; GFX7-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
1857; GFX7:       ; %bb.0: ; %entry
1858; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1859; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1860; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1861; GFX7-NEXT:    s_add_u32 s0, s0, 16
1862; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1863; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1864; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1865; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1866; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1867; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1868; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1869; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1870; GFX7-NEXT:    buffer_wbinvl1_vol
1871; GFX7-NEXT:    s_endpgm
1872;
1873; GFX10-WGP-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
1874; GFX10-WGP:       ; %bb.0: ; %entry
1875; GFX10-WGP-NEXT:    s_clause 0x1
1876; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1877; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1878; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1879; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1880; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1881; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1882; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1883; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1884; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1885; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1886; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1887; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1888; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1889; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1890; GFX10-WGP-NEXT:    buffer_gl0_inv
1891; GFX10-WGP-NEXT:    buffer_gl1_inv
1892; GFX10-WGP-NEXT:    s_endpgm
1893;
1894; GFX10-CU-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
1895; GFX10-CU:       ; %bb.0: ; %entry
1896; GFX10-CU-NEXT:    s_clause 0x1
1897; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1898; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1899; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1900; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1901; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1902; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1903; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1904; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1905; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1906; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1907; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1908; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1909; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1910; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1911; GFX10-CU-NEXT:    buffer_gl0_inv
1912; GFX10-CU-NEXT:    buffer_gl1_inv
1913; GFX10-CU-NEXT:    s_endpgm
1914;
1915; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
1916; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1917; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
1918; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
1919; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1920; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1921; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1922; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1923; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1924; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1925; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1926; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1927; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1928; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1929; SKIP-CACHE-INV-NEXT:    s_endpgm
1930;
1931; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
1932; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1933; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1934; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1935; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1936; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1937; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1938; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
1939; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1940; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
1941; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1942; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
1943; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1944; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1945; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1946;
1947; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
1948; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1949; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1950; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1951; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1952; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1953; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1954; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
1955; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1956; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
1957; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1958; GFX90A-TGSPLIT-NEXT:    buffer_invl2
1959; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1960; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1961; GFX90A-TGSPLIT-NEXT:    s_endpgm
1962    i32* %out, i32 %in, i32 %old) {
1963entry:
1964  %gep = getelementptr i32, i32* %out, i32 4
1965  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst monotonic
1966  ret void
1967}
1968
1969define amdgpu_kernel void @flat_system_acquire_acquire_cmpxchg(
1970; GFX7-LABEL: flat_system_acquire_acquire_cmpxchg:
1971; GFX7:       ; %bb.0: ; %entry
1972; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1973; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1974; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1975; GFX7-NEXT:    s_add_u32 s0, s0, 16
1976; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1977; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1978; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1979; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1980; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1981; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1982; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1983; GFX7-NEXT:    buffer_wbinvl1_vol
1984; GFX7-NEXT:    s_endpgm
1985;
1986; GFX10-WGP-LABEL: flat_system_acquire_acquire_cmpxchg:
1987; GFX10-WGP:       ; %bb.0: ; %entry
1988; GFX10-WGP-NEXT:    s_clause 0x1
1989; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1990; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1991; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1992; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1993; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1994; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1995; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1996; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1997; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1998; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1999; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2000; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2001; GFX10-WGP-NEXT:    buffer_gl0_inv
2002; GFX10-WGP-NEXT:    buffer_gl1_inv
2003; GFX10-WGP-NEXT:    s_endpgm
2004;
2005; GFX10-CU-LABEL: flat_system_acquire_acquire_cmpxchg:
2006; GFX10-CU:       ; %bb.0: ; %entry
2007; GFX10-CU-NEXT:    s_clause 0x1
2008; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2009; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2010; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2011; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2012; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2013; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2014; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2015; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2016; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2017; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2018; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2019; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2020; GFX10-CU-NEXT:    buffer_gl0_inv
2021; GFX10-CU-NEXT:    buffer_gl1_inv
2022; GFX10-CU-NEXT:    s_endpgm
2023;
2024; SKIP-CACHE-INV-LABEL: flat_system_acquire_acquire_cmpxchg:
2025; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2026; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
2027; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2028; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2029; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2030; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2031; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2032; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2033; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2034; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2035; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2036; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2037; SKIP-CACHE-INV-NEXT:    s_endpgm
2038;
2039; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_acquire_cmpxchg:
2040; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2041; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2042; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2043; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2044; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2045; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2046; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
2047; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2048; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
2049; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2050; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2051; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2052;
2053; GFX90A-TGSPLIT-LABEL: flat_system_acquire_acquire_cmpxchg:
2054; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2055; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2056; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2057; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2058; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2059; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2060; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
2061; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2062; GFX90A-TGSPLIT-NEXT:    buffer_invl2
2063; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2064; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2065; GFX90A-TGSPLIT-NEXT:    s_endpgm
2066    i32* %out, i32 %in, i32 %old) {
2067entry:
2068  %gep = getelementptr i32, i32* %out, i32 4
2069  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acquire acquire
2070  ret void
2071}
2072
2073define amdgpu_kernel void @flat_system_release_acquire_cmpxchg(
2074; GFX7-LABEL: flat_system_release_acquire_cmpxchg:
2075; GFX7:       ; %bb.0: ; %entry
2076; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2077; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2078; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2079; GFX7-NEXT:    s_add_u32 s0, s0, 16
2080; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2081; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2082; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2083; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2084; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2085; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2086; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2087; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2088; GFX7-NEXT:    buffer_wbinvl1_vol
2089; GFX7-NEXT:    s_endpgm
2090;
2091; GFX10-WGP-LABEL: flat_system_release_acquire_cmpxchg:
2092; GFX10-WGP:       ; %bb.0: ; %entry
2093; GFX10-WGP-NEXT:    s_clause 0x1
2094; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2095; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2096; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2097; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2098; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2099; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2100; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2101; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2102; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2103; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2104; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2105; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2106; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2107; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2108; GFX10-WGP-NEXT:    buffer_gl0_inv
2109; GFX10-WGP-NEXT:    buffer_gl1_inv
2110; GFX10-WGP-NEXT:    s_endpgm
2111;
2112; GFX10-CU-LABEL: flat_system_release_acquire_cmpxchg:
2113; GFX10-CU:       ; %bb.0: ; %entry
2114; GFX10-CU-NEXT:    s_clause 0x1
2115; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2116; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2117; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2118; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2119; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2120; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2121; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2122; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2123; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2124; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2125; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2126; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2127; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2128; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2129; GFX10-CU-NEXT:    buffer_gl0_inv
2130; GFX10-CU-NEXT:    buffer_gl1_inv
2131; GFX10-CU-NEXT:    s_endpgm
2132;
2133; SKIP-CACHE-INV-LABEL: flat_system_release_acquire_cmpxchg:
2134; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2135; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
2136; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2137; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2138; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2139; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2140; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2141; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2142; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2143; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2144; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2145; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2146; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2147; SKIP-CACHE-INV-NEXT:    s_endpgm
2148;
2149; GFX90A-NOTTGSPLIT-LABEL: flat_system_release_acquire_cmpxchg:
2150; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2151; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2152; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2153; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2154; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2155; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2156; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
2157; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2158; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
2159; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2160; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
2161; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2162; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2163; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2164;
2165; GFX90A-TGSPLIT-LABEL: flat_system_release_acquire_cmpxchg:
2166; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2167; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2168; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2169; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2170; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2171; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2172; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
2173; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2174; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
2175; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2176; GFX90A-TGSPLIT-NEXT:    buffer_invl2
2177; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2178; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2179; GFX90A-TGSPLIT-NEXT:    s_endpgm
2180    i32* %out, i32 %in, i32 %old) {
2181entry:
2182  %gep = getelementptr i32, i32* %out, i32 4
2183  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in release acquire
2184  ret void
2185}
2186
2187define amdgpu_kernel void @flat_system_acq_rel_acquire_cmpxchg(
2188; GFX7-LABEL: flat_system_acq_rel_acquire_cmpxchg:
2189; GFX7:       ; %bb.0: ; %entry
2190; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2191; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2192; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2193; GFX7-NEXT:    s_add_u32 s0, s0, 16
2194; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2195; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2196; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2197; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2198; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2199; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2200; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2201; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2202; GFX7-NEXT:    buffer_wbinvl1_vol
2203; GFX7-NEXT:    s_endpgm
2204;
2205; GFX10-WGP-LABEL: flat_system_acq_rel_acquire_cmpxchg:
2206; GFX10-WGP:       ; %bb.0: ; %entry
2207; GFX10-WGP-NEXT:    s_clause 0x1
2208; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2209; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2210; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2211; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2212; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2213; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2214; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2215; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2216; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2217; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2218; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2219; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2220; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2221; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2222; GFX10-WGP-NEXT:    buffer_gl0_inv
2223; GFX10-WGP-NEXT:    buffer_gl1_inv
2224; GFX10-WGP-NEXT:    s_endpgm
2225;
2226; GFX10-CU-LABEL: flat_system_acq_rel_acquire_cmpxchg:
2227; GFX10-CU:       ; %bb.0: ; %entry
2228; GFX10-CU-NEXT:    s_clause 0x1
2229; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2230; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2231; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2232; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2233; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2234; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2235; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2236; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2237; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2238; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2239; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2240; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2241; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2242; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2243; GFX10-CU-NEXT:    buffer_gl0_inv
2244; GFX10-CU-NEXT:    buffer_gl1_inv
2245; GFX10-CU-NEXT:    s_endpgm
2246;
2247; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_acquire_cmpxchg:
2248; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2249; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
2250; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2251; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2252; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2253; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2254; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2255; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2256; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2257; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2258; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2259; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2260; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2261; SKIP-CACHE-INV-NEXT:    s_endpgm
2262;
2263; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_acquire_cmpxchg:
2264; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2265; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2266; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2267; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2268; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2269; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2270; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
2271; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2272; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
2273; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2274; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
2275; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2276; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2277; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2278;
2279; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_acquire_cmpxchg:
2280; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2281; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2282; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2283; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2284; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2285; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2286; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
2287; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2288; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
2289; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2290; GFX90A-TGSPLIT-NEXT:    buffer_invl2
2291; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2292; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2293; GFX90A-TGSPLIT-NEXT:    s_endpgm
2294    i32* %out, i32 %in, i32 %old) {
2295entry:
2296  %gep = getelementptr i32, i32* %out, i32 4
2297  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acq_rel acquire
2298  ret void
2299}
2300
2301define amdgpu_kernel void @flat_system_seq_cst_acquire_cmpxchg(
2302; GFX7-LABEL: flat_system_seq_cst_acquire_cmpxchg:
2303; GFX7:       ; %bb.0: ; %entry
2304; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2305; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2306; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2307; GFX7-NEXT:    s_add_u32 s0, s0, 16
2308; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2309; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2310; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2311; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2312; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2313; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2314; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2315; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2316; GFX7-NEXT:    buffer_wbinvl1_vol
2317; GFX7-NEXT:    s_endpgm
2318;
2319; GFX10-WGP-LABEL: flat_system_seq_cst_acquire_cmpxchg:
2320; GFX10-WGP:       ; %bb.0: ; %entry
2321; GFX10-WGP-NEXT:    s_clause 0x1
2322; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2323; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2324; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2325; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2326; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2327; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2328; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2329; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2330; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2331; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2332; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2333; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2334; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2335; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2336; GFX10-WGP-NEXT:    buffer_gl0_inv
2337; GFX10-WGP-NEXT:    buffer_gl1_inv
2338; GFX10-WGP-NEXT:    s_endpgm
2339;
2340; GFX10-CU-LABEL: flat_system_seq_cst_acquire_cmpxchg:
2341; GFX10-CU:       ; %bb.0: ; %entry
2342; GFX10-CU-NEXT:    s_clause 0x1
2343; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2344; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2345; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2346; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2347; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2348; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2349; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2350; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2351; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2352; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2353; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2354; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2355; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2356; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2357; GFX10-CU-NEXT:    buffer_gl0_inv
2358; GFX10-CU-NEXT:    buffer_gl1_inv
2359; GFX10-CU-NEXT:    s_endpgm
2360;
2361; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_acquire_cmpxchg:
2362; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2363; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
2364; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2365; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2366; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2367; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2368; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2369; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2370; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2371; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2372; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2373; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2374; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2375; SKIP-CACHE-INV-NEXT:    s_endpgm
2376;
2377; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_acquire_cmpxchg:
2378; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2379; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2380; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2381; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2382; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2383; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2384; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
2385; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2386; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
2387; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2388; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
2389; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2390; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2391; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2392;
2393; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_acquire_cmpxchg:
2394; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2395; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2396; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2397; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2398; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2399; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2400; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
2401; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2402; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
2403; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2404; GFX90A-TGSPLIT-NEXT:    buffer_invl2
2405; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2406; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2407; GFX90A-TGSPLIT-NEXT:    s_endpgm
2408    i32* %out, i32 %in, i32 %old) {
2409entry:
2410  %gep = getelementptr i32, i32* %out, i32 4
2411  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst acquire
2412  ret void
2413}
2414
2415define amdgpu_kernel void @flat_system_seq_cst_seq_cst_cmpxchg(
2416; GFX7-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
2417; GFX7:       ; %bb.0: ; %entry
2418; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2419; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2420; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2421; GFX7-NEXT:    s_add_u32 s0, s0, 16
2422; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2423; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2424; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2425; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2426; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2427; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2428; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2429; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2430; GFX7-NEXT:    buffer_wbinvl1_vol
2431; GFX7-NEXT:    s_endpgm
2432;
2433; GFX10-WGP-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
2434; GFX10-WGP:       ; %bb.0: ; %entry
2435; GFX10-WGP-NEXT:    s_clause 0x1
2436; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2437; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2438; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2439; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2440; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2441; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2442; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2443; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2444; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2445; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2446; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2447; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2448; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2449; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2450; GFX10-WGP-NEXT:    buffer_gl0_inv
2451; GFX10-WGP-NEXT:    buffer_gl1_inv
2452; GFX10-WGP-NEXT:    s_endpgm
2453;
2454; GFX10-CU-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
2455; GFX10-CU:       ; %bb.0: ; %entry
2456; GFX10-CU-NEXT:    s_clause 0x1
2457; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2458; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2459; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2460; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2461; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2462; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2463; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2464; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2465; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2466; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2467; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2468; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2469; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2470; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2471; GFX10-CU-NEXT:    buffer_gl0_inv
2472; GFX10-CU-NEXT:    buffer_gl1_inv
2473; GFX10-CU-NEXT:    s_endpgm
2474;
2475; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
2476; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2477; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
2478; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2479; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2480; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2481; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2482; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2483; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2484; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2485; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2486; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2487; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2488; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2489; SKIP-CACHE-INV-NEXT:    s_endpgm
2490;
2491; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
2492; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2493; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2494; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2495; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2496; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2497; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2498; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
2499; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2500; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
2501; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2502; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
2503; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2504; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2505; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2506;
2507; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
2508; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2509; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2510; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2511; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2512; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2513; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2514; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
2515; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2516; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
2517; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2518; GFX90A-TGSPLIT-NEXT:    buffer_invl2
2519; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2520; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2521; GFX90A-TGSPLIT-NEXT:    s_endpgm
2522    i32* %out, i32 %in, i32 %old) {
2523entry:
2524  %gep = getelementptr i32, i32* %out, i32 4
2525  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst seq_cst
2526  ret void
2527}
2528
2529define amdgpu_kernel void @flat_system_acquire_monotonic_ret_cmpxchg(
2530; GFX7-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
2531; GFX7:       ; %bb.0: ; %entry
2532; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2533; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2534; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2535; GFX7-NEXT:    s_add_u32 s4, s0, 16
2536; GFX7-NEXT:    s_addc_u32 s5, s1, 0
2537; GFX7-NEXT:    v_mov_b32_e32 v0, s4
2538; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2539; GFX7-NEXT:    v_mov_b32_e32 v1, s5
2540; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2541; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2542; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2543; GFX7-NEXT:    buffer_wbinvl1_vol
2544; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2545; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2546; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2547; GFX7-NEXT:    flat_store_dword v[0:1], v2
2548; GFX7-NEXT:    s_endpgm
2549;
2550; GFX10-WGP-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
2551; GFX10-WGP:       ; %bb.0: ; %entry
2552; GFX10-WGP-NEXT:    s_clause 0x1
2553; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2554; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2555; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2556; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
2557; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
2558; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
2559; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2560; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
2561; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2562; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2563; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2564; GFX10-WGP-NEXT:    buffer_gl0_inv
2565; GFX10-WGP-NEXT:    buffer_gl1_inv
2566; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2567; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2568; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2569; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
2570; GFX10-WGP-NEXT:    s_endpgm
2571;
2572; GFX10-CU-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
2573; GFX10-CU:       ; %bb.0: ; %entry
2574; GFX10-CU-NEXT:    s_clause 0x1
2575; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2576; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2577; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2578; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
2579; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
2580; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
2581; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2582; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
2583; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2584; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2585; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2586; GFX10-CU-NEXT:    buffer_gl0_inv
2587; GFX10-CU-NEXT:    buffer_gl1_inv
2588; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2589; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2590; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2591; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
2592; GFX10-CU-NEXT:    s_endpgm
2593;
2594; SKIP-CACHE-INV-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
2595; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2596; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
2597; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2598; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2599; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
2600; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
2601; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2602; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2603; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2604; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2605; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2606; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2607; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2608; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2609; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
2610; SKIP-CACHE-INV-NEXT:    s_endpgm
2611;
2612; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
2613; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2614; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2615; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2616; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2617; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2618; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2619; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc scc
2620; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2621; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
2622; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2623; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2624; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2625; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2626;
2627; GFX90A-TGSPLIT-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
2628; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2629; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2630; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2631; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2632; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2633; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2634; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc scc
2635; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2636; GFX90A-TGSPLIT-NEXT:    buffer_invl2
2637; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2638; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2639; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2640; GFX90A-TGSPLIT-NEXT:    s_endpgm
2641    i32* %out, i32 %in, i32 %old) {
2642entry:
2643  %gep = getelementptr i32, i32* %out, i32 4
2644  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acquire monotonic
2645  %val0 = extractvalue { i32, i1 } %val, 0
2646  store i32 %val0, i32* %out, align 4
2647  ret void
2648}
2649
2650define amdgpu_kernel void @flat_system_acq_rel_monotonic_ret_cmpxchg(
2651; GFX7-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
2652; GFX7:       ; %bb.0: ; %entry
2653; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2654; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2655; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2656; GFX7-NEXT:    s_add_u32 s4, s0, 16
2657; GFX7-NEXT:    s_addc_u32 s5, s1, 0
2658; GFX7-NEXT:    v_mov_b32_e32 v0, s4
2659; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2660; GFX7-NEXT:    v_mov_b32_e32 v1, s5
2661; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2662; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2663; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2664; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2665; GFX7-NEXT:    buffer_wbinvl1_vol
2666; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2667; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2668; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2669; GFX7-NEXT:    flat_store_dword v[0:1], v2
2670; GFX7-NEXT:    s_endpgm
2671;
2672; GFX10-WGP-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
2673; GFX10-WGP:       ; %bb.0: ; %entry
2674; GFX10-WGP-NEXT:    s_clause 0x1
2675; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2676; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2677; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2678; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
2679; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
2680; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
2681; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2682; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
2683; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2684; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2685; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2686; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2687; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2688; GFX10-WGP-NEXT:    buffer_gl0_inv
2689; GFX10-WGP-NEXT:    buffer_gl1_inv
2690; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2691; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2692; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2693; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
2694; GFX10-WGP-NEXT:    s_endpgm
2695;
2696; GFX10-CU-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
2697; GFX10-CU:       ; %bb.0: ; %entry
2698; GFX10-CU-NEXT:    s_clause 0x1
2699; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2700; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2701; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2702; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
2703; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
2704; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
2705; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2706; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
2707; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2708; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2709; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2710; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2711; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2712; GFX10-CU-NEXT:    buffer_gl0_inv
2713; GFX10-CU-NEXT:    buffer_gl1_inv
2714; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2715; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2716; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2717; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
2718; GFX10-CU-NEXT:    s_endpgm
2719;
2720; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
2721; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2722; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
2723; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2724; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2725; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
2726; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
2727; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2728; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2729; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2730; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2731; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2732; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2733; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2734; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2735; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2736; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
2737; SKIP-CACHE-INV-NEXT:    s_endpgm
2738;
2739; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
2740; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2741; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2742; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2743; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2744; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2745; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2746; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
2747; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2748; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc scc
2749; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2750; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
2751; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2752; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2753; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2754; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2755;
2756; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
2757; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2758; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2759; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2760; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2761; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2762; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2763; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
2764; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2765; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc scc
2766; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2767; GFX90A-TGSPLIT-NEXT:    buffer_invl2
2768; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2769; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2770; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2771; GFX90A-TGSPLIT-NEXT:    s_endpgm
2772    i32* %out, i32 %in, i32 %old) {
2773entry:
2774  %gep = getelementptr i32, i32* %out, i32 4
2775  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acq_rel monotonic
2776  %val0 = extractvalue { i32, i1 } %val, 0
2777  store i32 %val0, i32* %out, align 4
2778  ret void
2779}
2780
2781define amdgpu_kernel void @flat_system_seq_cst_monotonic_ret_cmpxchg(
2782; GFX7-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
2783; GFX7:       ; %bb.0: ; %entry
2784; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2785; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2786; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2787; GFX7-NEXT:    s_add_u32 s4, s0, 16
2788; GFX7-NEXT:    s_addc_u32 s5, s1, 0
2789; GFX7-NEXT:    v_mov_b32_e32 v0, s4
2790; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2791; GFX7-NEXT:    v_mov_b32_e32 v1, s5
2792; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2793; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2794; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2795; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2796; GFX7-NEXT:    buffer_wbinvl1_vol
2797; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2798; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2799; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2800; GFX7-NEXT:    flat_store_dword v[0:1], v2
2801; GFX7-NEXT:    s_endpgm
2802;
2803; GFX10-WGP-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
2804; GFX10-WGP:       ; %bb.0: ; %entry
2805; GFX10-WGP-NEXT:    s_clause 0x1
2806; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2807; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2808; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2809; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
2810; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
2811; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
2812; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2813; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
2814; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2815; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2816; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2817; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2818; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2819; GFX10-WGP-NEXT:    buffer_gl0_inv
2820; GFX10-WGP-NEXT:    buffer_gl1_inv
2821; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2822; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2823; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2824; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
2825; GFX10-WGP-NEXT:    s_endpgm
2826;
2827; GFX10-CU-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
2828; GFX10-CU:       ; %bb.0: ; %entry
2829; GFX10-CU-NEXT:    s_clause 0x1
2830; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2831; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2832; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2833; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
2834; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
2835; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
2836; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2837; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
2838; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2839; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2840; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2841; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2842; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2843; GFX10-CU-NEXT:    buffer_gl0_inv
2844; GFX10-CU-NEXT:    buffer_gl1_inv
2845; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2846; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2847; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2848; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
2849; GFX10-CU-NEXT:    s_endpgm
2850;
2851; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
2852; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2853; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
2854; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2855; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2856; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
2857; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
2858; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2859; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2860; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2861; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2862; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2863; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2864; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2865; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2866; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2867; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
2868; SKIP-CACHE-INV-NEXT:    s_endpgm
2869;
2870; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
2871; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2872; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2873; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2874; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2875; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2876; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2877; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
2878; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2879; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc scc
2880; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2881; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
2882; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2883; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2884; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2885; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2886;
2887; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
2888; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2889; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2890; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2891; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2892; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2893; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2894; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
2895; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2896; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc scc
2897; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2898; GFX90A-TGSPLIT-NEXT:    buffer_invl2
2899; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2900; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2901; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
2902; GFX90A-TGSPLIT-NEXT:    s_endpgm
2903    i32* %out, i32 %in, i32 %old) {
2904entry:
2905  %gep = getelementptr i32, i32* %out, i32 4
2906  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst monotonic
2907  %val0 = extractvalue { i32, i1 } %val, 0
2908  store i32 %val0, i32* %out, align 4
2909  ret void
2910}
2911
2912define amdgpu_kernel void @flat_system_acquire_acquire_ret_cmpxchg(
2913; GFX7-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
2914; GFX7:       ; %bb.0: ; %entry
2915; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2916; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2917; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2918; GFX7-NEXT:    s_add_u32 s4, s0, 16
2919; GFX7-NEXT:    s_addc_u32 s5, s1, 0
2920; GFX7-NEXT:    v_mov_b32_e32 v0, s4
2921; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2922; GFX7-NEXT:    v_mov_b32_e32 v1, s5
2923; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2924; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2925; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2926; GFX7-NEXT:    buffer_wbinvl1_vol
2927; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2928; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2929; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2930; GFX7-NEXT:    flat_store_dword v[0:1], v2
2931; GFX7-NEXT:    s_endpgm
2932;
2933; GFX10-WGP-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
2934; GFX10-WGP:       ; %bb.0: ; %entry
2935; GFX10-WGP-NEXT:    s_clause 0x1
2936; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2937; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2938; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2939; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
2940; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
2941; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
2942; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2943; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
2944; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2945; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2946; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2947; GFX10-WGP-NEXT:    buffer_gl0_inv
2948; GFX10-WGP-NEXT:    buffer_gl1_inv
2949; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2950; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2951; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2952; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
2953; GFX10-WGP-NEXT:    s_endpgm
2954;
2955; GFX10-CU-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
2956; GFX10-CU:       ; %bb.0: ; %entry
2957; GFX10-CU-NEXT:    s_clause 0x1
2958; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2959; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2960; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2961; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
2962; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
2963; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
2964; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2965; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
2966; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2967; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2968; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2969; GFX10-CU-NEXT:    buffer_gl0_inv
2970; GFX10-CU-NEXT:    buffer_gl1_inv
2971; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2972; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2973; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2974; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
2975; GFX10-CU-NEXT:    s_endpgm
2976;
2977; SKIP-CACHE-INV-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
2978; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2979; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
2980; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2981; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2982; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
2983; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
2984; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
2985; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2986; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
2987; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2988; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2989; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2990; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2991; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2992; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
2993; SKIP-CACHE-INV-NEXT:    s_endpgm
2994;
2995; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
2996; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2997; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2998; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2999; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3000; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3001; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3002; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc scc
3003; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3004; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
3005; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3006; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3007; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3008; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3009;
3010; GFX90A-TGSPLIT-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
3011; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3012; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3013; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3014; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3015; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3016; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3017; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc scc
3018; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3019; GFX90A-TGSPLIT-NEXT:    buffer_invl2
3020; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3021; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3022; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3023; GFX90A-TGSPLIT-NEXT:    s_endpgm
3024    i32* %out, i32 %in, i32 %old) {
3025entry:
3026  %gep = getelementptr i32, i32* %out, i32 4
3027  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acquire acquire
3028  %val0 = extractvalue { i32, i1 } %val, 0
3029  store i32 %val0, i32* %out, align 4
3030  ret void
3031}
3032
3033define amdgpu_kernel void @flat_system_release_acquire_ret_cmpxchg(
3034; GFX7-LABEL: flat_system_release_acquire_ret_cmpxchg:
3035; GFX7:       ; %bb.0: ; %entry
3036; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3037; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3038; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3039; GFX7-NEXT:    s_add_u32 s4, s0, 16
3040; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3041; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3042; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3043; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3044; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3045; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3046; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3047; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3048; GFX7-NEXT:    buffer_wbinvl1_vol
3049; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3050; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3051; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3052; GFX7-NEXT:    flat_store_dword v[0:1], v2
3053; GFX7-NEXT:    s_endpgm
3054;
3055; GFX10-WGP-LABEL: flat_system_release_acquire_ret_cmpxchg:
3056; GFX10-WGP:       ; %bb.0: ; %entry
3057; GFX10-WGP-NEXT:    s_clause 0x1
3058; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3059; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3060; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3061; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3062; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3063; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3064; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3065; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
3066; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3067; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3068; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3069; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3070; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3071; GFX10-WGP-NEXT:    buffer_gl0_inv
3072; GFX10-WGP-NEXT:    buffer_gl1_inv
3073; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3074; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3075; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3076; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3077; GFX10-WGP-NEXT:    s_endpgm
3078;
3079; GFX10-CU-LABEL: flat_system_release_acquire_ret_cmpxchg:
3080; GFX10-CU:       ; %bb.0: ; %entry
3081; GFX10-CU-NEXT:    s_clause 0x1
3082; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3083; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3084; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3085; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3086; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3087; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3088; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3089; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3090; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3091; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3092; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3093; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3094; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3095; GFX10-CU-NEXT:    buffer_gl0_inv
3096; GFX10-CU-NEXT:    buffer_gl1_inv
3097; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3098; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3099; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3100; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3101; GFX10-CU-NEXT:    s_endpgm
3102;
3103; SKIP-CACHE-INV-LABEL: flat_system_release_acquire_ret_cmpxchg:
3104; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3105; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
3106; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
3107; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3108; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3109; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3110; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3111; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3112; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3113; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3114; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3115; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3116; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3117; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3118; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3119; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3120; SKIP-CACHE-INV-NEXT:    s_endpgm
3121;
3122; GFX90A-NOTTGSPLIT-LABEL: flat_system_release_acquire_ret_cmpxchg:
3123; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3124; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3125; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3126; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3127; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3128; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3129; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
3130; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3131; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc scc
3132; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3133; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
3134; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3135; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3136; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3137; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3138;
3139; GFX90A-TGSPLIT-LABEL: flat_system_release_acquire_ret_cmpxchg:
3140; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3141; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3142; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3143; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3144; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3145; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3146; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
3147; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3148; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc scc
3149; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3150; GFX90A-TGSPLIT-NEXT:    buffer_invl2
3151; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3152; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3153; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3154; GFX90A-TGSPLIT-NEXT:    s_endpgm
3155    i32* %out, i32 %in, i32 %old) {
3156entry:
3157  %gep = getelementptr i32, i32* %out, i32 4
3158  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in release acquire
3159  %val0 = extractvalue { i32, i1 } %val, 0
3160  store i32 %val0, i32* %out, align 4
3161  ret void
3162}
3163
3164define amdgpu_kernel void @flat_system_acq_rel_acquire_ret_cmpxchg(
3165; GFX7-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
3166; GFX7:       ; %bb.0: ; %entry
3167; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3168; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3169; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3170; GFX7-NEXT:    s_add_u32 s4, s0, 16
3171; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3172; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3173; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3174; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3175; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3176; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3177; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3178; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3179; GFX7-NEXT:    buffer_wbinvl1_vol
3180; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3181; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3182; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3183; GFX7-NEXT:    flat_store_dword v[0:1], v2
3184; GFX7-NEXT:    s_endpgm
3185;
3186; GFX10-WGP-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
3187; GFX10-WGP:       ; %bb.0: ; %entry
3188; GFX10-WGP-NEXT:    s_clause 0x1
3189; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3190; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3191; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3192; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3193; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3194; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3195; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3196; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
3197; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3198; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3199; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3200; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3201; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3202; GFX10-WGP-NEXT:    buffer_gl0_inv
3203; GFX10-WGP-NEXT:    buffer_gl1_inv
3204; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3205; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3206; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3207; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3208; GFX10-WGP-NEXT:    s_endpgm
3209;
3210; GFX10-CU-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
3211; GFX10-CU:       ; %bb.0: ; %entry
3212; GFX10-CU-NEXT:    s_clause 0x1
3213; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3214; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3215; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3216; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3217; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3218; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3219; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3220; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3221; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3222; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3223; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3224; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3225; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3226; GFX10-CU-NEXT:    buffer_gl0_inv
3227; GFX10-CU-NEXT:    buffer_gl1_inv
3228; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3229; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3230; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3231; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3232; GFX10-CU-NEXT:    s_endpgm
3233;
3234; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
3235; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3236; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
3237; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
3238; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3239; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3240; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3241; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3242; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3243; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3244; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3245; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3246; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3247; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3248; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3249; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3250; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3251; SKIP-CACHE-INV-NEXT:    s_endpgm
3252;
3253; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
3254; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3255; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3256; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3257; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3258; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3259; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3260; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
3261; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3262; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc scc
3263; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3264; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
3265; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3266; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3267; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3268; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3269;
3270; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
3271; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3272; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3273; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3274; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3275; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3276; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3277; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
3278; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3279; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc scc
3280; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3281; GFX90A-TGSPLIT-NEXT:    buffer_invl2
3282; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3283; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3284; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3285; GFX90A-TGSPLIT-NEXT:    s_endpgm
3286    i32* %out, i32 %in, i32 %old) {
3287entry:
3288  %gep = getelementptr i32, i32* %out, i32 4
3289  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acq_rel acquire
3290  %val0 = extractvalue { i32, i1 } %val, 0
3291  store i32 %val0, i32* %out, align 4
3292  ret void
3293}
3294
3295define amdgpu_kernel void @flat_system_seq_cst_acquire_ret_cmpxchg(
3296; GFX7-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
3297; GFX7:       ; %bb.0: ; %entry
3298; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3299; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3300; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3301; GFX7-NEXT:    s_add_u32 s4, s0, 16
3302; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3303; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3304; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3305; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3306; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3307; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3308; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3309; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3310; GFX7-NEXT:    buffer_wbinvl1_vol
3311; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3312; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3313; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3314; GFX7-NEXT:    flat_store_dword v[0:1], v2
3315; GFX7-NEXT:    s_endpgm
3316;
3317; GFX10-WGP-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
3318; GFX10-WGP:       ; %bb.0: ; %entry
3319; GFX10-WGP-NEXT:    s_clause 0x1
3320; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3321; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3322; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3323; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3324; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3325; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3326; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3327; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
3328; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3329; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3330; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3331; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3332; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3333; GFX10-WGP-NEXT:    buffer_gl0_inv
3334; GFX10-WGP-NEXT:    buffer_gl1_inv
3335; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3336; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3337; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3338; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3339; GFX10-WGP-NEXT:    s_endpgm
3340;
3341; GFX10-CU-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
3342; GFX10-CU:       ; %bb.0: ; %entry
3343; GFX10-CU-NEXT:    s_clause 0x1
3344; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3345; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3346; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3347; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3348; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3349; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3350; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3351; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3352; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3353; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3354; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3355; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3356; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3357; GFX10-CU-NEXT:    buffer_gl0_inv
3358; GFX10-CU-NEXT:    buffer_gl1_inv
3359; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3360; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3361; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3362; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3363; GFX10-CU-NEXT:    s_endpgm
3364;
3365; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
3366; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3367; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
3368; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
3369; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3370; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3371; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3372; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3373; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3374; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3375; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3376; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3377; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3378; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3379; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3380; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3381; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3382; SKIP-CACHE-INV-NEXT:    s_endpgm
3383;
3384; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
3385; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3386; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3387; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3388; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3389; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3390; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3391; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
3392; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3393; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc scc
3394; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3395; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
3396; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3397; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3398; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3399; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3400;
3401; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
3402; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3403; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3404; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3405; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3406; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3407; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3408; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
3409; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3410; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc scc
3411; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3412; GFX90A-TGSPLIT-NEXT:    buffer_invl2
3413; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3414; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3415; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3416; GFX90A-TGSPLIT-NEXT:    s_endpgm
3417    i32* %out, i32 %in, i32 %old) {
3418entry:
3419  %gep = getelementptr i32, i32* %out, i32 4
3420  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst acquire
3421  %val0 = extractvalue { i32, i1 } %val, 0
3422  store i32 %val0, i32* %out, align 4
3423  ret void
3424}
3425
3426define amdgpu_kernel void @flat_system_seq_cst_seq_cst_ret_cmpxchg(
3427; GFX7-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
3428; GFX7:       ; %bb.0: ; %entry
3429; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3430; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3431; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3432; GFX7-NEXT:    s_add_u32 s4, s0, 16
3433; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3434; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3435; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3436; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3437; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3438; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3439; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3440; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3441; GFX7-NEXT:    buffer_wbinvl1_vol
3442; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3443; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3444; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3445; GFX7-NEXT:    flat_store_dword v[0:1], v2
3446; GFX7-NEXT:    s_endpgm
3447;
3448; GFX10-WGP-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
3449; GFX10-WGP:       ; %bb.0: ; %entry
3450; GFX10-WGP-NEXT:    s_clause 0x1
3451; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3452; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3453; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3454; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3455; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3456; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3457; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3458; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
3459; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3460; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3461; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3462; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3463; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3464; GFX10-WGP-NEXT:    buffer_gl0_inv
3465; GFX10-WGP-NEXT:    buffer_gl1_inv
3466; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3467; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3468; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3469; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3470; GFX10-WGP-NEXT:    s_endpgm
3471;
3472; GFX10-CU-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
3473; GFX10-CU:       ; %bb.0: ; %entry
3474; GFX10-CU-NEXT:    s_clause 0x1
3475; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3476; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3477; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3478; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3479; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3480; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3481; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3482; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3483; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3484; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3485; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3486; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3487; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3488; GFX10-CU-NEXT:    buffer_gl0_inv
3489; GFX10-CU-NEXT:    buffer_gl1_inv
3490; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3491; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3492; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3493; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3494; GFX10-CU-NEXT:    s_endpgm
3495;
3496; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
3497; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3498; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
3499; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
3500; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3501; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3502; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3503; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3504; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3505; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3506; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3507; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3508; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3509; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3510; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3511; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3512; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3513; SKIP-CACHE-INV-NEXT:    s_endpgm
3514;
3515; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
3516; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3517; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3518; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3519; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3520; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3521; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3522; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
3523; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3524; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc scc
3525; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3526; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
3527; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3528; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3529; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3530; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3531;
3532; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
3533; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3534; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3535; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3536; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3537; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3538; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3539; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
3540; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3541; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc scc
3542; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3543; GFX90A-TGSPLIT-NEXT:    buffer_invl2
3544; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3545; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3546; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3547; GFX90A-TGSPLIT-NEXT:    s_endpgm
3548    i32* %out, i32 %in, i32 %old) {
3549entry:
3550  %gep = getelementptr i32, i32* %out, i32 4
3551  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst seq_cst
3552  %val0 = extractvalue { i32, i1 } %val, 0
3553  store i32 %val0, i32* %out, align 4
3554  ret void
3555}
3556
3557define amdgpu_kernel void @flat_system_one_as_unordered_load(
3558; GFX7-LABEL: flat_system_one_as_unordered_load:
3559; GFX7:       ; %bb.0: ; %entry
3560; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3561; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3562; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3563; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3564; GFX7-NEXT:    flat_load_dword v0, v[0:1]
3565; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3566; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3567; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3568; GFX7-NEXT:    flat_store_dword v[2:3], v0
3569; GFX7-NEXT:    s_endpgm
3570;
3571; GFX10-WGP-LABEL: flat_system_one_as_unordered_load:
3572; GFX10-WGP:       ; %bb.0: ; %entry
3573; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3574; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3575; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3576; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3577; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
3578; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
3579; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
3580; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3581; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3582; GFX10-WGP-NEXT:    s_endpgm
3583;
3584; GFX10-CU-LABEL: flat_system_one_as_unordered_load:
3585; GFX10-CU:       ; %bb.0: ; %entry
3586; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3587; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3588; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3589; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3590; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
3591; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
3592; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
3593; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3594; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3595; GFX10-CU-NEXT:    s_endpgm
3596;
3597; SKIP-CACHE-INV-LABEL: flat_system_one_as_unordered_load:
3598; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3599; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
3600; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3601; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3602; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
3603; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
3604; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
3605; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
3606; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3607; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
3608; SKIP-CACHE-INV-NEXT:    s_endpgm
3609;
3610; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_unordered_load:
3611; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3612; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3613; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3614; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3615; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
3616; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v0, v[0:1]
3617; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3618; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
3619; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3620; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[2:3], v0
3621; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3622;
3623; GFX90A-TGSPLIT-LABEL: flat_system_one_as_unordered_load:
3624; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3625; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3626; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3627; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3628; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
3629; GFX90A-TGSPLIT-NEXT:    flat_load_dword v0, v[0:1]
3630; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3631; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
3632; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3633; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[2:3], v0
3634; GFX90A-TGSPLIT-NEXT:    s_endpgm
3635    i32* %in, i32* %out) {
3636entry:
3637  %val = load atomic i32, i32* %in syncscope("one-as") unordered, align 4
3638  store i32 %val, i32* %out
3639  ret void
3640}
3641
3642define amdgpu_kernel void @flat_system_one_as_monotonic_load(
3643; GFX7-LABEL: flat_system_one_as_monotonic_load:
3644; GFX7:       ; %bb.0: ; %entry
3645; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3646; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3647; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3648; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3649; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
3650; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3651; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3652; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3653; GFX7-NEXT:    flat_store_dword v[2:3], v0
3654; GFX7-NEXT:    s_endpgm
3655;
3656; GFX10-WGP-LABEL: flat_system_one_as_monotonic_load:
3657; GFX10-WGP:       ; %bb.0: ; %entry
3658; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3659; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3660; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3661; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3662; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
3663; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
3664; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
3665; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3666; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3667; GFX10-WGP-NEXT:    s_endpgm
3668;
3669; GFX10-CU-LABEL: flat_system_one_as_monotonic_load:
3670; GFX10-CU:       ; %bb.0: ; %entry
3671; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3672; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3673; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3674; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3675; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
3676; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
3677; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
3678; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3679; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3680; GFX10-CU-NEXT:    s_endpgm
3681;
3682; SKIP-CACHE-INV-LABEL: flat_system_one_as_monotonic_load:
3683; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3684; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
3685; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3686; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3687; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
3688; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1] glc
3689; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
3690; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
3691; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3692; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
3693; SKIP-CACHE-INV-NEXT:    s_endpgm
3694;
3695; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_load:
3696; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3697; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3698; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3699; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3700; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
3701; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v0, v[0:1] glc scc
3702; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3703; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
3704; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3705; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[2:3], v0
3706; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3707;
3708; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_load:
3709; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3710; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3711; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3712; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3713; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
3714; GFX90A-TGSPLIT-NEXT:    flat_load_dword v0, v[0:1] glc scc
3715; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3716; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
3717; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3718; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[2:3], v0
3719; GFX90A-TGSPLIT-NEXT:    s_endpgm
3720    i32* %in, i32* %out) {
3721entry:
3722  %val = load atomic i32, i32* %in syncscope("one-as") monotonic, align 4
3723  store i32 %val, i32* %out
3724  ret void
3725}
3726
3727define amdgpu_kernel void @flat_system_one_as_acquire_load(
3728; GFX7-LABEL: flat_system_one_as_acquire_load:
3729; GFX7:       ; %bb.0: ; %entry
3730; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3731; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3732; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3733; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3734; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
3735; GFX7-NEXT:    s_waitcnt vmcnt(0)
3736; GFX7-NEXT:    buffer_wbinvl1_vol
3737; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3738; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3739; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3740; GFX7-NEXT:    flat_store_dword v[2:3], v0
3741; GFX7-NEXT:    s_endpgm
3742;
3743; GFX10-WGP-LABEL: flat_system_one_as_acquire_load:
3744; GFX10-WGP:       ; %bb.0: ; %entry
3745; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3746; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3747; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3748; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3749; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
3750; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3751; GFX10-WGP-NEXT:    buffer_gl0_inv
3752; GFX10-WGP-NEXT:    buffer_gl1_inv
3753; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
3754; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
3755; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3756; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3757; GFX10-WGP-NEXT:    s_endpgm
3758;
3759; GFX10-CU-LABEL: flat_system_one_as_acquire_load:
3760; GFX10-CU:       ; %bb.0: ; %entry
3761; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3762; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3763; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3764; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3765; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
3766; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3767; GFX10-CU-NEXT:    buffer_gl0_inv
3768; GFX10-CU-NEXT:    buffer_gl1_inv
3769; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
3770; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
3771; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3772; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3773; GFX10-CU-NEXT:    s_endpgm
3774;
3775; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_load:
3776; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3777; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
3778; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3779; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3780; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
3781; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1] glc
3782; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3783; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
3784; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
3785; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3786; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
3787; SKIP-CACHE-INV-NEXT:    s_endpgm
3788;
3789; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_load:
3790; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3791; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3792; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3793; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3794; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
3795; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v0, v[0:1] glc scc
3796; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3797; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
3798; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3799; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3800; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3801; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
3802; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3803; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[2:3], v0
3804; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3805;
3806; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_load:
3807; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3808; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3809; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3810; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3811; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
3812; GFX90A-TGSPLIT-NEXT:    flat_load_dword v0, v[0:1] glc scc
3813; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3814; GFX90A-TGSPLIT-NEXT:    buffer_invl2
3815; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3816; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3817; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3818; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
3819; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[2:3], v0
3820; GFX90A-TGSPLIT-NEXT:    s_endpgm
3821    i32* %in, i32* %out) {
3822entry:
3823  %val = load atomic i32, i32* %in syncscope("one-as") acquire, align 4
3824  store i32 %val, i32* %out
3825  ret void
3826}
3827
3828define amdgpu_kernel void @flat_system_one_as_seq_cst_load(
3829; GFX7-LABEL: flat_system_one_as_seq_cst_load:
3830; GFX7:       ; %bb.0: ; %entry
3831; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3832; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3833; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3834; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3835; GFX7-NEXT:    s_waitcnt vmcnt(0)
3836; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
3837; GFX7-NEXT:    s_waitcnt vmcnt(0)
3838; GFX7-NEXT:    buffer_wbinvl1_vol
3839; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3840; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3841; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3842; GFX7-NEXT:    flat_store_dword v[2:3], v0
3843; GFX7-NEXT:    s_endpgm
3844;
3845; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_load:
3846; GFX10-WGP:       ; %bb.0: ; %entry
3847; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3848; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3849; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3850; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3851; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3852; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3853; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
3854; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3855; GFX10-WGP-NEXT:    buffer_gl0_inv
3856; GFX10-WGP-NEXT:    buffer_gl1_inv
3857; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
3858; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
3859; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3860; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3861; GFX10-WGP-NEXT:    s_endpgm
3862;
3863; GFX10-CU-LABEL: flat_system_one_as_seq_cst_load:
3864; GFX10-CU:       ; %bb.0: ; %entry
3865; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3866; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3867; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3868; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3869; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3870; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3871; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
3872; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3873; GFX10-CU-NEXT:    buffer_gl0_inv
3874; GFX10-CU-NEXT:    buffer_gl1_inv
3875; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
3876; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
3877; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3878; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3879; GFX10-CU-NEXT:    s_endpgm
3880;
3881; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_load:
3882; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3883; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
3884; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3885; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3886; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
3887; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3888; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1] glc
3889; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3890; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
3891; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
3892; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3893; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
3894; SKIP-CACHE-INV-NEXT:    s_endpgm
3895;
3896; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_load:
3897; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3898; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3899; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3900; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3901; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
3902; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3903; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v0, v[0:1] glc scc
3904; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3905; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
3906; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3907; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3908; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3909; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
3910; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3911; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[2:3], v0
3912; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3913;
3914; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_load:
3915; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3916; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
3917; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3918; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
3919; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
3920; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3921; GFX90A-TGSPLIT-NEXT:    flat_load_dword v0, v[0:1] glc scc
3922; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3923; GFX90A-TGSPLIT-NEXT:    buffer_invl2
3924; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3925; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3926; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3927; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v3, s3
3928; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[2:3], v0
3929; GFX90A-TGSPLIT-NEXT:    s_endpgm
3930    i32* %in, i32* %out) {
3931entry:
3932  %val = load atomic i32, i32* %in syncscope("one-as") seq_cst, align 4
3933  store i32 %val, i32* %out
3934  ret void
3935}
3936
3937define amdgpu_kernel void @flat_system_one_as_unordered_store(
3938; GFX7-LABEL: flat_system_one_as_unordered_store:
3939; GFX7:       ; %bb.0: ; %entry
3940; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
3941; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
3942; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3943; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3944; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3945; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3946; GFX7-NEXT:    flat_store_dword v[0:1], v2
3947; GFX7-NEXT:    s_endpgm
3948;
3949; GFX10-WGP-LABEL: flat_system_one_as_unordered_store:
3950; GFX10-WGP:       ; %bb.0: ; %entry
3951; GFX10-WGP-NEXT:    s_clause 0x1
3952; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3953; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
3954; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3955; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3956; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3957; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3958; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3959; GFX10-WGP-NEXT:    s_endpgm
3960;
3961; GFX10-CU-LABEL: flat_system_one_as_unordered_store:
3962; GFX10-CU:       ; %bb.0: ; %entry
3963; GFX10-CU-NEXT:    s_clause 0x1
3964; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3965; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
3966; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3967; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3968; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3969; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3970; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3971; GFX10-CU-NEXT:    s_endpgm
3972;
3973; SKIP-CACHE-INV-LABEL: flat_system_one_as_unordered_store:
3974; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3975; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
3976; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
3977; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3978; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
3979; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3980; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
3981; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3982; SKIP-CACHE-INV-NEXT:    s_endpgm
3983;
3984; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_unordered_store:
3985; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3986; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
3987; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3988; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3989; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
3990; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3991; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3992; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3993;
3994; GFX90A-TGSPLIT-LABEL: flat_system_one_as_unordered_store:
3995; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3996; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
3997; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3998; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3999; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4000; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4001; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4002; GFX90A-TGSPLIT-NEXT:    s_endpgm
4003    i32 %in, i32* %out) {
4004entry:
4005  store atomic i32 %in, i32* %out syncscope("one-as") unordered, align 4
4006  ret void
4007}
4008
4009define amdgpu_kernel void @flat_system_one_as_monotonic_store(
4010; GFX7-LABEL: flat_system_one_as_monotonic_store:
4011; GFX7:       ; %bb.0: ; %entry
4012; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
4013; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
4014; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4015; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4016; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4017; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4018; GFX7-NEXT:    flat_store_dword v[0:1], v2
4019; GFX7-NEXT:    s_endpgm
4020;
4021; GFX10-WGP-LABEL: flat_system_one_as_monotonic_store:
4022; GFX10-WGP:       ; %bb.0: ; %entry
4023; GFX10-WGP-NEXT:    s_clause 0x1
4024; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4025; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
4026; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4027; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4028; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4029; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4030; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4031; GFX10-WGP-NEXT:    s_endpgm
4032;
4033; GFX10-CU-LABEL: flat_system_one_as_monotonic_store:
4034; GFX10-CU:       ; %bb.0: ; %entry
4035; GFX10-CU-NEXT:    s_clause 0x1
4036; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4037; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
4038; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4039; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4040; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4041; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4042; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4043; GFX10-CU-NEXT:    s_endpgm
4044;
4045; SKIP-CACHE-INV-LABEL: flat_system_one_as_monotonic_store:
4046; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4047; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
4048; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
4049; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4050; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
4051; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4052; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
4053; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4054; SKIP-CACHE-INV-NEXT:    s_endpgm
4055;
4056; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_store:
4057; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4058; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
4059; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4060; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4061; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4062; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4063; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 scc
4064; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4065;
4066; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_store:
4067; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4068; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
4069; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4070; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4071; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4072; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4073; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 scc
4074; GFX90A-TGSPLIT-NEXT:    s_endpgm
4075    i32 %in, i32* %out) {
4076entry:
4077  store atomic i32 %in, i32* %out syncscope("one-as") monotonic, align 4
4078  ret void
4079}
4080
4081define amdgpu_kernel void @flat_system_one_as_release_store(
4082; GFX7-LABEL: flat_system_one_as_release_store:
4083; GFX7:       ; %bb.0: ; %entry
4084; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
4085; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
4086; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4087; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4088; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4089; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4090; GFX7-NEXT:    s_waitcnt vmcnt(0)
4091; GFX7-NEXT:    flat_store_dword v[0:1], v2
4092; GFX7-NEXT:    s_endpgm
4093;
4094; GFX10-WGP-LABEL: flat_system_one_as_release_store:
4095; GFX10-WGP:       ; %bb.0: ; %entry
4096; GFX10-WGP-NEXT:    s_clause 0x1
4097; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4098; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
4099; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4100; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4101; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4102; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4103; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4104; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4105; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4106; GFX10-WGP-NEXT:    s_endpgm
4107;
4108; GFX10-CU-LABEL: flat_system_one_as_release_store:
4109; GFX10-CU:       ; %bb.0: ; %entry
4110; GFX10-CU-NEXT:    s_clause 0x1
4111; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4112; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
4113; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4114; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4115; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4116; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4117; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4118; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4119; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4120; GFX10-CU-NEXT:    s_endpgm
4121;
4122; SKIP-CACHE-INV-LABEL: flat_system_one_as_release_store:
4123; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4124; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
4125; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
4126; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4127; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
4128; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4129; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
4130; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4131; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4132; SKIP-CACHE-INV-NEXT:    s_endpgm
4133;
4134; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_release_store:
4135; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4136; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
4137; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4138; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4139; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4140; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4141; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
4142; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4143; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 scc
4144; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4145;
4146; GFX90A-TGSPLIT-LABEL: flat_system_one_as_release_store:
4147; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4148; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
4149; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4150; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4151; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4152; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4153; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
4154; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4155; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 scc
4156; GFX90A-TGSPLIT-NEXT:    s_endpgm
4157    i32 %in, i32* %out) {
4158entry:
4159  store atomic i32 %in, i32* %out syncscope("one-as") release, align 4
4160  ret void
4161}
4162
4163define amdgpu_kernel void @flat_system_one_as_seq_cst_store(
4164; GFX7-LABEL: flat_system_one_as_seq_cst_store:
4165; GFX7:       ; %bb.0: ; %entry
4166; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
4167; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
4168; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4169; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4170; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4171; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4172; GFX7-NEXT:    s_waitcnt vmcnt(0)
4173; GFX7-NEXT:    flat_store_dword v[0:1], v2
4174; GFX7-NEXT:    s_endpgm
4175;
4176; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_store:
4177; GFX10-WGP:       ; %bb.0: ; %entry
4178; GFX10-WGP-NEXT:    s_clause 0x1
4179; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4180; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
4181; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4182; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4183; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4184; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4185; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4186; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4187; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4188; GFX10-WGP-NEXT:    s_endpgm
4189;
4190; GFX10-CU-LABEL: flat_system_one_as_seq_cst_store:
4191; GFX10-CU:       ; %bb.0: ; %entry
4192; GFX10-CU-NEXT:    s_clause 0x1
4193; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4194; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
4195; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4196; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4197; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4198; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4199; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4200; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4201; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4202; GFX10-CU-NEXT:    s_endpgm
4203;
4204; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_store:
4205; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4206; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
4207; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
4208; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4209; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
4210; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4211; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
4212; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4213; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4214; SKIP-CACHE-INV-NEXT:    s_endpgm
4215;
4216; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_store:
4217; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4218; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
4219; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4220; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4221; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4222; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4223; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
4224; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4225; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 scc
4226; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4227;
4228; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_store:
4229; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4230; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
4231; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4232; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4233; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4234; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4235; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
4236; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4237; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 scc
4238; GFX90A-TGSPLIT-NEXT:    s_endpgm
4239    i32 %in, i32* %out) {
4240entry:
4241  store atomic i32 %in, i32* %out syncscope("one-as") seq_cst, align 4
4242  ret void
4243}
4244
4245define amdgpu_kernel void @flat_system_one_as_monotonic_atomicrmw(
4246; GFX7-LABEL: flat_system_one_as_monotonic_atomicrmw:
4247; GFX7:       ; %bb.0: ; %entry
4248; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4249; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
4250; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4251; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4252; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4253; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4254; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
4255; GFX7-NEXT:    s_endpgm
4256;
4257; GFX10-WGP-LABEL: flat_system_one_as_monotonic_atomicrmw:
4258; GFX10-WGP:       ; %bb.0: ; %entry
4259; GFX10-WGP-NEXT:    s_clause 0x1
4260; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4261; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
4262; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4263; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4264; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4265; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4266; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
4267; GFX10-WGP-NEXT:    s_endpgm
4268;
4269; GFX10-CU-LABEL: flat_system_one_as_monotonic_atomicrmw:
4270; GFX10-CU:       ; %bb.0: ; %entry
4271; GFX10-CU-NEXT:    s_clause 0x1
4272; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4273; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
4274; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4275; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4276; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4277; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4278; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
4279; GFX10-CU-NEXT:    s_endpgm
4280;
4281; SKIP-CACHE-INV-LABEL: flat_system_one_as_monotonic_atomicrmw:
4282; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4283; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
4284; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
4285; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4286; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4287; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4288; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4289; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
4290; SKIP-CACHE-INV-NEXT:    s_endpgm
4291;
4292; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_atomicrmw:
4293; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4294; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4295; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4296; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4297; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4298; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4299; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 scc
4300; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4301;
4302; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_atomicrmw:
4303; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4304; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4305; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4306; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4307; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4308; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4309; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 scc
4310; GFX90A-TGSPLIT-NEXT:    s_endpgm
4311    i32* %out, i32 %in) {
4312entry:
4313  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") monotonic
4314  ret void
4315}
4316
4317define amdgpu_kernel void @flat_system_one_as_acquire_atomicrmw(
4318; GFX7-LABEL: flat_system_one_as_acquire_atomicrmw:
4319; GFX7:       ; %bb.0: ; %entry
4320; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4321; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
4322; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4323; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4324; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4325; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4326; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
4327; GFX7-NEXT:    s_waitcnt vmcnt(0)
4328; GFX7-NEXT:    buffer_wbinvl1_vol
4329; GFX7-NEXT:    s_endpgm
4330;
4331; GFX10-WGP-LABEL: flat_system_one_as_acquire_atomicrmw:
4332; GFX10-WGP:       ; %bb.0: ; %entry
4333; GFX10-WGP-NEXT:    s_clause 0x1
4334; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4335; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
4336; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4337; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4338; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4339; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4340; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
4341; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4342; GFX10-WGP-NEXT:    buffer_gl0_inv
4343; GFX10-WGP-NEXT:    buffer_gl1_inv
4344; GFX10-WGP-NEXT:    s_endpgm
4345;
4346; GFX10-CU-LABEL: flat_system_one_as_acquire_atomicrmw:
4347; GFX10-CU:       ; %bb.0: ; %entry
4348; GFX10-CU-NEXT:    s_clause 0x1
4349; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4350; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
4351; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4352; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4353; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4354; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4355; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
4356; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4357; GFX10-CU-NEXT:    buffer_gl0_inv
4358; GFX10-CU-NEXT:    buffer_gl1_inv
4359; GFX10-CU-NEXT:    s_endpgm
4360;
4361; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_atomicrmw:
4362; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4363; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
4364; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
4365; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4366; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4367; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4368; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4369; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
4370; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4371; SKIP-CACHE-INV-NEXT:    s_endpgm
4372;
4373; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_atomicrmw:
4374; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4375; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4376; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4377; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4378; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4379; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4380; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 scc
4381; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4382; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
4383; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4384; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4385; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4386;
4387; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_atomicrmw:
4388; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4389; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4390; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4391; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4392; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4393; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4394; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 scc
4395; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4396; GFX90A-TGSPLIT-NEXT:    buffer_invl2
4397; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4398; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4399; GFX90A-TGSPLIT-NEXT:    s_endpgm
4400    i32* %out, i32 %in) {
4401entry:
4402  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") acquire
4403  ret void
4404}
4405
4406define amdgpu_kernel void @flat_system_one_as_release_atomicrmw(
4407; GFX7-LABEL: flat_system_one_as_release_atomicrmw:
4408; GFX7:       ; %bb.0: ; %entry
4409; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4410; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
4411; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4412; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4413; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4414; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4415; GFX7-NEXT:    s_waitcnt vmcnt(0)
4416; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
4417; GFX7-NEXT:    s_endpgm
4418;
4419; GFX10-WGP-LABEL: flat_system_one_as_release_atomicrmw:
4420; GFX10-WGP:       ; %bb.0: ; %entry
4421; GFX10-WGP-NEXT:    s_clause 0x1
4422; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4423; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
4424; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4425; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4426; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4427; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4428; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4429; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4430; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
4431; GFX10-WGP-NEXT:    s_endpgm
4432;
4433; GFX10-CU-LABEL: flat_system_one_as_release_atomicrmw:
4434; GFX10-CU:       ; %bb.0: ; %entry
4435; GFX10-CU-NEXT:    s_clause 0x1
4436; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4437; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
4438; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4439; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4440; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4441; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4442; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4443; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4444; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
4445; GFX10-CU-NEXT:    s_endpgm
4446;
4447; SKIP-CACHE-INV-LABEL: flat_system_one_as_release_atomicrmw:
4448; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4449; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
4450; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
4451; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4452; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4453; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4454; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4455; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4456; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
4457; SKIP-CACHE-INV-NEXT:    s_endpgm
4458;
4459; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_release_atomicrmw:
4460; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4461; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4462; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4463; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4464; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4465; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4466; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
4467; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4468; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 scc
4469; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4470;
4471; GFX90A-TGSPLIT-LABEL: flat_system_one_as_release_atomicrmw:
4472; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4473; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4474; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4475; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4476; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4477; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4478; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
4479; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4480; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 scc
4481; GFX90A-TGSPLIT-NEXT:    s_endpgm
4482    i32* %out, i32 %in) {
4483entry:
4484  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") release
4485  ret void
4486}
4487
4488define amdgpu_kernel void @flat_system_one_as_acq_rel_atomicrmw(
4489; GFX7-LABEL: flat_system_one_as_acq_rel_atomicrmw:
4490; GFX7:       ; %bb.0: ; %entry
4491; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4492; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
4493; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4494; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4495; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4496; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4497; GFX7-NEXT:    s_waitcnt vmcnt(0)
4498; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
4499; GFX7-NEXT:    s_waitcnt vmcnt(0)
4500; GFX7-NEXT:    buffer_wbinvl1_vol
4501; GFX7-NEXT:    s_endpgm
4502;
4503; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_atomicrmw:
4504; GFX10-WGP:       ; %bb.0: ; %entry
4505; GFX10-WGP-NEXT:    s_clause 0x1
4506; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4507; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
4508; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4509; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4510; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4511; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4512; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4513; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4514; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
4515; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4516; GFX10-WGP-NEXT:    buffer_gl0_inv
4517; GFX10-WGP-NEXT:    buffer_gl1_inv
4518; GFX10-WGP-NEXT:    s_endpgm
4519;
4520; GFX10-CU-LABEL: flat_system_one_as_acq_rel_atomicrmw:
4521; GFX10-CU:       ; %bb.0: ; %entry
4522; GFX10-CU-NEXT:    s_clause 0x1
4523; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4524; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
4525; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4526; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4527; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4528; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4529; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4530; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4531; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
4532; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4533; GFX10-CU-NEXT:    buffer_gl0_inv
4534; GFX10-CU-NEXT:    buffer_gl1_inv
4535; GFX10-CU-NEXT:    s_endpgm
4536;
4537; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_atomicrmw:
4538; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4539; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
4540; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
4541; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4542; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4543; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4544; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4545; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4546; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
4547; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4548; SKIP-CACHE-INV-NEXT:    s_endpgm
4549;
4550; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_atomicrmw:
4551; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4552; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4553; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4554; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4555; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4556; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4557; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
4558; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4559; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 scc
4560; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4561; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
4562; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4563; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4564; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4565;
4566; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_atomicrmw:
4567; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4568; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4569; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4570; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4571; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4572; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4573; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
4574; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4575; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 scc
4576; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4577; GFX90A-TGSPLIT-NEXT:    buffer_invl2
4578; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4579; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4580; GFX90A-TGSPLIT-NEXT:    s_endpgm
4581    i32* %out, i32 %in) {
4582entry:
4583  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") acq_rel
4584  ret void
4585}
4586
4587define amdgpu_kernel void @flat_system_one_as_seq_cst_atomicrmw(
4588; GFX7-LABEL: flat_system_one_as_seq_cst_atomicrmw:
4589; GFX7:       ; %bb.0: ; %entry
4590; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4591; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
4592; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4593; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4594; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4595; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4596; GFX7-NEXT:    s_waitcnt vmcnt(0)
4597; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
4598; GFX7-NEXT:    s_waitcnt vmcnt(0)
4599; GFX7-NEXT:    buffer_wbinvl1_vol
4600; GFX7-NEXT:    s_endpgm
4601;
4602; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_atomicrmw:
4603; GFX10-WGP:       ; %bb.0: ; %entry
4604; GFX10-WGP-NEXT:    s_clause 0x1
4605; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4606; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
4607; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4608; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4609; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4610; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4611; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4612; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4613; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
4614; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4615; GFX10-WGP-NEXT:    buffer_gl0_inv
4616; GFX10-WGP-NEXT:    buffer_gl1_inv
4617; GFX10-WGP-NEXT:    s_endpgm
4618;
4619; GFX10-CU-LABEL: flat_system_one_as_seq_cst_atomicrmw:
4620; GFX10-CU:       ; %bb.0: ; %entry
4621; GFX10-CU-NEXT:    s_clause 0x1
4622; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4623; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
4624; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4625; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4626; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4627; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4628; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4629; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4630; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
4631; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4632; GFX10-CU-NEXT:    buffer_gl0_inv
4633; GFX10-CU-NEXT:    buffer_gl1_inv
4634; GFX10-CU-NEXT:    s_endpgm
4635;
4636; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_atomicrmw:
4637; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4638; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
4639; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
4640; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4641; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4642; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4643; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4644; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4645; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
4646; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4647; SKIP-CACHE-INV-NEXT:    s_endpgm
4648;
4649; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_atomicrmw:
4650; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4651; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4652; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4653; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4654; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4655; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4656; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
4657; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4658; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 scc
4659; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4660; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
4661; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4662; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4663; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4664;
4665; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_atomicrmw:
4666; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4667; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4668; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4669; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4670; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4671; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4672; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
4673; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4674; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 scc
4675; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4676; GFX90A-TGSPLIT-NEXT:    buffer_invl2
4677; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4678; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4679; GFX90A-TGSPLIT-NEXT:    s_endpgm
4680    i32* %out, i32 %in) {
4681entry:
4682  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") seq_cst
4683  ret void
4684}
4685
4686define amdgpu_kernel void @flat_system_one_as_acquire_ret_atomicrmw(
4687; GFX7-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
4688; GFX7:       ; %bb.0: ; %entry
4689; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4690; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
4691; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4692; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4693; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4694; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4695; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4696; GFX7-NEXT:    s_waitcnt vmcnt(0)
4697; GFX7-NEXT:    buffer_wbinvl1_vol
4698; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4699; GFX7-NEXT:    flat_store_dword v[0:1], v2
4700; GFX7-NEXT:    s_endpgm
4701;
4702; GFX10-WGP-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
4703; GFX10-WGP:       ; %bb.0: ; %entry
4704; GFX10-WGP-NEXT:    s_clause 0x1
4705; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4706; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
4707; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4708; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4709; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4710; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4711; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4712; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4713; GFX10-WGP-NEXT:    buffer_gl0_inv
4714; GFX10-WGP-NEXT:    buffer_gl1_inv
4715; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4716; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4717; GFX10-WGP-NEXT:    s_endpgm
4718;
4719; GFX10-CU-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
4720; GFX10-CU:       ; %bb.0: ; %entry
4721; GFX10-CU-NEXT:    s_clause 0x1
4722; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4723; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
4724; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4725; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4726; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4727; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4728; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4729; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4730; GFX10-CU-NEXT:    buffer_gl0_inv
4731; GFX10-CU-NEXT:    buffer_gl1_inv
4732; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4733; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4734; GFX10-CU-NEXT:    s_endpgm
4735;
4736; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
4737; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4738; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
4739; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
4740; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4741; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4742; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4743; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4744; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4745; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4746; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4747; SKIP-CACHE-INV-NEXT:    s_endpgm
4748;
4749; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
4750; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4751; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4752; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4753; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4754; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4755; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4756; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc scc
4757; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4758; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
4759; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4760; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4761; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4762; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4763; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4764;
4765; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
4766; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4767; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4768; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4769; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4770; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4771; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4772; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc scc
4773; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4774; GFX90A-TGSPLIT-NEXT:    buffer_invl2
4775; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4776; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4777; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4778; GFX90A-TGSPLIT-NEXT:    s_endpgm
4779    i32* %out, i32 %in) {
4780entry:
4781  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") acquire
4782  store i32 %val, i32* %out, align 4
4783  ret void
4784}
4785
4786define amdgpu_kernel void @flat_system_one_as_acq_rel_ret_atomicrmw(
4787; GFX7-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
4788; GFX7:       ; %bb.0: ; %entry
4789; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4790; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
4791; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4792; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4793; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4794; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4795; GFX7-NEXT:    s_waitcnt vmcnt(0)
4796; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4797; GFX7-NEXT:    s_waitcnt vmcnt(0)
4798; GFX7-NEXT:    buffer_wbinvl1_vol
4799; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4800; GFX7-NEXT:    flat_store_dword v[0:1], v2
4801; GFX7-NEXT:    s_endpgm
4802;
4803; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
4804; GFX10-WGP:       ; %bb.0: ; %entry
4805; GFX10-WGP-NEXT:    s_clause 0x1
4806; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4807; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
4808; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4809; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4810; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4811; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4812; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4813; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4814; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4815; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4816; GFX10-WGP-NEXT:    buffer_gl0_inv
4817; GFX10-WGP-NEXT:    buffer_gl1_inv
4818; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4819; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4820; GFX10-WGP-NEXT:    s_endpgm
4821;
4822; GFX10-CU-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
4823; GFX10-CU:       ; %bb.0: ; %entry
4824; GFX10-CU-NEXT:    s_clause 0x1
4825; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4826; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
4827; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4828; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4829; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4830; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4831; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4832; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4833; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4834; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4835; GFX10-CU-NEXT:    buffer_gl0_inv
4836; GFX10-CU-NEXT:    buffer_gl1_inv
4837; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4838; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4839; GFX10-CU-NEXT:    s_endpgm
4840;
4841; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
4842; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4843; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
4844; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
4845; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4846; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4847; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4848; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4849; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4850; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4851; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4852; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4853; SKIP-CACHE-INV-NEXT:    s_endpgm
4854;
4855; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
4856; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4857; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4858; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4859; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4860; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4861; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4862; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
4863; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4864; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc scc
4865; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4866; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
4867; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4868; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4869; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4870; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4871; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4872;
4873; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
4874; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4875; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4876; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4877; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4878; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4879; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4880; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
4881; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4882; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc scc
4883; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4884; GFX90A-TGSPLIT-NEXT:    buffer_invl2
4885; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4886; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4887; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4888; GFX90A-TGSPLIT-NEXT:    s_endpgm
4889    i32* %out, i32 %in) {
4890entry:
4891  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") acq_rel
4892  store i32 %val, i32* %out, align 4
4893  ret void
4894}
4895
4896define amdgpu_kernel void @flat_system_one_as_seq_cst_ret_atomicrmw(
4897; GFX7-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
4898; GFX7:       ; %bb.0: ; %entry
4899; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4900; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
4901; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4902; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4903; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4904; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4905; GFX7-NEXT:    s_waitcnt vmcnt(0)
4906; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4907; GFX7-NEXT:    s_waitcnt vmcnt(0)
4908; GFX7-NEXT:    buffer_wbinvl1_vol
4909; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4910; GFX7-NEXT:    flat_store_dword v[0:1], v2
4911; GFX7-NEXT:    s_endpgm
4912;
4913; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
4914; GFX10-WGP:       ; %bb.0: ; %entry
4915; GFX10-WGP-NEXT:    s_clause 0x1
4916; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4917; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
4918; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4919; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4920; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4921; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4922; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4923; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4924; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4925; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4926; GFX10-WGP-NEXT:    buffer_gl0_inv
4927; GFX10-WGP-NEXT:    buffer_gl1_inv
4928; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4929; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4930; GFX10-WGP-NEXT:    s_endpgm
4931;
4932; GFX10-CU-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
4933; GFX10-CU:       ; %bb.0: ; %entry
4934; GFX10-CU-NEXT:    s_clause 0x1
4935; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4936; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
4937; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4938; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4939; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4940; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4941; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4942; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4943; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4944; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4945; GFX10-CU-NEXT:    buffer_gl0_inv
4946; GFX10-CU-NEXT:    buffer_gl1_inv
4947; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4948; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4949; GFX10-CU-NEXT:    s_endpgm
4950;
4951; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
4952; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4953; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
4954; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
4955; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4956; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4957; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4958; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4959; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4960; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
4961; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4962; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4963; SKIP-CACHE-INV-NEXT:    s_endpgm
4964;
4965; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
4966; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4967; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4968; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4969; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4970; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4971; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4972; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
4973; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4974; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc scc
4975; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4976; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
4977; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4978; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4979; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4980; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4981; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4982;
4983; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
4984; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4985; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4986; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
4987; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4988; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4989; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
4990; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
4991; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4992; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc scc
4993; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4994; GFX90A-TGSPLIT-NEXT:    buffer_invl2
4995; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4996; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4997; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4998; GFX90A-TGSPLIT-NEXT:    s_endpgm
4999    i32* %out, i32 %in) {
5000entry:
5001  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") seq_cst
5002  store i32 %val, i32* %out, align 4
5003  ret void
5004}
5005
5006define amdgpu_kernel void @flat_system_one_as_monotonic_monotonic_cmpxchg(
5007; GFX7-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
5008; GFX7:       ; %bb.0: ; %entry
5009; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5010; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5011; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5012; GFX7-NEXT:    s_add_u32 s0, s0, 16
5013; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5014; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5015; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5016; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5017; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5018; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5019; GFX7-NEXT:    s_endpgm
5020;
5021; GFX10-WGP-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
5022; GFX10-WGP:       ; %bb.0: ; %entry
5023; GFX10-WGP-NEXT:    s_clause 0x1
5024; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5025; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5026; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5027; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
5028; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
5029; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5030; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5031; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5032; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5033; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5034; GFX10-WGP-NEXT:    s_endpgm
5035;
5036; GFX10-CU-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
5037; GFX10-CU:       ; %bb.0: ; %entry
5038; GFX10-CU-NEXT:    s_clause 0x1
5039; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5040; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5041; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5042; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
5043; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
5044; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5045; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5046; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5047; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5048; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5049; GFX10-CU-NEXT:    s_endpgm
5050;
5051; SKIP-CACHE-INV-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
5052; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5053; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
5054; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5055; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5056; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
5057; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
5058; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5059; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5060; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5061; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5062; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5063; SKIP-CACHE-INV-NEXT:    s_endpgm
5064;
5065; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
5066; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5067; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5068; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5069; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5070; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5071; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5072; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
5073; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5074;
5075; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
5076; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5077; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5078; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5079; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5080; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5081; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5082; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
5083; GFX90A-TGSPLIT-NEXT:    s_endpgm
5084    i32* %out, i32 %in, i32 %old) {
5085entry:
5086  %gep = getelementptr i32, i32* %out, i32 4
5087  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") monotonic monotonic
5088  ret void
5089}
5090
5091define amdgpu_kernel void @flat_system_one_as_acquire_monotonic_cmpxchg(
5092; GFX7-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
5093; GFX7:       ; %bb.0: ; %entry
5094; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5095; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5096; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5097; GFX7-NEXT:    s_add_u32 s0, s0, 16
5098; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5099; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5100; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5101; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5102; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5103; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5104; GFX7-NEXT:    s_waitcnt vmcnt(0)
5105; GFX7-NEXT:    buffer_wbinvl1_vol
5106; GFX7-NEXT:    s_endpgm
5107;
5108; GFX10-WGP-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
5109; GFX10-WGP:       ; %bb.0: ; %entry
5110; GFX10-WGP-NEXT:    s_clause 0x1
5111; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5112; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5113; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5114; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
5115; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
5116; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5117; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5118; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5119; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5120; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5121; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5122; GFX10-WGP-NEXT:    buffer_gl0_inv
5123; GFX10-WGP-NEXT:    buffer_gl1_inv
5124; GFX10-WGP-NEXT:    s_endpgm
5125;
5126; GFX10-CU-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
5127; GFX10-CU:       ; %bb.0: ; %entry
5128; GFX10-CU-NEXT:    s_clause 0x1
5129; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5130; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5131; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5132; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
5133; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
5134; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5135; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5136; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5137; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5138; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5139; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5140; GFX10-CU-NEXT:    buffer_gl0_inv
5141; GFX10-CU-NEXT:    buffer_gl1_inv
5142; GFX10-CU-NEXT:    s_endpgm
5143;
5144; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
5145; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5146; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
5147; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5148; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5149; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
5150; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
5151; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5152; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5153; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5154; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5155; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5156; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5157; SKIP-CACHE-INV-NEXT:    s_endpgm
5158;
5159; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
5160; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5161; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5162; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5163; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5164; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5165; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5166; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
5167; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5168; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
5169; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5170; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5171; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5172;
5173; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
5174; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5175; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5176; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5177; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5178; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5179; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5180; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
5181; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5182; GFX90A-TGSPLIT-NEXT:    buffer_invl2
5183; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5184; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5185; GFX90A-TGSPLIT-NEXT:    s_endpgm
5186    i32* %out, i32 %in, i32 %old) {
5187entry:
5188  %gep = getelementptr i32, i32* %out, i32 4
5189  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acquire monotonic
5190  ret void
5191}
5192
5193define amdgpu_kernel void @flat_system_one_as_release_monotonic_cmpxchg(
5194; GFX7-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
5195; GFX7:       ; %bb.0: ; %entry
5196; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5197; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5198; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5199; GFX7-NEXT:    s_add_u32 s0, s0, 16
5200; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5201; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5202; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5203; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5204; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5205; GFX7-NEXT:    s_waitcnt vmcnt(0)
5206; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5207; GFX7-NEXT:    s_endpgm
5208;
5209; GFX10-WGP-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
5210; GFX10-WGP:       ; %bb.0: ; %entry
5211; GFX10-WGP-NEXT:    s_clause 0x1
5212; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5213; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5214; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5215; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
5216; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
5217; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5218; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5219; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5220; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5221; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5222; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5223; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5224; GFX10-WGP-NEXT:    s_endpgm
5225;
5226; GFX10-CU-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
5227; GFX10-CU:       ; %bb.0: ; %entry
5228; GFX10-CU-NEXT:    s_clause 0x1
5229; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5230; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5231; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5232; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
5233; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
5234; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5235; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5236; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5237; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5238; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5239; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5240; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5241; GFX10-CU-NEXT:    s_endpgm
5242;
5243; SKIP-CACHE-INV-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
5244; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5245; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
5246; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5247; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5248; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
5249; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
5250; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5251; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5252; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5253; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5254; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5255; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5256; SKIP-CACHE-INV-NEXT:    s_endpgm
5257;
5258; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
5259; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5260; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5261; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5262; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5263; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5264; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5265; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
5266; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5267; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
5268; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5269;
5270; GFX90A-TGSPLIT-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
5271; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5272; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5273; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5274; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5275; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5276; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5277; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
5278; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5279; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
5280; GFX90A-TGSPLIT-NEXT:    s_endpgm
5281    i32* %out, i32 %in, i32 %old) {
5282entry:
5283  %gep = getelementptr i32, i32* %out, i32 4
5284  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") release monotonic
5285  ret void
5286}
5287
5288define amdgpu_kernel void @flat_system_one_as_acq_rel_monotonic_cmpxchg(
5289; GFX7-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
5290; GFX7:       ; %bb.0: ; %entry
5291; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5292; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5293; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5294; GFX7-NEXT:    s_add_u32 s0, s0, 16
5295; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5296; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5297; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5298; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5299; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5300; GFX7-NEXT:    s_waitcnt vmcnt(0)
5301; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5302; GFX7-NEXT:    s_waitcnt vmcnt(0)
5303; GFX7-NEXT:    buffer_wbinvl1_vol
5304; GFX7-NEXT:    s_endpgm
5305;
5306; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
5307; GFX10-WGP:       ; %bb.0: ; %entry
5308; GFX10-WGP-NEXT:    s_clause 0x1
5309; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5310; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5311; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5312; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
5313; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
5314; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5315; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5316; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5317; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5318; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5319; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5320; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5321; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5322; GFX10-WGP-NEXT:    buffer_gl0_inv
5323; GFX10-WGP-NEXT:    buffer_gl1_inv
5324; GFX10-WGP-NEXT:    s_endpgm
5325;
5326; GFX10-CU-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
5327; GFX10-CU:       ; %bb.0: ; %entry
5328; GFX10-CU-NEXT:    s_clause 0x1
5329; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5330; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5331; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5332; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
5333; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
5334; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5335; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5336; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5337; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5338; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5339; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5340; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5341; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5342; GFX10-CU-NEXT:    buffer_gl0_inv
5343; GFX10-CU-NEXT:    buffer_gl1_inv
5344; GFX10-CU-NEXT:    s_endpgm
5345;
5346; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
5347; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5348; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
5349; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5350; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5351; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
5352; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
5353; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5354; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5355; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5356; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5357; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5358; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5359; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5360; SKIP-CACHE-INV-NEXT:    s_endpgm
5361;
5362; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
5363; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5364; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5365; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5366; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5367; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5368; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5369; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
5370; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5371; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
5372; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5373; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
5374; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5375; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5376; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5377;
5378; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
5379; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5380; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5381; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5382; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5383; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5384; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5385; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
5386; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5387; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
5388; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5389; GFX90A-TGSPLIT-NEXT:    buffer_invl2
5390; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5391; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5392; GFX90A-TGSPLIT-NEXT:    s_endpgm
5393    i32* %out, i32 %in, i32 %old) {
5394entry:
5395  %gep = getelementptr i32, i32* %out, i32 4
5396  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel monotonic
5397  ret void
5398}
5399
5400define amdgpu_kernel void @flat_system_one_as_seq_cst_monotonic_cmpxchg(
5401; GFX7-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
5402; GFX7:       ; %bb.0: ; %entry
5403; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5404; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5405; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5406; GFX7-NEXT:    s_add_u32 s0, s0, 16
5407; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5408; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5409; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5410; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5411; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5412; GFX7-NEXT:    s_waitcnt vmcnt(0)
5413; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5414; GFX7-NEXT:    s_waitcnt vmcnt(0)
5415; GFX7-NEXT:    buffer_wbinvl1_vol
5416; GFX7-NEXT:    s_endpgm
5417;
5418; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
5419; GFX10-WGP:       ; %bb.0: ; %entry
5420; GFX10-WGP-NEXT:    s_clause 0x1
5421; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5422; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5423; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5424; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
5425; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
5426; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5427; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5428; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5429; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5430; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5431; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5432; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5433; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5434; GFX10-WGP-NEXT:    buffer_gl0_inv
5435; GFX10-WGP-NEXT:    buffer_gl1_inv
5436; GFX10-WGP-NEXT:    s_endpgm
5437;
5438; GFX10-CU-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
5439; GFX10-CU:       ; %bb.0: ; %entry
5440; GFX10-CU-NEXT:    s_clause 0x1
5441; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5442; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5443; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5444; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
5445; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
5446; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5447; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5448; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5449; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5450; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5451; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5452; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5453; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5454; GFX10-CU-NEXT:    buffer_gl0_inv
5455; GFX10-CU-NEXT:    buffer_gl1_inv
5456; GFX10-CU-NEXT:    s_endpgm
5457;
5458; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
5459; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5460; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
5461; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5462; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5463; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
5464; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
5465; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5466; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5467; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5468; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5469; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5470; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5471; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5472; SKIP-CACHE-INV-NEXT:    s_endpgm
5473;
5474; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
5475; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5476; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5477; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5478; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5479; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5480; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5481; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
5482; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5483; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
5484; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5485; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
5486; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5487; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5488; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5489;
5490; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
5491; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5492; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5493; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5494; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5495; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5496; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5497; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
5498; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5499; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
5500; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5501; GFX90A-TGSPLIT-NEXT:    buffer_invl2
5502; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5503; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5504; GFX90A-TGSPLIT-NEXT:    s_endpgm
5505    i32* %out, i32 %in, i32 %old) {
5506entry:
5507  %gep = getelementptr i32, i32* %out, i32 4
5508  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst monotonic
5509  ret void
5510}
5511
5512define amdgpu_kernel void @flat_system_one_as_acquire_acquire_cmpxchg(
5513; GFX7-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
5514; GFX7:       ; %bb.0: ; %entry
5515; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5516; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5517; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5518; GFX7-NEXT:    s_add_u32 s0, s0, 16
5519; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5520; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5521; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5522; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5523; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5524; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5525; GFX7-NEXT:    s_waitcnt vmcnt(0)
5526; GFX7-NEXT:    buffer_wbinvl1_vol
5527; GFX7-NEXT:    s_endpgm
5528;
5529; GFX10-WGP-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
5530; GFX10-WGP:       ; %bb.0: ; %entry
5531; GFX10-WGP-NEXT:    s_clause 0x1
5532; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5533; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5534; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5535; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
5536; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
5537; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5538; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5539; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5540; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5541; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5542; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5543; GFX10-WGP-NEXT:    buffer_gl0_inv
5544; GFX10-WGP-NEXT:    buffer_gl1_inv
5545; GFX10-WGP-NEXT:    s_endpgm
5546;
5547; GFX10-CU-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
5548; GFX10-CU:       ; %bb.0: ; %entry
5549; GFX10-CU-NEXT:    s_clause 0x1
5550; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5551; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5552; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5553; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
5554; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
5555; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5556; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5557; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5558; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5559; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5560; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5561; GFX10-CU-NEXT:    buffer_gl0_inv
5562; GFX10-CU-NEXT:    buffer_gl1_inv
5563; GFX10-CU-NEXT:    s_endpgm
5564;
5565; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
5566; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5567; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
5568; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5569; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5570; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
5571; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
5572; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5573; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5574; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5575; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5576; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5577; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5578; SKIP-CACHE-INV-NEXT:    s_endpgm
5579;
5580; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
5581; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5582; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5583; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5584; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5585; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5586; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5587; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
5588; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5589; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
5590; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5591; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5592; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5593;
5594; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
5595; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5596; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5597; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5598; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5599; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5600; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5601; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
5602; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5603; GFX90A-TGSPLIT-NEXT:    buffer_invl2
5604; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5605; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5606; GFX90A-TGSPLIT-NEXT:    s_endpgm
5607    i32* %out, i32 %in, i32 %old) {
5608entry:
5609  %gep = getelementptr i32, i32* %out, i32 4
5610  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acquire acquire
5611  ret void
5612}
5613
5614define amdgpu_kernel void @flat_system_one_as_release_acquire_cmpxchg(
5615; GFX7-LABEL: flat_system_one_as_release_acquire_cmpxchg:
5616; GFX7:       ; %bb.0: ; %entry
5617; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5618; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5619; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5620; GFX7-NEXT:    s_add_u32 s0, s0, 16
5621; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5622; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5623; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5624; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5625; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5626; GFX7-NEXT:    s_waitcnt vmcnt(0)
5627; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5628; GFX7-NEXT:    s_waitcnt vmcnt(0)
5629; GFX7-NEXT:    buffer_wbinvl1_vol
5630; GFX7-NEXT:    s_endpgm
5631;
5632; GFX10-WGP-LABEL: flat_system_one_as_release_acquire_cmpxchg:
5633; GFX10-WGP:       ; %bb.0: ; %entry
5634; GFX10-WGP-NEXT:    s_clause 0x1
5635; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5636; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5637; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5638; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
5639; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
5640; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5641; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5642; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5643; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5644; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5645; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5646; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5647; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5648; GFX10-WGP-NEXT:    buffer_gl0_inv
5649; GFX10-WGP-NEXT:    buffer_gl1_inv
5650; GFX10-WGP-NEXT:    s_endpgm
5651;
5652; GFX10-CU-LABEL: flat_system_one_as_release_acquire_cmpxchg:
5653; GFX10-CU:       ; %bb.0: ; %entry
5654; GFX10-CU-NEXT:    s_clause 0x1
5655; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5656; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5657; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5658; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
5659; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
5660; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5661; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5662; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5663; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5664; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5665; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5666; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5667; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5668; GFX10-CU-NEXT:    buffer_gl0_inv
5669; GFX10-CU-NEXT:    buffer_gl1_inv
5670; GFX10-CU-NEXT:    s_endpgm
5671;
5672; SKIP-CACHE-INV-LABEL: flat_system_one_as_release_acquire_cmpxchg:
5673; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5674; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
5675; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5676; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5677; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
5678; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
5679; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5680; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5681; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5682; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5683; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5684; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5685; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5686; SKIP-CACHE-INV-NEXT:    s_endpgm
5687;
5688; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_release_acquire_cmpxchg:
5689; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5690; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5691; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5692; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5693; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5694; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5695; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
5696; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5697; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
5698; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5699; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
5700; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5701; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5702; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5703;
5704; GFX90A-TGSPLIT-LABEL: flat_system_one_as_release_acquire_cmpxchg:
5705; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5706; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5707; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5708; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5709; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5710; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5711; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
5712; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5713; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
5714; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5715; GFX90A-TGSPLIT-NEXT:    buffer_invl2
5716; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5717; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5718; GFX90A-TGSPLIT-NEXT:    s_endpgm
5719    i32* %out, i32 %in, i32 %old) {
5720entry:
5721  %gep = getelementptr i32, i32* %out, i32 4
5722  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") release acquire
5723  ret void
5724}
5725
5726define amdgpu_kernel void @flat_system_one_as_acq_rel_acquire_cmpxchg(
5727; GFX7-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
5728; GFX7:       ; %bb.0: ; %entry
5729; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5730; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5731; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5732; GFX7-NEXT:    s_add_u32 s0, s0, 16
5733; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5734; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5735; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5736; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5737; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5738; GFX7-NEXT:    s_waitcnt vmcnt(0)
5739; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5740; GFX7-NEXT:    s_waitcnt vmcnt(0)
5741; GFX7-NEXT:    buffer_wbinvl1_vol
5742; GFX7-NEXT:    s_endpgm
5743;
5744; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
5745; GFX10-WGP:       ; %bb.0: ; %entry
5746; GFX10-WGP-NEXT:    s_clause 0x1
5747; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5748; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5749; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5750; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
5751; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
5752; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5753; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5754; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5755; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5756; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5757; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5758; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5759; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5760; GFX10-WGP-NEXT:    buffer_gl0_inv
5761; GFX10-WGP-NEXT:    buffer_gl1_inv
5762; GFX10-WGP-NEXT:    s_endpgm
5763;
5764; GFX10-CU-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
5765; GFX10-CU:       ; %bb.0: ; %entry
5766; GFX10-CU-NEXT:    s_clause 0x1
5767; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5768; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5769; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5770; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
5771; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
5772; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5773; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5774; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5775; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5776; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5777; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5778; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5779; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5780; GFX10-CU-NEXT:    buffer_gl0_inv
5781; GFX10-CU-NEXT:    buffer_gl1_inv
5782; GFX10-CU-NEXT:    s_endpgm
5783;
5784; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
5785; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5786; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
5787; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5788; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5789; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
5790; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
5791; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5792; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5793; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5794; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5795; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5796; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5797; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5798; SKIP-CACHE-INV-NEXT:    s_endpgm
5799;
5800; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
5801; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5802; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5803; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5804; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5805; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5806; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5807; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
5808; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5809; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
5810; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5811; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
5812; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5813; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5814; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5815;
5816; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
5817; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5818; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5819; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5820; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5821; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5822; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5823; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
5824; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5825; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
5826; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5827; GFX90A-TGSPLIT-NEXT:    buffer_invl2
5828; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5829; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5830; GFX90A-TGSPLIT-NEXT:    s_endpgm
5831    i32* %out, i32 %in, i32 %old) {
5832entry:
5833  %gep = getelementptr i32, i32* %out, i32 4
5834  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel acquire
5835  ret void
5836}
5837
5838define amdgpu_kernel void @flat_system_one_as_seq_cst_acquire_cmpxchg(
5839; GFX7-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
5840; GFX7:       ; %bb.0: ; %entry
5841; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5842; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5843; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5844; GFX7-NEXT:    s_add_u32 s0, s0, 16
5845; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5846; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5847; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5848; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5849; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5850; GFX7-NEXT:    s_waitcnt vmcnt(0)
5851; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5852; GFX7-NEXT:    s_waitcnt vmcnt(0)
5853; GFX7-NEXT:    buffer_wbinvl1_vol
5854; GFX7-NEXT:    s_endpgm
5855;
5856; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
5857; GFX10-WGP:       ; %bb.0: ; %entry
5858; GFX10-WGP-NEXT:    s_clause 0x1
5859; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5860; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5861; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5862; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
5863; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
5864; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5865; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5866; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5867; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5868; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5869; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5870; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5871; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5872; GFX10-WGP-NEXT:    buffer_gl0_inv
5873; GFX10-WGP-NEXT:    buffer_gl1_inv
5874; GFX10-WGP-NEXT:    s_endpgm
5875;
5876; GFX10-CU-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
5877; GFX10-CU:       ; %bb.0: ; %entry
5878; GFX10-CU-NEXT:    s_clause 0x1
5879; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5880; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5881; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5882; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
5883; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
5884; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5885; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5886; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5887; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5888; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5889; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5890; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5891; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5892; GFX10-CU-NEXT:    buffer_gl0_inv
5893; GFX10-CU-NEXT:    buffer_gl1_inv
5894; GFX10-CU-NEXT:    s_endpgm
5895;
5896; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
5897; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5898; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
5899; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5900; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5901; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
5902; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
5903; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5904; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5905; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5906; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5907; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5908; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5909; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5910; SKIP-CACHE-INV-NEXT:    s_endpgm
5911;
5912; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
5913; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5914; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5915; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5916; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5917; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5918; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5919; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
5920; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5921; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
5922; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5923; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
5924; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5925; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5926; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5927;
5928; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
5929; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5930; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5931; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5932; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5933; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5934; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5935; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
5936; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5937; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
5938; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5939; GFX90A-TGSPLIT-NEXT:    buffer_invl2
5940; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5941; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5942; GFX90A-TGSPLIT-NEXT:    s_endpgm
5943    i32* %out, i32 %in, i32 %old) {
5944entry:
5945  %gep = getelementptr i32, i32* %out, i32 4
5946  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst acquire
5947  ret void
5948}
5949
5950define amdgpu_kernel void @flat_system_one_as_seq_cst_seq_cst_cmpxchg(
5951; GFX7-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
5952; GFX7:       ; %bb.0: ; %entry
5953; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5954; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5955; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5956; GFX7-NEXT:    s_add_u32 s0, s0, 16
5957; GFX7-NEXT:    s_addc_u32 s1, s1, 0
5958; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5959; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5960; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5961; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5962; GFX7-NEXT:    s_waitcnt vmcnt(0)
5963; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5964; GFX7-NEXT:    s_waitcnt vmcnt(0)
5965; GFX7-NEXT:    buffer_wbinvl1_vol
5966; GFX7-NEXT:    s_endpgm
5967;
5968; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
5969; GFX10-WGP:       ; %bb.0: ; %entry
5970; GFX10-WGP-NEXT:    s_clause 0x1
5971; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5972; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5973; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5974; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
5975; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
5976; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5977; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5978; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5979; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5980; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5981; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5982; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
5983; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5984; GFX10-WGP-NEXT:    buffer_gl0_inv
5985; GFX10-WGP-NEXT:    buffer_gl1_inv
5986; GFX10-WGP-NEXT:    s_endpgm
5987;
5988; GFX10-CU-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
5989; GFX10-CU:       ; %bb.0: ; %entry
5990; GFX10-CU-NEXT:    s_clause 0x1
5991; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5992; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5993; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5994; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
5995; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
5996; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5997; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5998; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5999; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6000; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6001; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6002; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6003; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6004; GFX10-CU-NEXT:    buffer_gl0_inv
6005; GFX10-CU-NEXT:    buffer_gl1_inv
6006; GFX10-CU-NEXT:    s_endpgm
6007;
6008; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
6009; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6010; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
6011; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
6012; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6013; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
6014; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
6015; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6016; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6017; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6018; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6019; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6020; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6021; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6022; SKIP-CACHE-INV-NEXT:    s_endpgm
6023;
6024; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
6025; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6026; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6027; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6028; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6029; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6030; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6031; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
6032; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6033; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
6034; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6035; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
6036; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6037; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6038; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6039;
6040; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
6041; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6042; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6043; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6044; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6045; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6046; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6047; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
6048; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6049; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 scc
6050; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6051; GFX90A-TGSPLIT-NEXT:    buffer_invl2
6052; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6053; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6054; GFX90A-TGSPLIT-NEXT:    s_endpgm
6055    i32* %out, i32 %in, i32 %old) {
6056entry:
6057  %gep = getelementptr i32, i32* %out, i32 4
6058  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst seq_cst
6059  ret void
6060}
6061
6062define amdgpu_kernel void @flat_system_one_as_acquire_monotonic_ret_cmpxchg(
6063; GFX7-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
6064; GFX7:       ; %bb.0: ; %entry
6065; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6066; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6067; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6068; GFX7-NEXT:    s_add_u32 s4, s0, 16
6069; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6070; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6071; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6072; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6073; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6074; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6075; GFX7-NEXT:    s_waitcnt vmcnt(0)
6076; GFX7-NEXT:    buffer_wbinvl1_vol
6077; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6078; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6079; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6080; GFX7-NEXT:    flat_store_dword v[0:1], v2
6081; GFX7-NEXT:    s_endpgm
6082;
6083; GFX10-WGP-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
6084; GFX10-WGP:       ; %bb.0: ; %entry
6085; GFX10-WGP-NEXT:    s_clause 0x1
6086; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6087; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6088; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6089; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6090; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6091; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6092; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6093; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6094; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6095; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6096; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6097; GFX10-WGP-NEXT:    buffer_gl0_inv
6098; GFX10-WGP-NEXT:    buffer_gl1_inv
6099; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6100; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6101; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6102; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6103; GFX10-WGP-NEXT:    s_endpgm
6104;
6105; GFX10-CU-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
6106; GFX10-CU:       ; %bb.0: ; %entry
6107; GFX10-CU-NEXT:    s_clause 0x1
6108; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6109; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6110; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6111; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6112; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6113; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6114; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6115; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6116; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6117; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6118; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6119; GFX10-CU-NEXT:    buffer_gl0_inv
6120; GFX10-CU-NEXT:    buffer_gl1_inv
6121; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6122; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6123; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6124; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6125; GFX10-CU-NEXT:    s_endpgm
6126;
6127; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
6128; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6129; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
6130; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
6131; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6132; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6133; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6134; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6135; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6136; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6137; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6138; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6139; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6140; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6141; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6142; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6143; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6144; SKIP-CACHE-INV-NEXT:    s_endpgm
6145;
6146; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
6147; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6148; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6149; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6150; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6151; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6152; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6153; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc scc
6154; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6155; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
6156; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6157; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6158; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6159; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6160; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6161;
6162; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
6163; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6164; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6165; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6166; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6167; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6168; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6169; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc scc
6170; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6171; GFX90A-TGSPLIT-NEXT:    buffer_invl2
6172; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6173; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6174; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6175; GFX90A-TGSPLIT-NEXT:    s_endpgm
6176    i32* %out, i32 %in, i32 %old) {
6177entry:
6178  %gep = getelementptr i32, i32* %out, i32 4
6179  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acquire monotonic
6180  %val0 = extractvalue { i32, i1 } %val, 0
6181  store i32 %val0, i32* %out, align 4
6182  ret void
6183}
6184
6185define amdgpu_kernel void @flat_system_one_as_acq_rel_monotonic_ret_cmpxchg(
6186; GFX7-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
6187; GFX7:       ; %bb.0: ; %entry
6188; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6189; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6190; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6191; GFX7-NEXT:    s_add_u32 s4, s0, 16
6192; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6193; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6194; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6195; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6196; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6197; GFX7-NEXT:    s_waitcnt vmcnt(0)
6198; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6199; GFX7-NEXT:    s_waitcnt vmcnt(0)
6200; GFX7-NEXT:    buffer_wbinvl1_vol
6201; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6202; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6203; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6204; GFX7-NEXT:    flat_store_dword v[0:1], v2
6205; GFX7-NEXT:    s_endpgm
6206;
6207; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
6208; GFX10-WGP:       ; %bb.0: ; %entry
6209; GFX10-WGP-NEXT:    s_clause 0x1
6210; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6211; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6212; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6213; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6214; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6215; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6216; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6217; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6218; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6219; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6220; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6221; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6222; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6223; GFX10-WGP-NEXT:    buffer_gl0_inv
6224; GFX10-WGP-NEXT:    buffer_gl1_inv
6225; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6226; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6227; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6228; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6229; GFX10-WGP-NEXT:    s_endpgm
6230;
6231; GFX10-CU-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
6232; GFX10-CU:       ; %bb.0: ; %entry
6233; GFX10-CU-NEXT:    s_clause 0x1
6234; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6235; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6236; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6237; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6238; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6239; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6240; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6241; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6242; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6243; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6244; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6245; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6246; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6247; GFX10-CU-NEXT:    buffer_gl0_inv
6248; GFX10-CU-NEXT:    buffer_gl1_inv
6249; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6250; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6251; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6252; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6253; GFX10-CU-NEXT:    s_endpgm
6254;
6255; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
6256; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6257; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
6258; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
6259; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6260; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6261; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6262; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6263; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6264; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6265; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6266; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6267; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6268; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6269; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6270; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6271; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6272; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6273; SKIP-CACHE-INV-NEXT:    s_endpgm
6274;
6275; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
6276; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6277; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6278; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6279; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6280; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6281; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6282; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
6283; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6284; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc scc
6285; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6286; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
6287; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6288; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6289; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6290; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6291; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6292;
6293; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
6294; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6295; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6296; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6297; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6298; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6299; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6300; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
6301; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6302; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc scc
6303; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6304; GFX90A-TGSPLIT-NEXT:    buffer_invl2
6305; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6306; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6307; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6308; GFX90A-TGSPLIT-NEXT:    s_endpgm
6309    i32* %out, i32 %in, i32 %old) {
6310entry:
6311  %gep = getelementptr i32, i32* %out, i32 4
6312  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel monotonic
6313  %val0 = extractvalue { i32, i1 } %val, 0
6314  store i32 %val0, i32* %out, align 4
6315  ret void
6316}
6317
6318define amdgpu_kernel void @flat_system_one_as_seq_cst_monotonic_ret_cmpxchg(
6319; GFX7-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
6320; GFX7:       ; %bb.0: ; %entry
6321; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6322; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6323; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6324; GFX7-NEXT:    s_add_u32 s4, s0, 16
6325; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6326; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6327; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6328; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6329; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6330; GFX7-NEXT:    s_waitcnt vmcnt(0)
6331; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6332; GFX7-NEXT:    s_waitcnt vmcnt(0)
6333; GFX7-NEXT:    buffer_wbinvl1_vol
6334; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6335; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6336; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6337; GFX7-NEXT:    flat_store_dword v[0:1], v2
6338; GFX7-NEXT:    s_endpgm
6339;
6340; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
6341; GFX10-WGP:       ; %bb.0: ; %entry
6342; GFX10-WGP-NEXT:    s_clause 0x1
6343; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6344; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6345; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6346; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6347; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6348; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6349; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6350; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6351; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6352; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6353; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6354; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6355; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6356; GFX10-WGP-NEXT:    buffer_gl0_inv
6357; GFX10-WGP-NEXT:    buffer_gl1_inv
6358; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6359; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6360; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6361; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6362; GFX10-WGP-NEXT:    s_endpgm
6363;
6364; GFX10-CU-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
6365; GFX10-CU:       ; %bb.0: ; %entry
6366; GFX10-CU-NEXT:    s_clause 0x1
6367; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6368; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6369; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6370; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6371; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6372; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6373; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6374; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6375; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6376; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6377; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6378; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6379; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6380; GFX10-CU-NEXT:    buffer_gl0_inv
6381; GFX10-CU-NEXT:    buffer_gl1_inv
6382; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6383; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6384; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6385; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6386; GFX10-CU-NEXT:    s_endpgm
6387;
6388; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
6389; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6390; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
6391; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
6392; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6393; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6394; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6395; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6396; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6397; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6398; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6399; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6400; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6401; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6402; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6403; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6404; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6405; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6406; SKIP-CACHE-INV-NEXT:    s_endpgm
6407;
6408; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
6409; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6410; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6411; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6412; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6413; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6414; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6415; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
6416; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6417; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc scc
6418; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6419; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
6420; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6421; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6422; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6423; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6424; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6425;
6426; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
6427; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6428; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6429; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6430; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6431; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6432; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6433; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
6434; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6435; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc scc
6436; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6437; GFX90A-TGSPLIT-NEXT:    buffer_invl2
6438; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6439; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6440; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6441; GFX90A-TGSPLIT-NEXT:    s_endpgm
6442    i32* %out, i32 %in, i32 %old) {
6443entry:
6444  %gep = getelementptr i32, i32* %out, i32 4
6445  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst monotonic
6446  %val0 = extractvalue { i32, i1 } %val, 0
6447  store i32 %val0, i32* %out, align 4
6448  ret void
6449}
6450
6451define amdgpu_kernel void @flat_system_one_as_acquire_acquire_ret_cmpxchg(
6452; GFX7-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
6453; GFX7:       ; %bb.0: ; %entry
6454; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6455; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6456; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6457; GFX7-NEXT:    s_add_u32 s4, s0, 16
6458; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6459; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6460; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6461; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6462; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6463; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6464; GFX7-NEXT:    s_waitcnt vmcnt(0)
6465; GFX7-NEXT:    buffer_wbinvl1_vol
6466; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6467; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6468; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6469; GFX7-NEXT:    flat_store_dword v[0:1], v2
6470; GFX7-NEXT:    s_endpgm
6471;
6472; GFX10-WGP-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
6473; GFX10-WGP:       ; %bb.0: ; %entry
6474; GFX10-WGP-NEXT:    s_clause 0x1
6475; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6476; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6477; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6478; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6479; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6480; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6481; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6482; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6483; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6484; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6485; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6486; GFX10-WGP-NEXT:    buffer_gl0_inv
6487; GFX10-WGP-NEXT:    buffer_gl1_inv
6488; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6489; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6490; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6491; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6492; GFX10-WGP-NEXT:    s_endpgm
6493;
6494; GFX10-CU-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
6495; GFX10-CU:       ; %bb.0: ; %entry
6496; GFX10-CU-NEXT:    s_clause 0x1
6497; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6498; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6499; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6500; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6501; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6502; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6503; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6504; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6505; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6506; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6507; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6508; GFX10-CU-NEXT:    buffer_gl0_inv
6509; GFX10-CU-NEXT:    buffer_gl1_inv
6510; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6511; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6512; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6513; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6514; GFX10-CU-NEXT:    s_endpgm
6515;
6516; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
6517; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6518; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
6519; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
6520; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6521; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6522; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6523; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6524; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6525; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6526; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6527; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6528; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6529; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6530; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6531; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6532; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6533; SKIP-CACHE-INV-NEXT:    s_endpgm
6534;
6535; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
6536; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6537; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6538; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6539; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6540; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6541; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6542; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc scc
6543; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6544; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
6545; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6546; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6547; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6548; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6549; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6550;
6551; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
6552; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6553; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6554; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6555; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6556; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6557; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6558; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc scc
6559; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6560; GFX90A-TGSPLIT-NEXT:    buffer_invl2
6561; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6562; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6563; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6564; GFX90A-TGSPLIT-NEXT:    s_endpgm
6565    i32* %out, i32 %in, i32 %old) {
6566entry:
6567  %gep = getelementptr i32, i32* %out, i32 4
6568  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acquire acquire
6569  %val0 = extractvalue { i32, i1 } %val, 0
6570  store i32 %val0, i32* %out, align 4
6571  ret void
6572}
6573
6574define amdgpu_kernel void @flat_system_one_as_release_acquire_ret_cmpxchg(
6575; GFX7-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
6576; GFX7:       ; %bb.0: ; %entry
6577; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6578; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6579; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6580; GFX7-NEXT:    s_add_u32 s4, s0, 16
6581; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6582; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6583; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6584; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6585; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6586; GFX7-NEXT:    s_waitcnt vmcnt(0)
6587; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6588; GFX7-NEXT:    s_waitcnt vmcnt(0)
6589; GFX7-NEXT:    buffer_wbinvl1_vol
6590; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6591; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6592; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6593; GFX7-NEXT:    flat_store_dword v[0:1], v2
6594; GFX7-NEXT:    s_endpgm
6595;
6596; GFX10-WGP-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
6597; GFX10-WGP:       ; %bb.0: ; %entry
6598; GFX10-WGP-NEXT:    s_clause 0x1
6599; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6600; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6601; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6602; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6603; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6604; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6605; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6606; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6607; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6608; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6609; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6610; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6611; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6612; GFX10-WGP-NEXT:    buffer_gl0_inv
6613; GFX10-WGP-NEXT:    buffer_gl1_inv
6614; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6615; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6616; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6617; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6618; GFX10-WGP-NEXT:    s_endpgm
6619;
6620; GFX10-CU-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
6621; GFX10-CU:       ; %bb.0: ; %entry
6622; GFX10-CU-NEXT:    s_clause 0x1
6623; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6624; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6625; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6626; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6627; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6628; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6629; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6630; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6631; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6632; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6633; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6634; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6635; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6636; GFX10-CU-NEXT:    buffer_gl0_inv
6637; GFX10-CU-NEXT:    buffer_gl1_inv
6638; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6639; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6640; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6641; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6642; GFX10-CU-NEXT:    s_endpgm
6643;
6644; SKIP-CACHE-INV-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
6645; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6646; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
6647; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
6648; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6649; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6650; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6651; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6652; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6653; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6654; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6655; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6656; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6657; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6658; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6659; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6660; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6661; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6662; SKIP-CACHE-INV-NEXT:    s_endpgm
6663;
6664; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
6665; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6666; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6667; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6668; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6669; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6670; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6671; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
6672; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6673; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc scc
6674; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6675; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
6676; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6677; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6678; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6679; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6680; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6681;
6682; GFX90A-TGSPLIT-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
6683; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6684; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6685; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6686; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6687; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6688; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6689; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
6690; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6691; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc scc
6692; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6693; GFX90A-TGSPLIT-NEXT:    buffer_invl2
6694; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6695; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6696; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6697; GFX90A-TGSPLIT-NEXT:    s_endpgm
6698    i32* %out, i32 %in, i32 %old) {
6699entry:
6700  %gep = getelementptr i32, i32* %out, i32 4
6701  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") release acquire
6702  %val0 = extractvalue { i32, i1 } %val, 0
6703  store i32 %val0, i32* %out, align 4
6704  ret void
6705}
6706
6707define amdgpu_kernel void @flat_system_one_as_acq_rel_acquire_ret_cmpxchg(
6708; GFX7-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
6709; GFX7:       ; %bb.0: ; %entry
6710; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6711; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6712; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6713; GFX7-NEXT:    s_add_u32 s4, s0, 16
6714; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6715; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6716; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6717; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6718; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6719; GFX7-NEXT:    s_waitcnt vmcnt(0)
6720; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6721; GFX7-NEXT:    s_waitcnt vmcnt(0)
6722; GFX7-NEXT:    buffer_wbinvl1_vol
6723; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6724; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6725; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6726; GFX7-NEXT:    flat_store_dword v[0:1], v2
6727; GFX7-NEXT:    s_endpgm
6728;
6729; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
6730; GFX10-WGP:       ; %bb.0: ; %entry
6731; GFX10-WGP-NEXT:    s_clause 0x1
6732; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6733; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6734; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6735; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6736; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6737; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6738; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6739; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6740; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6741; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6742; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6743; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6744; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6745; GFX10-WGP-NEXT:    buffer_gl0_inv
6746; GFX10-WGP-NEXT:    buffer_gl1_inv
6747; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6748; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6749; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6750; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6751; GFX10-WGP-NEXT:    s_endpgm
6752;
6753; GFX10-CU-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
6754; GFX10-CU:       ; %bb.0: ; %entry
6755; GFX10-CU-NEXT:    s_clause 0x1
6756; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6757; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6758; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6759; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6760; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6761; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6762; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6763; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6764; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6765; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6766; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6767; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6768; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6769; GFX10-CU-NEXT:    buffer_gl0_inv
6770; GFX10-CU-NEXT:    buffer_gl1_inv
6771; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6772; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6773; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6774; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6775; GFX10-CU-NEXT:    s_endpgm
6776;
6777; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
6778; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6779; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
6780; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
6781; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6782; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6783; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6784; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6785; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6786; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6787; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6788; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6789; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6790; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6791; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6792; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6793; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6794; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6795; SKIP-CACHE-INV-NEXT:    s_endpgm
6796;
6797; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
6798; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6799; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6800; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6801; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6802; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6803; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6804; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
6805; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6806; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc scc
6807; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6808; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
6809; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6810; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6811; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6812; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6813; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6814;
6815; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
6816; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6817; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6818; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6819; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6820; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6821; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6822; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
6823; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6824; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc scc
6825; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6826; GFX90A-TGSPLIT-NEXT:    buffer_invl2
6827; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6828; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6829; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6830; GFX90A-TGSPLIT-NEXT:    s_endpgm
6831    i32* %out, i32 %in, i32 %old) {
6832entry:
6833  %gep = getelementptr i32, i32* %out, i32 4
6834  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel acquire
6835  %val0 = extractvalue { i32, i1 } %val, 0
6836  store i32 %val0, i32* %out, align 4
6837  ret void
6838}
6839
6840define amdgpu_kernel void @flat_system_one_as_seq_cst_acquire_ret_cmpxchg(
6841; GFX7-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
6842; GFX7:       ; %bb.0: ; %entry
6843; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6844; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6845; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6846; GFX7-NEXT:    s_add_u32 s4, s0, 16
6847; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6848; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6849; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6850; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6851; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6852; GFX7-NEXT:    s_waitcnt vmcnt(0)
6853; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6854; GFX7-NEXT:    s_waitcnt vmcnt(0)
6855; GFX7-NEXT:    buffer_wbinvl1_vol
6856; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6857; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6858; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6859; GFX7-NEXT:    flat_store_dword v[0:1], v2
6860; GFX7-NEXT:    s_endpgm
6861;
6862; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
6863; GFX10-WGP:       ; %bb.0: ; %entry
6864; GFX10-WGP-NEXT:    s_clause 0x1
6865; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6866; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6867; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6868; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
6869; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
6870; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
6871; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6872; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
6873; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
6874; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6875; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6876; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6877; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6878; GFX10-WGP-NEXT:    buffer_gl0_inv
6879; GFX10-WGP-NEXT:    buffer_gl1_inv
6880; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6881; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6882; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6883; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6884; GFX10-WGP-NEXT:    s_endpgm
6885;
6886; GFX10-CU-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
6887; GFX10-CU:       ; %bb.0: ; %entry
6888; GFX10-CU-NEXT:    s_clause 0x1
6889; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6890; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6891; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6892; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
6893; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
6894; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
6895; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6896; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
6897; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6898; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6899; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6900; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6901; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6902; GFX10-CU-NEXT:    buffer_gl0_inv
6903; GFX10-CU-NEXT:    buffer_gl1_inv
6904; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6905; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6906; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6907; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6908; GFX10-CU-NEXT:    s_endpgm
6909;
6910; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
6911; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6912; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
6913; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
6914; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6915; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6916; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6917; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6918; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6919; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6920; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6921; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6922; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6923; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6924; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6925; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6926; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6927; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6928; SKIP-CACHE-INV-NEXT:    s_endpgm
6929;
6930; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
6931; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6932; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6933; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6934; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6935; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6936; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6937; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
6938; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6939; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc scc
6940; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6941; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
6942; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6943; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6944; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6945; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6946; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6947;
6948; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
6949; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6950; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6951; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6952; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6953; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6954; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6955; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
6956; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6957; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc scc
6958; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6959; GFX90A-TGSPLIT-NEXT:    buffer_invl2
6960; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6961; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6962; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6963; GFX90A-TGSPLIT-NEXT:    s_endpgm
6964    i32* %out, i32 %in, i32 %old) {
6965entry:
6966  %gep = getelementptr i32, i32* %out, i32 4
6967  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst acquire
6968  %val0 = extractvalue { i32, i1 } %val, 0
6969  store i32 %val0, i32* %out, align 4
6970  ret void
6971}
6972
6973define amdgpu_kernel void @flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg(
6974; GFX7-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
6975; GFX7:       ; %bb.0: ; %entry
6976; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6977; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6978; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6979; GFX7-NEXT:    s_add_u32 s4, s0, 16
6980; GFX7-NEXT:    s_addc_u32 s5, s1, 0
6981; GFX7-NEXT:    v_mov_b32_e32 v0, s4
6982; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6983; GFX7-NEXT:    v_mov_b32_e32 v1, s5
6984; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6985; GFX7-NEXT:    s_waitcnt vmcnt(0)
6986; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6987; GFX7-NEXT:    s_waitcnt vmcnt(0)
6988; GFX7-NEXT:    buffer_wbinvl1_vol
6989; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6990; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6991; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6992; GFX7-NEXT:    flat_store_dword v[0:1], v2
6993; GFX7-NEXT:    s_endpgm
6994;
6995; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
6996; GFX10-WGP:       ; %bb.0: ; %entry
6997; GFX10-WGP-NEXT:    s_clause 0x1
6998; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6999; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7000; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7001; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
7002; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
7003; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
7004; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7005; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
7006; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7007; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7008; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7009; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7010; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7011; GFX10-WGP-NEXT:    buffer_gl0_inv
7012; GFX10-WGP-NEXT:    buffer_gl1_inv
7013; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7014; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7015; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7016; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7017; GFX10-WGP-NEXT:    s_endpgm
7018;
7019; GFX10-CU-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
7020; GFX10-CU:       ; %bb.0: ; %entry
7021; GFX10-CU-NEXT:    s_clause 0x1
7022; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7023; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7024; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7025; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
7026; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
7027; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
7028; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7029; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
7030; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7031; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7032; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7033; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7034; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7035; GFX10-CU-NEXT:    buffer_gl0_inv
7036; GFX10-CU-NEXT:    buffer_gl1_inv
7037; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7038; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7039; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7040; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7041; GFX10-CU-NEXT:    s_endpgm
7042;
7043; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
7044; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7045; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
7046; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
7047; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7048; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
7049; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
7050; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
7051; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
7052; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
7053; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7054; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7055; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7056; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7057; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7058; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7059; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7060; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7061; SKIP-CACHE-INV-NEXT:    s_endpgm
7062;
7063; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
7064; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7065; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7066; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7067; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7068; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7069; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7070; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
7071; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7072; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc scc
7073; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7074; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
7075; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7076; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7077; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7078; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7079; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7080;
7081; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
7082; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7083; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7084; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7085; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7086; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7087; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7088; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
7089; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7090; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc scc
7091; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7092; GFX90A-TGSPLIT-NEXT:    buffer_invl2
7093; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7094; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7095; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7096; GFX90A-TGSPLIT-NEXT:    s_endpgm
7097    i32* %out, i32 %in, i32 %old) {
7098entry:
7099  %gep = getelementptr i32, i32* %out, i32 4
7100  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst seq_cst
7101  %val0 = extractvalue { i32, i1 } %val, 0
7102  store i32 %val0, i32* %out, align 4
7103  ret void
7104}
7105
7106