1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
3; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
4; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
5; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
6; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-NOTTGSPLIT %s
7; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-TGSPLIT %s
8; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-NOTTGSPLIT %s
9; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
10
11define amdgpu_kernel void @flat_system_unordered_load(
12; GFX7-LABEL: flat_system_unordered_load:
13; GFX7:       ; %bb.0: ; %entry
14; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
15; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
16; GFX7-NEXT:    v_mov_b32_e32 v0, s0
17; GFX7-NEXT:    v_mov_b32_e32 v1, s1
18; GFX7-NEXT:    flat_load_dword v2, v[0:1]
19; GFX7-NEXT:    v_mov_b32_e32 v0, s2
20; GFX7-NEXT:    v_mov_b32_e32 v1, s3
21; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
22; GFX7-NEXT:    flat_store_dword v[0:1], v2
23; GFX7-NEXT:    s_endpgm
24;
25; GFX10-WGP-LABEL: flat_system_unordered_load:
26; GFX10-WGP:       ; %bb.0: ; %entry
27; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
28; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
29; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
30; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
31; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
32; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
33; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
34; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
35; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
36; GFX10-WGP-NEXT:    s_endpgm
37;
38; GFX10-CU-LABEL: flat_system_unordered_load:
39; GFX10-CU:       ; %bb.0: ; %entry
40; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
41; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
42; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
43; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
44; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
45; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
46; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
47; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
48; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
49; GFX10-CU-NEXT:    s_endpgm
50;
51; SKIP-CACHE-INV-LABEL: flat_system_unordered_load:
52; SKIP-CACHE-INV:       ; %bb.0: ; %entry
53; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
54; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
55; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
56; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
57; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
58; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
59; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
60; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
61; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
62; SKIP-CACHE-INV-NEXT:    s_endpgm
63;
64; GFX90A-NOTTGSPLIT-LABEL: flat_system_unordered_load:
65; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
66; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
67; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
68; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
69; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
70; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
71; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
72; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
73; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
74; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
75; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
76;
77; GFX90A-TGSPLIT-LABEL: flat_system_unordered_load:
78; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
79; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
80; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
81; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
82; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
83; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
84; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
85; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
86; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
87; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
88; GFX90A-TGSPLIT-NEXT:    s_endpgm
89;
90; GFX940-NOTTGSPLIT-LABEL: flat_system_unordered_load:
91; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
92; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
93; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
94; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
95; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
96; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
97; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
98; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
99; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
100; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
101; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
102;
103; GFX940-TGSPLIT-LABEL: flat_system_unordered_load:
104; GFX940-TGSPLIT:       ; %bb.0: ; %entry
105; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
106; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
107; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
108; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
109; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
110; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
111; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
112; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
113; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
114; GFX940-TGSPLIT-NEXT:    s_endpgm
115    i32* %in, i32* %out) {
116entry:
117  %val = load atomic i32, i32* %in unordered, align 4
118  store i32 %val, i32* %out
119  ret void
120}
121
122define amdgpu_kernel void @flat_system_monotonic_load(
123; GFX7-LABEL: flat_system_monotonic_load:
124; GFX7:       ; %bb.0: ; %entry
125; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
126; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
127; GFX7-NEXT:    v_mov_b32_e32 v0, s0
128; GFX7-NEXT:    v_mov_b32_e32 v1, s1
129; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
130; GFX7-NEXT:    v_mov_b32_e32 v0, s2
131; GFX7-NEXT:    v_mov_b32_e32 v1, s3
132; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
133; GFX7-NEXT:    flat_store_dword v[0:1], v2
134; GFX7-NEXT:    s_endpgm
135;
136; GFX10-WGP-LABEL: flat_system_monotonic_load:
137; GFX10-WGP:       ; %bb.0: ; %entry
138; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
139; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
140; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
141; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
142; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
143; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
144; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
145; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
146; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
147; GFX10-WGP-NEXT:    s_endpgm
148;
149; GFX10-CU-LABEL: flat_system_monotonic_load:
150; GFX10-CU:       ; %bb.0: ; %entry
151; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
152; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
153; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
154; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
155; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
156; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
157; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
158; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
159; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
160; GFX10-CU-NEXT:    s_endpgm
161;
162; SKIP-CACHE-INV-LABEL: flat_system_monotonic_load:
163; SKIP-CACHE-INV:       ; %bb.0: ; %entry
164; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
165; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
166; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
167; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
168; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
169; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
170; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
171; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
172; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
173; SKIP-CACHE-INV-NEXT:    s_endpgm
174;
175; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_load:
176; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
177; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
178; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
179; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
180; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
181; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
182; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
183; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
184; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
185; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
186; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
187;
188; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_load:
189; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
190; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
191; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
192; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
193; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
194; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
195; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
196; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
197; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
198; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
199; GFX90A-TGSPLIT-NEXT:    s_endpgm
200;
201; GFX940-NOTTGSPLIT-LABEL: flat_system_monotonic_load:
202; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
203; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
204; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
205; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
206; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
207; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0 sc1
208; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
209; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
210; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
211; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
212; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
213;
214; GFX940-TGSPLIT-LABEL: flat_system_monotonic_load:
215; GFX940-TGSPLIT:       ; %bb.0: ; %entry
216; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
217; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
218; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
219; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
220; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0 sc1
221; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
222; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
223; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
224; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
225; GFX940-TGSPLIT-NEXT:    s_endpgm
226    i32* %in, i32* %out) {
227entry:
228  %val = load atomic i32, i32* %in monotonic, align 4
229  store i32 %val, i32* %out
230  ret void
231}
232
233define amdgpu_kernel void @flat_system_acquire_load(
234; GFX7-LABEL: flat_system_acquire_load:
235; GFX7:       ; %bb.0: ; %entry
236; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
237; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
238; GFX7-NEXT:    v_mov_b32_e32 v0, s0
239; GFX7-NEXT:    v_mov_b32_e32 v1, s1
240; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
241; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
242; GFX7-NEXT:    buffer_wbinvl1_vol
243; GFX7-NEXT:    v_mov_b32_e32 v0, s2
244; GFX7-NEXT:    v_mov_b32_e32 v1, s3
245; GFX7-NEXT:    flat_store_dword v[0:1], v2
246; GFX7-NEXT:    s_endpgm
247;
248; GFX10-WGP-LABEL: flat_system_acquire_load:
249; GFX10-WGP:       ; %bb.0: ; %entry
250; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
251; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
252; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
253; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
254; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
255; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
256; GFX10-WGP-NEXT:    buffer_gl0_inv
257; GFX10-WGP-NEXT:    buffer_gl1_inv
258; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
259; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
260; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
261; GFX10-WGP-NEXT:    s_endpgm
262;
263; GFX10-CU-LABEL: flat_system_acquire_load:
264; GFX10-CU:       ; %bb.0: ; %entry
265; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
266; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
267; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
268; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
269; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
270; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
271; GFX10-CU-NEXT:    buffer_gl0_inv
272; GFX10-CU-NEXT:    buffer_gl1_inv
273; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
274; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
275; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
276; GFX10-CU-NEXT:    s_endpgm
277;
278; SKIP-CACHE-INV-LABEL: flat_system_acquire_load:
279; SKIP-CACHE-INV:       ; %bb.0: ; %entry
280; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
281; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
282; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
283; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
284; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
285; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
286; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
287; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
288; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
289; SKIP-CACHE-INV-NEXT:    s_endpgm
290;
291; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_load:
292; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
293; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
294; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
295; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
296; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
297; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
298; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
299; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
300; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
301; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
302; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
303; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
304; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
305;
306; GFX90A-TGSPLIT-LABEL: flat_system_acquire_load:
307; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
308; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
309; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
310; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
311; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
312; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
313; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
314; GFX90A-TGSPLIT-NEXT:    buffer_invl2
315; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
316; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
317; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
318; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
319; GFX90A-TGSPLIT-NEXT:    s_endpgm
320;
321; GFX940-NOTTGSPLIT-LABEL: flat_system_acquire_load:
322; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
323; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
324; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
325; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
326; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
327; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0 sc1
328; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
329; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
330; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
331; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
332; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
333; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
334;
335; GFX940-TGSPLIT-LABEL: flat_system_acquire_load:
336; GFX940-TGSPLIT:       ; %bb.0: ; %entry
337; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
338; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
339; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
340; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
341; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0 sc1
342; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
343; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
344; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
345; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
346; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
347; GFX940-TGSPLIT-NEXT:    s_endpgm
348    i32* %in, i32* %out) {
349entry:
350  %val = load atomic i32, i32* %in acquire, align 4
351  store i32 %val, i32* %out
352  ret void
353}
354
355define amdgpu_kernel void @flat_system_seq_cst_load(
356; GFX7-LABEL: flat_system_seq_cst_load:
357; GFX7:       ; %bb.0: ; %entry
358; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
359; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
360; GFX7-NEXT:    v_mov_b32_e32 v0, s0
361; GFX7-NEXT:    v_mov_b32_e32 v1, s1
362; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
363; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
364; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
365; GFX7-NEXT:    buffer_wbinvl1_vol
366; GFX7-NEXT:    v_mov_b32_e32 v0, s2
367; GFX7-NEXT:    v_mov_b32_e32 v1, s3
368; GFX7-NEXT:    flat_store_dword v[0:1], v2
369; GFX7-NEXT:    s_endpgm
370;
371; GFX10-WGP-LABEL: flat_system_seq_cst_load:
372; GFX10-WGP:       ; %bb.0: ; %entry
373; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
374; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
375; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
376; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
377; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
378; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
379; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
380; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
381; GFX10-WGP-NEXT:    buffer_gl0_inv
382; GFX10-WGP-NEXT:    buffer_gl1_inv
383; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
384; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
385; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
386; GFX10-WGP-NEXT:    s_endpgm
387;
388; GFX10-CU-LABEL: flat_system_seq_cst_load:
389; GFX10-CU:       ; %bb.0: ; %entry
390; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
391; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
392; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
393; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
394; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
395; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
396; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
397; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
398; GFX10-CU-NEXT:    buffer_gl0_inv
399; GFX10-CU-NEXT:    buffer_gl1_inv
400; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
401; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
402; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
403; GFX10-CU-NEXT:    s_endpgm
404;
405; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_load:
406; SKIP-CACHE-INV:       ; %bb.0: ; %entry
407; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
408; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
409; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
410; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
411; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
412; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
413; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
414; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
415; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
416; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
417; SKIP-CACHE-INV-NEXT:    s_endpgm
418;
419; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_load:
420; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
421; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
422; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
423; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
424; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
425; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
426; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
427; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
428; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
429; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
430; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
431; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
432; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
433; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
434;
435; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_load:
436; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
437; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
438; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
439; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
440; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
441; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
442; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
443; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
444; GFX90A-TGSPLIT-NEXT:    buffer_invl2
445; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
446; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
447; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
448; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
449; GFX90A-TGSPLIT-NEXT:    s_endpgm
450;
451; GFX940-NOTTGSPLIT-LABEL: flat_system_seq_cst_load:
452; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
453; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
454; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
455; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
456; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
457; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
458; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0 sc1
459; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
460; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
461; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
462; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
463; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
464; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
465;
466; GFX940-TGSPLIT-LABEL: flat_system_seq_cst_load:
467; GFX940-TGSPLIT:       ; %bb.0: ; %entry
468; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
469; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
470; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
471; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
472; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
473; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0 sc1
474; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
475; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
476; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
477; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
478; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
479; GFX940-TGSPLIT-NEXT:    s_endpgm
480    i32* %in, i32* %out) {
481entry:
482  %val = load atomic i32, i32* %in seq_cst, align 4
483  store i32 %val, i32* %out
484  ret void
485}
486
487define amdgpu_kernel void @flat_system_unordered_store(
488; GFX7-LABEL: flat_system_unordered_store:
489; GFX7:       ; %bb.0: ; %entry
490; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
491; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
492; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
493; GFX7-NEXT:    v_mov_b32_e32 v0, s0
494; GFX7-NEXT:    v_mov_b32_e32 v1, s1
495; GFX7-NEXT:    v_mov_b32_e32 v2, s2
496; GFX7-NEXT:    flat_store_dword v[0:1], v2
497; GFX7-NEXT:    s_endpgm
498;
499; GFX10-WGP-LABEL: flat_system_unordered_store:
500; GFX10-WGP:       ; %bb.0: ; %entry
501; GFX10-WGP-NEXT:    s_clause 0x1
502; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
503; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
504; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
505; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
506; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
507; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
508; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
509; GFX10-WGP-NEXT:    s_endpgm
510;
511; GFX10-CU-LABEL: flat_system_unordered_store:
512; GFX10-CU:       ; %bb.0: ; %entry
513; GFX10-CU-NEXT:    s_clause 0x1
514; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
515; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
516; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
517; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
518; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
519; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
520; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
521; GFX10-CU-NEXT:    s_endpgm
522;
523; SKIP-CACHE-INV-LABEL: flat_system_unordered_store:
524; SKIP-CACHE-INV:       ; %bb.0: ; %entry
525; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
526; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
527; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
528; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
529; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
530; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
531; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
532; SKIP-CACHE-INV-NEXT:    s_endpgm
533;
534; GFX90A-NOTTGSPLIT-LABEL: flat_system_unordered_store:
535; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
536; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
537; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
538; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
539; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
540; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
541; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
542; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
543;
544; GFX90A-TGSPLIT-LABEL: flat_system_unordered_store:
545; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
546; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
547; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
548; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
549; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
550; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
551; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
552; GFX90A-TGSPLIT-NEXT:    s_endpgm
553;
554; GFX940-NOTTGSPLIT-LABEL: flat_system_unordered_store:
555; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
556; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
557; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
558; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
559; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
560; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
561; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
562; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
563;
564; GFX940-TGSPLIT-LABEL: flat_system_unordered_store:
565; GFX940-TGSPLIT:       ; %bb.0: ; %entry
566; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
567; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
568; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
569; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
570; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
571; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
572; GFX940-TGSPLIT-NEXT:    s_endpgm
573    i32 %in, i32* %out) {
574entry:
575  store atomic i32 %in, i32* %out unordered, align 4
576  ret void
577}
578
579define amdgpu_kernel void @flat_system_monotonic_store(
580; GFX7-LABEL: flat_system_monotonic_store:
581; GFX7:       ; %bb.0: ; %entry
582; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
583; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
584; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
585; GFX7-NEXT:    v_mov_b32_e32 v0, s0
586; GFX7-NEXT:    v_mov_b32_e32 v1, s1
587; GFX7-NEXT:    v_mov_b32_e32 v2, s2
588; GFX7-NEXT:    flat_store_dword v[0:1], v2
589; GFX7-NEXT:    s_endpgm
590;
591; GFX10-WGP-LABEL: flat_system_monotonic_store:
592; GFX10-WGP:       ; %bb.0: ; %entry
593; GFX10-WGP-NEXT:    s_clause 0x1
594; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
595; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
596; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
597; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
598; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
599; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
600; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
601; GFX10-WGP-NEXT:    s_endpgm
602;
603; GFX10-CU-LABEL: flat_system_monotonic_store:
604; GFX10-CU:       ; %bb.0: ; %entry
605; GFX10-CU-NEXT:    s_clause 0x1
606; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
607; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
608; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
609; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
610; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
611; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
612; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
613; GFX10-CU-NEXT:    s_endpgm
614;
615; SKIP-CACHE-INV-LABEL: flat_system_monotonic_store:
616; SKIP-CACHE-INV:       ; %bb.0: ; %entry
617; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
618; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
619; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
620; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
621; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
622; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
623; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
624; SKIP-CACHE-INV-NEXT:    s_endpgm
625;
626; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_store:
627; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
628; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
629; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
630; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
631; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
632; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
633; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
634; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
635;
636; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_store:
637; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
638; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
639; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
640; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
641; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
642; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
643; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
644; GFX90A-TGSPLIT-NEXT:    s_endpgm
645;
646; GFX940-NOTTGSPLIT-LABEL: flat_system_monotonic_store:
647; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
648; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
649; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
650; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
651; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
652; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
653; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
654; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
655;
656; GFX940-TGSPLIT-LABEL: flat_system_monotonic_store:
657; GFX940-TGSPLIT:       ; %bb.0: ; %entry
658; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
659; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
660; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
661; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
662; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
663; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
664; GFX940-TGSPLIT-NEXT:    s_endpgm
665    i32 %in, i32* %out) {
666entry:
667  store atomic i32 %in, i32* %out monotonic, align 4
668  ret void
669}
670
671define amdgpu_kernel void @flat_system_release_store(
672; GFX7-LABEL: flat_system_release_store:
673; GFX7:       ; %bb.0: ; %entry
674; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
675; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
676; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
677; GFX7-NEXT:    v_mov_b32_e32 v0, s0
678; GFX7-NEXT:    v_mov_b32_e32 v1, s1
679; GFX7-NEXT:    v_mov_b32_e32 v2, s2
680; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
681; GFX7-NEXT:    flat_store_dword v[0:1], v2
682; GFX7-NEXT:    s_endpgm
683;
684; GFX10-WGP-LABEL: flat_system_release_store:
685; GFX10-WGP:       ; %bb.0: ; %entry
686; GFX10-WGP-NEXT:    s_clause 0x1
687; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
688; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
689; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
690; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
691; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
692; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
693; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
694; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
695; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
696; GFX10-WGP-NEXT:    s_endpgm
697;
698; GFX10-CU-LABEL: flat_system_release_store:
699; GFX10-CU:       ; %bb.0: ; %entry
700; GFX10-CU-NEXT:    s_clause 0x1
701; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
702; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
703; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
704; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
705; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
706; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
707; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
708; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
709; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
710; GFX10-CU-NEXT:    s_endpgm
711;
712; SKIP-CACHE-INV-LABEL: flat_system_release_store:
713; SKIP-CACHE-INV:       ; %bb.0: ; %entry
714; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
715; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
716; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
717; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
718; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
719; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
720; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
721; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
722; SKIP-CACHE-INV-NEXT:    s_endpgm
723;
724; GFX90A-NOTTGSPLIT-LABEL: flat_system_release_store:
725; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
726; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
727; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
728; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
729; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
730; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
731; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
732; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
733; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
734; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
735;
736; GFX90A-TGSPLIT-LABEL: flat_system_release_store:
737; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
738; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
739; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
740; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
741; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
742; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
743; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
744; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
745; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
746; GFX90A-TGSPLIT-NEXT:    s_endpgm
747;
748; GFX940-NOTTGSPLIT-LABEL: flat_system_release_store:
749; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
750; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
751; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
752; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
753; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
754; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
755; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
756; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
757; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
758; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
759;
760; GFX940-TGSPLIT-LABEL: flat_system_release_store:
761; GFX940-TGSPLIT:       ; %bb.0: ; %entry
762; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
763; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
764; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
765; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
766; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
767; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
768; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
769; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
770; GFX940-TGSPLIT-NEXT:    s_endpgm
771    i32 %in, i32* %out) {
772entry:
773  store atomic i32 %in, i32* %out release, align 4
774  ret void
775}
776
777define amdgpu_kernel void @flat_system_seq_cst_store(
778; GFX7-LABEL: flat_system_seq_cst_store:
779; GFX7:       ; %bb.0: ; %entry
780; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
781; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
782; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
783; GFX7-NEXT:    v_mov_b32_e32 v0, s0
784; GFX7-NEXT:    v_mov_b32_e32 v1, s1
785; GFX7-NEXT:    v_mov_b32_e32 v2, s2
786; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
787; GFX7-NEXT:    flat_store_dword v[0:1], v2
788; GFX7-NEXT:    s_endpgm
789;
790; GFX10-WGP-LABEL: flat_system_seq_cst_store:
791; GFX10-WGP:       ; %bb.0: ; %entry
792; GFX10-WGP-NEXT:    s_clause 0x1
793; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
794; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
795; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
796; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
797; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
798; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
799; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
800; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
801; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
802; GFX10-WGP-NEXT:    s_endpgm
803;
804; GFX10-CU-LABEL: flat_system_seq_cst_store:
805; GFX10-CU:       ; %bb.0: ; %entry
806; GFX10-CU-NEXT:    s_clause 0x1
807; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
808; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
809; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
810; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
811; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
812; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
813; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
814; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
815; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
816; GFX10-CU-NEXT:    s_endpgm
817;
818; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_store:
819; SKIP-CACHE-INV:       ; %bb.0: ; %entry
820; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
821; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
822; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
823; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
824; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
825; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
826; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
827; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
828; SKIP-CACHE-INV-NEXT:    s_endpgm
829;
830; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_store:
831; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
832; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
833; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
834; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
835; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
836; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
837; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
838; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
839; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
840; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
841;
842; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_store:
843; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
844; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
845; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
846; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
847; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
848; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
849; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
850; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
851; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
852; GFX90A-TGSPLIT-NEXT:    s_endpgm
853;
854; GFX940-NOTTGSPLIT-LABEL: flat_system_seq_cst_store:
855; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
856; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
857; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
858; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
859; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
860; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
861; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
862; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
863; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
864; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
865;
866; GFX940-TGSPLIT-LABEL: flat_system_seq_cst_store:
867; GFX940-TGSPLIT:       ; %bb.0: ; %entry
868; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
869; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
870; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
871; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
872; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
873; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
874; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
875; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
876; GFX940-TGSPLIT-NEXT:    s_endpgm
877    i32 %in, i32* %out) {
878entry:
879  store atomic i32 %in, i32* %out seq_cst, align 4
880  ret void
881}
882
883define amdgpu_kernel void @flat_system_monotonic_atomicrmw(
884; GFX7-LABEL: flat_system_monotonic_atomicrmw:
885; GFX7:       ; %bb.0: ; %entry
886; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
887; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
888; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
889; GFX7-NEXT:    v_mov_b32_e32 v0, s0
890; GFX7-NEXT:    v_mov_b32_e32 v1, s1
891; GFX7-NEXT:    v_mov_b32_e32 v2, s2
892; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
893; GFX7-NEXT:    s_endpgm
894;
895; GFX10-WGP-LABEL: flat_system_monotonic_atomicrmw:
896; GFX10-WGP:       ; %bb.0: ; %entry
897; GFX10-WGP-NEXT:    s_clause 0x1
898; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
899; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
900; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
901; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
902; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
903; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
904; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
905; GFX10-WGP-NEXT:    s_endpgm
906;
907; GFX10-CU-LABEL: flat_system_monotonic_atomicrmw:
908; GFX10-CU:       ; %bb.0: ; %entry
909; GFX10-CU-NEXT:    s_clause 0x1
910; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
911; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
912; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
913; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
914; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
915; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
916; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
917; GFX10-CU-NEXT:    s_endpgm
918;
919; SKIP-CACHE-INV-LABEL: flat_system_monotonic_atomicrmw:
920; SKIP-CACHE-INV:       ; %bb.0: ; %entry
921; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
922; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
923; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
924; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
925; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
926; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
927; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
928; SKIP-CACHE-INV-NEXT:    s_endpgm
929;
930; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_atomicrmw:
931; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
932; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
933; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
934; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
935; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
936; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
937; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
938; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
939;
940; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_atomicrmw:
941; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
942; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
943; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
944; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
945; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
946; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
947; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
948; GFX90A-TGSPLIT-NEXT:    s_endpgm
949;
950; GFX940-NOTTGSPLIT-LABEL: flat_system_monotonic_atomicrmw:
951; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
952; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
953; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
954; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
955; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
956; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
957; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
958; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
959;
960; GFX940-TGSPLIT-LABEL: flat_system_monotonic_atomicrmw:
961; GFX940-TGSPLIT:       ; %bb.0: ; %entry
962; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
963; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
964; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
965; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
966; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
967; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
968; GFX940-TGSPLIT-NEXT:    s_endpgm
969    i32* %out, i32 %in) {
970entry:
971  %val = atomicrmw volatile xchg i32* %out, i32 %in monotonic
972  ret void
973}
974
975define amdgpu_kernel void @flat_system_acquire_atomicrmw(
976; GFX7-LABEL: flat_system_acquire_atomicrmw:
977; GFX7:       ; %bb.0: ; %entry
978; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
979; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
980; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
981; GFX7-NEXT:    v_mov_b32_e32 v0, s0
982; GFX7-NEXT:    v_mov_b32_e32 v1, s1
983; GFX7-NEXT:    v_mov_b32_e32 v2, s2
984; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
985; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
986; GFX7-NEXT:    buffer_wbinvl1_vol
987; GFX7-NEXT:    s_endpgm
988;
989; GFX10-WGP-LABEL: flat_system_acquire_atomicrmw:
990; GFX10-WGP:       ; %bb.0: ; %entry
991; GFX10-WGP-NEXT:    s_clause 0x1
992; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
993; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
994; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
995; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
996; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
997; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
998; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
999; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1000; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1001; GFX10-WGP-NEXT:    buffer_gl0_inv
1002; GFX10-WGP-NEXT:    buffer_gl1_inv
1003; GFX10-WGP-NEXT:    s_endpgm
1004;
1005; GFX10-CU-LABEL: flat_system_acquire_atomicrmw:
1006; GFX10-CU:       ; %bb.0: ; %entry
1007; GFX10-CU-NEXT:    s_clause 0x1
1008; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1009; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1010; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1011; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1012; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1013; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1014; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1015; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1016; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1017; GFX10-CU-NEXT:    buffer_gl0_inv
1018; GFX10-CU-NEXT:    buffer_gl1_inv
1019; GFX10-CU-NEXT:    s_endpgm
1020;
1021; SKIP-CACHE-INV-LABEL: flat_system_acquire_atomicrmw:
1022; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1023; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1024; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1025; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1026; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1027; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1028; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1029; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1030; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1031; SKIP-CACHE-INV-NEXT:    s_endpgm
1032;
1033; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_atomicrmw:
1034; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1035; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1036; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1037; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1038; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1039; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1040; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1041; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1042; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
1043; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1044; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1045;
1046; GFX90A-TGSPLIT-LABEL: flat_system_acquire_atomicrmw:
1047; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1048; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1049; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1050; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1051; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1052; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1053; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1054; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1055; GFX90A-TGSPLIT-NEXT:    buffer_invl2
1056; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1057; GFX90A-TGSPLIT-NEXT:    s_endpgm
1058;
1059; GFX940-NOTTGSPLIT-LABEL: flat_system_acquire_atomicrmw:
1060; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1061; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1062; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1063; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1064; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1065; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1066; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
1067; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1068; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
1069; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1070;
1071; GFX940-TGSPLIT-LABEL: flat_system_acquire_atomicrmw:
1072; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1073; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1074; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1075; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1076; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1077; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1078; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
1079; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1080; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
1081; GFX940-TGSPLIT-NEXT:    s_endpgm
1082    i32* %out, i32 %in) {
1083entry:
1084  %val = atomicrmw volatile xchg i32* %out, i32 %in acquire
1085  ret void
1086}
1087
1088define amdgpu_kernel void @flat_system_release_atomicrmw(
1089; GFX7-LABEL: flat_system_release_atomicrmw:
1090; GFX7:       ; %bb.0: ; %entry
1091; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1092; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1093; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1094; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1095; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1096; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1097; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1098; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1099; GFX7-NEXT:    s_endpgm
1100;
1101; GFX10-WGP-LABEL: flat_system_release_atomicrmw:
1102; GFX10-WGP:       ; %bb.0: ; %entry
1103; GFX10-WGP-NEXT:    s_clause 0x1
1104; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1105; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1106; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1107; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1108; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1109; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1110; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1111; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1112; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1113; GFX10-WGP-NEXT:    s_endpgm
1114;
1115; GFX10-CU-LABEL: flat_system_release_atomicrmw:
1116; GFX10-CU:       ; %bb.0: ; %entry
1117; GFX10-CU-NEXT:    s_clause 0x1
1118; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1119; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1120; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1121; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1122; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1123; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1124; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1125; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1126; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1127; GFX10-CU-NEXT:    s_endpgm
1128;
1129; SKIP-CACHE-INV-LABEL: flat_system_release_atomicrmw:
1130; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1131; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1132; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1133; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1134; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1135; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1136; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1137; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1138; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1139; SKIP-CACHE-INV-NEXT:    s_endpgm
1140;
1141; GFX90A-NOTTGSPLIT-LABEL: flat_system_release_atomicrmw:
1142; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1143; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1144; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1145; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1146; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1147; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1148; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
1149; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1150; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1151; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1152;
1153; GFX90A-TGSPLIT-LABEL: flat_system_release_atomicrmw:
1154; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1155; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1156; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1157; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1158; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1159; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1160; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
1161; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1162; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1163; GFX90A-TGSPLIT-NEXT:    s_endpgm
1164;
1165; GFX940-NOTTGSPLIT-LABEL: flat_system_release_atomicrmw:
1166; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1167; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1168; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1169; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1170; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1171; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1172; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
1173; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1174; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
1175; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1176;
1177; GFX940-TGSPLIT-LABEL: flat_system_release_atomicrmw:
1178; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1179; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1180; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1181; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1182; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1183; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1184; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
1185; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1186; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
1187; GFX940-TGSPLIT-NEXT:    s_endpgm
1188    i32* %out, i32 %in) {
1189entry:
1190  %val = atomicrmw volatile xchg i32* %out, i32 %in release
1191  ret void
1192}
1193
1194define amdgpu_kernel void @flat_system_acq_rel_atomicrmw(
1195; GFX7-LABEL: flat_system_acq_rel_atomicrmw:
1196; GFX7:       ; %bb.0: ; %entry
1197; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1198; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1199; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1200; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1201; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1202; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1203; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1204; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1205; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1206; GFX7-NEXT:    buffer_wbinvl1_vol
1207; GFX7-NEXT:    s_endpgm
1208;
1209; GFX10-WGP-LABEL: flat_system_acq_rel_atomicrmw:
1210; GFX10-WGP:       ; %bb.0: ; %entry
1211; GFX10-WGP-NEXT:    s_clause 0x1
1212; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1213; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1214; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1215; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1216; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1217; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1218; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1219; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1220; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1221; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1222; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1223; GFX10-WGP-NEXT:    buffer_gl0_inv
1224; GFX10-WGP-NEXT:    buffer_gl1_inv
1225; GFX10-WGP-NEXT:    s_endpgm
1226;
1227; GFX10-CU-LABEL: flat_system_acq_rel_atomicrmw:
1228; GFX10-CU:       ; %bb.0: ; %entry
1229; GFX10-CU-NEXT:    s_clause 0x1
1230; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1231; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1232; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1233; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1234; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1235; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1236; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1237; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1238; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1239; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1240; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1241; GFX10-CU-NEXT:    buffer_gl0_inv
1242; GFX10-CU-NEXT:    buffer_gl1_inv
1243; GFX10-CU-NEXT:    s_endpgm
1244;
1245; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_atomicrmw:
1246; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1247; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1248; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1249; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1250; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1251; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1252; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1253; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1254; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1255; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1256; SKIP-CACHE-INV-NEXT:    s_endpgm
1257;
1258; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_atomicrmw:
1259; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1260; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1261; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1262; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1263; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1264; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1265; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
1266; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1267; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1268; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1269; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
1270; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1271; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1272;
1273; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_atomicrmw:
1274; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1275; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1276; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1277; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1278; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1279; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1280; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
1281; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1282; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1283; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1284; GFX90A-TGSPLIT-NEXT:    buffer_invl2
1285; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1286; GFX90A-TGSPLIT-NEXT:    s_endpgm
1287;
1288; GFX940-NOTTGSPLIT-LABEL: flat_system_acq_rel_atomicrmw:
1289; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1290; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1291; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1292; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1293; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1294; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1295; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
1296; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1297; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
1298; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1299; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
1300; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1301;
1302; GFX940-TGSPLIT-LABEL: flat_system_acq_rel_atomicrmw:
1303; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1304; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1305; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1306; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1307; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1308; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1309; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
1310; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1311; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
1312; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1313; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
1314; GFX940-TGSPLIT-NEXT:    s_endpgm
1315    i32* %out, i32 %in) {
1316entry:
1317  %val = atomicrmw volatile xchg i32* %out, i32 %in acq_rel
1318  ret void
1319}
1320
1321define amdgpu_kernel void @flat_system_seq_cst_atomicrmw(
1322; GFX7-LABEL: flat_system_seq_cst_atomicrmw:
1323; GFX7:       ; %bb.0: ; %entry
1324; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1325; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1326; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1327; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1328; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1329; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1330; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1331; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1332; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1333; GFX7-NEXT:    buffer_wbinvl1_vol
1334; GFX7-NEXT:    s_endpgm
1335;
1336; GFX10-WGP-LABEL: flat_system_seq_cst_atomicrmw:
1337; GFX10-WGP:       ; %bb.0: ; %entry
1338; GFX10-WGP-NEXT:    s_clause 0x1
1339; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1340; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1341; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1342; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1343; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1344; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1345; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1346; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1347; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
1348; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1349; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1350; GFX10-WGP-NEXT:    buffer_gl0_inv
1351; GFX10-WGP-NEXT:    buffer_gl1_inv
1352; GFX10-WGP-NEXT:    s_endpgm
1353;
1354; GFX10-CU-LABEL: flat_system_seq_cst_atomicrmw:
1355; GFX10-CU:       ; %bb.0: ; %entry
1356; GFX10-CU-NEXT:    s_clause 0x1
1357; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1358; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1359; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1360; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1361; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1362; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1363; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1364; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1365; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
1366; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1367; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1368; GFX10-CU-NEXT:    buffer_gl0_inv
1369; GFX10-CU-NEXT:    buffer_gl1_inv
1370; GFX10-CU-NEXT:    s_endpgm
1371;
1372; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_atomicrmw:
1373; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1374; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1375; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1376; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1377; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1378; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1379; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1380; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1381; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
1382; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1383; SKIP-CACHE-INV-NEXT:    s_endpgm
1384;
1385; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_atomicrmw:
1386; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1387; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1388; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1389; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1390; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1391; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1392; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
1393; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1394; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1395; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1396; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
1397; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1398; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1399;
1400; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_atomicrmw:
1401; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1402; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1403; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1404; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1405; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1406; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1407; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
1408; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1409; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
1410; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1411; GFX90A-TGSPLIT-NEXT:    buffer_invl2
1412; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1413; GFX90A-TGSPLIT-NEXT:    s_endpgm
1414;
1415; GFX940-NOTTGSPLIT-LABEL: flat_system_seq_cst_atomicrmw:
1416; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1417; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1418; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1419; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1420; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1421; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1422; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
1423; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1424; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
1425; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1426; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
1427; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1428;
1429; GFX940-TGSPLIT-LABEL: flat_system_seq_cst_atomicrmw:
1430; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1431; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1432; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1433; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1434; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1435; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1436; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
1437; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1438; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
1439; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1440; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
1441; GFX940-TGSPLIT-NEXT:    s_endpgm
1442    i32* %out, i32 %in) {
1443entry:
1444  %val = atomicrmw volatile xchg i32* %out, i32 %in seq_cst
1445  ret void
1446}
1447
1448define amdgpu_kernel void @flat_system_acquire_ret_atomicrmw(
1449; GFX7-LABEL: flat_system_acquire_ret_atomicrmw:
1450; GFX7:       ; %bb.0: ; %entry
1451; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1452; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1453; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1454; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1455; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1456; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1457; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1458; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1459; GFX7-NEXT:    buffer_wbinvl1_vol
1460; GFX7-NEXT:    flat_store_dword v[0:1], v2
1461; GFX7-NEXT:    s_endpgm
1462;
1463; GFX10-WGP-LABEL: flat_system_acquire_ret_atomicrmw:
1464; GFX10-WGP:       ; %bb.0: ; %entry
1465; GFX10-WGP-NEXT:    s_clause 0x1
1466; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1467; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1468; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1469; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1470; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1471; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1472; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1473; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1474; GFX10-WGP-NEXT:    buffer_gl0_inv
1475; GFX10-WGP-NEXT:    buffer_gl1_inv
1476; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1477; GFX10-WGP-NEXT:    s_endpgm
1478;
1479; GFX10-CU-LABEL: flat_system_acquire_ret_atomicrmw:
1480; GFX10-CU:       ; %bb.0: ; %entry
1481; GFX10-CU-NEXT:    s_clause 0x1
1482; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1483; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1484; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1485; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1486; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1487; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1488; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1489; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1490; GFX10-CU-NEXT:    buffer_gl0_inv
1491; GFX10-CU-NEXT:    buffer_gl1_inv
1492; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1493; GFX10-CU-NEXT:    s_endpgm
1494;
1495; SKIP-CACHE-INV-LABEL: flat_system_acquire_ret_atomicrmw:
1496; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1497; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1498; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1499; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1500; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1501; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1502; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1503; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1504; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1505; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1506; SKIP-CACHE-INV-NEXT:    s_endpgm
1507;
1508; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_ret_atomicrmw:
1509; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1510; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1511; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1512; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1513; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1514; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1515; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1516; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1517; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
1518; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1519; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1520; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1521;
1522; GFX90A-TGSPLIT-LABEL: flat_system_acquire_ret_atomicrmw:
1523; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1524; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1525; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1526; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1527; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1528; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1529; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1530; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1531; GFX90A-TGSPLIT-NEXT:    buffer_invl2
1532; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1533; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1534; GFX90A-TGSPLIT-NEXT:    s_endpgm
1535;
1536; GFX940-NOTTGSPLIT-LABEL: flat_system_acquire_ret_atomicrmw:
1537; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1538; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1539; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1540; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1541; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1542; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1543; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0 sc1
1544; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1545; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
1546; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1547; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1548;
1549; GFX940-TGSPLIT-LABEL: flat_system_acquire_ret_atomicrmw:
1550; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1551; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1552; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1553; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1554; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1555; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1556; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0 sc1
1557; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1558; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
1559; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1560; GFX940-TGSPLIT-NEXT:    s_endpgm
1561    i32* %out, i32 %in) {
1562entry:
1563  %val = atomicrmw volatile xchg i32* %out, i32 %in acquire
1564  store i32 %val, i32* %out, align 4
1565  ret void
1566}
1567
1568define amdgpu_kernel void @flat_system_acq_rel_ret_atomicrmw(
1569; GFX7-LABEL: flat_system_acq_rel_ret_atomicrmw:
1570; GFX7:       ; %bb.0: ; %entry
1571; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1572; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1573; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1574; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1575; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1576; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1577; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1578; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1579; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1580; GFX7-NEXT:    buffer_wbinvl1_vol
1581; GFX7-NEXT:    flat_store_dword v[0:1], v2
1582; GFX7-NEXT:    s_endpgm
1583;
1584; GFX10-WGP-LABEL: flat_system_acq_rel_ret_atomicrmw:
1585; GFX10-WGP:       ; %bb.0: ; %entry
1586; GFX10-WGP-NEXT:    s_clause 0x1
1587; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1588; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1589; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1590; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1591; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1592; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1593; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1594; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1595; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1596; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1597; GFX10-WGP-NEXT:    buffer_gl0_inv
1598; GFX10-WGP-NEXT:    buffer_gl1_inv
1599; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1600; GFX10-WGP-NEXT:    s_endpgm
1601;
1602; GFX10-CU-LABEL: flat_system_acq_rel_ret_atomicrmw:
1603; GFX10-CU:       ; %bb.0: ; %entry
1604; GFX10-CU-NEXT:    s_clause 0x1
1605; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1606; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1607; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1608; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1609; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1610; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1611; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1612; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1613; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1614; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1615; GFX10-CU-NEXT:    buffer_gl0_inv
1616; GFX10-CU-NEXT:    buffer_gl1_inv
1617; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1618; GFX10-CU-NEXT:    s_endpgm
1619;
1620; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_ret_atomicrmw:
1621; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1622; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1623; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1624; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1625; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1626; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1627; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1628; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1629; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1630; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1631; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1632; SKIP-CACHE-INV-NEXT:    s_endpgm
1633;
1634; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_ret_atomicrmw:
1635; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1636; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1637; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1638; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1639; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1640; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1641; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
1642; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1643; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1644; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1645; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
1646; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1647; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1648; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1649;
1650; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_ret_atomicrmw:
1651; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1652; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1653; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1654; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1655; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1656; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1657; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
1658; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1659; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1660; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1661; GFX90A-TGSPLIT-NEXT:    buffer_invl2
1662; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1663; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1664; GFX90A-TGSPLIT-NEXT:    s_endpgm
1665;
1666; GFX940-NOTTGSPLIT-LABEL: flat_system_acq_rel_ret_atomicrmw:
1667; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1668; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1669; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1670; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1671; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1672; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1673; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
1674; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1675; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0 sc1
1676; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1677; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
1678; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1679; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1680;
1681; GFX940-TGSPLIT-LABEL: flat_system_acq_rel_ret_atomicrmw:
1682; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1683; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1684; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1685; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1686; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1687; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1688; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
1689; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1690; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0 sc1
1691; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1692; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
1693; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1694; GFX940-TGSPLIT-NEXT:    s_endpgm
1695    i32* %out, i32 %in) {
1696entry:
1697  %val = atomicrmw volatile xchg i32* %out, i32 %in acq_rel
1698  store i32 %val, i32* %out, align 4
1699  ret void
1700}
1701
1702define amdgpu_kernel void @flat_system_seq_cst_ret_atomicrmw(
1703; GFX7-LABEL: flat_system_seq_cst_ret_atomicrmw:
1704; GFX7:       ; %bb.0: ; %entry
1705; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1706; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1707; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1708; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1709; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1710; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1711; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1712; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1713; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1714; GFX7-NEXT:    buffer_wbinvl1_vol
1715; GFX7-NEXT:    flat_store_dword v[0:1], v2
1716; GFX7-NEXT:    s_endpgm
1717;
1718; GFX10-WGP-LABEL: flat_system_seq_cst_ret_atomicrmw:
1719; GFX10-WGP:       ; %bb.0: ; %entry
1720; GFX10-WGP-NEXT:    s_clause 0x1
1721; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1722; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1723; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1724; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1725; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1726; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1727; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1728; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1729; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1730; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1731; GFX10-WGP-NEXT:    buffer_gl0_inv
1732; GFX10-WGP-NEXT:    buffer_gl1_inv
1733; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
1734; GFX10-WGP-NEXT:    s_endpgm
1735;
1736; GFX10-CU-LABEL: flat_system_seq_cst_ret_atomicrmw:
1737; GFX10-CU:       ; %bb.0: ; %entry
1738; GFX10-CU-NEXT:    s_clause 0x1
1739; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1740; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1741; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1742; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1743; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1744; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1745; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1746; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1747; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1748; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1749; GFX10-CU-NEXT:    buffer_gl0_inv
1750; GFX10-CU-NEXT:    buffer_gl1_inv
1751; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
1752; GFX10-CU-NEXT:    s_endpgm
1753;
1754; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_ret_atomicrmw:
1755; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1756; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1757; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
1758; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1759; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1760; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1761; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1762; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1763; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1764; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1765; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
1766; SKIP-CACHE-INV-NEXT:    s_endpgm
1767;
1768; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_ret_atomicrmw:
1769; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1770; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1771; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1772; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1773; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1774; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1775; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
1776; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1777; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1778; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1779; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
1780; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1781; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1782; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1783;
1784; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_ret_atomicrmw:
1785; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1786; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1787; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1788; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1789; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1790; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
1791; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
1792; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1793; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1794; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1795; GFX90A-TGSPLIT-NEXT:    buffer_invl2
1796; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1797; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1798; GFX90A-TGSPLIT-NEXT:    s_endpgm
1799;
1800; GFX940-NOTTGSPLIT-LABEL: flat_system_seq_cst_ret_atomicrmw:
1801; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1802; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1803; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1804; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1805; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1806; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1807; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
1808; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1809; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0 sc1
1810; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1811; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
1812; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1813; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1814;
1815; GFX940-TGSPLIT-LABEL: flat_system_seq_cst_ret_atomicrmw:
1816; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1817; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1818; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
1819; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1820; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1821; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
1822; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
1823; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1824; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0 sc1
1825; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1826; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
1827; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
1828; GFX940-TGSPLIT-NEXT:    s_endpgm
1829    i32* %out, i32 %in) {
1830entry:
1831  %val = atomicrmw volatile xchg i32* %out, i32 %in seq_cst
1832  store i32 %val, i32* %out, align 4
1833  ret void
1834}
1835
1836define amdgpu_kernel void @flat_system_monotonic_monotonic_cmpxchg(
1837; GFX7-LABEL: flat_system_monotonic_monotonic_cmpxchg:
1838; GFX7:       ; %bb.0: ; %entry
1839; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1840; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1841; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1842; GFX7-NEXT:    s_add_u32 s0, s0, 16
1843; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1844; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1845; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1846; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1847; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1848; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1849; GFX7-NEXT:    s_endpgm
1850;
1851; GFX10-WGP-LABEL: flat_system_monotonic_monotonic_cmpxchg:
1852; GFX10-WGP:       ; %bb.0: ; %entry
1853; GFX10-WGP-NEXT:    s_clause 0x1
1854; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1855; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1856; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1857; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1858; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1859; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1860; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1861; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1862; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1863; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1864; GFX10-WGP-NEXT:    s_endpgm
1865;
1866; GFX10-CU-LABEL: flat_system_monotonic_monotonic_cmpxchg:
1867; GFX10-CU:       ; %bb.0: ; %entry
1868; GFX10-CU-NEXT:    s_clause 0x1
1869; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1870; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1871; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1872; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1873; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1874; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1875; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1876; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1877; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1878; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1879; GFX10-CU-NEXT:    s_endpgm
1880;
1881; SKIP-CACHE-INV-LABEL: flat_system_monotonic_monotonic_cmpxchg:
1882; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1883; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1884; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
1885; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1886; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
1887; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
1888; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
1889; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
1890; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
1891; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
1892; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1893; SKIP-CACHE-INV-NEXT:    s_endpgm
1894;
1895; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_monotonic_cmpxchg:
1896; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1897; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1898; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1899; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1900; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1901; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1902; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1903; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1904;
1905; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_monotonic_cmpxchg:
1906; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1907; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1908; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1909; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1910; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1911; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
1912; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
1913; GFX90A-TGSPLIT-NEXT:    s_endpgm
1914;
1915; GFX940-NOTTGSPLIT-LABEL: flat_system_monotonic_monotonic_cmpxchg:
1916; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
1917; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1918; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
1919; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1920; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1921; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
1922; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
1923; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
1924;
1925; GFX940-TGSPLIT-LABEL: flat_system_monotonic_monotonic_cmpxchg:
1926; GFX940-TGSPLIT:       ; %bb.0: ; %entry
1927; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1928; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
1929; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1930; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
1931; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
1932; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
1933; GFX940-TGSPLIT-NEXT:    s_endpgm
1934    i32* %out, i32 %in, i32 %old) {
1935entry:
1936  %gep = getelementptr i32, i32* %out, i32 4
1937  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in monotonic monotonic
1938  ret void
1939}
1940
1941define amdgpu_kernel void @flat_system_acquire_monotonic_cmpxchg(
1942; GFX7-LABEL: flat_system_acquire_monotonic_cmpxchg:
1943; GFX7:       ; %bb.0: ; %entry
1944; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1945; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1946; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1947; GFX7-NEXT:    s_add_u32 s0, s0, 16
1948; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1949; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1950; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1951; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1952; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1953; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1954; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1955; GFX7-NEXT:    buffer_wbinvl1_vol
1956; GFX7-NEXT:    s_endpgm
1957;
1958; GFX10-WGP-LABEL: flat_system_acquire_monotonic_cmpxchg:
1959; GFX10-WGP:       ; %bb.0: ; %entry
1960; GFX10-WGP-NEXT:    s_clause 0x1
1961; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1962; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1963; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1964; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
1965; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
1966; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1967; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
1968; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1969; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
1970; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1971; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1972; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1973; GFX10-WGP-NEXT:    buffer_gl0_inv
1974; GFX10-WGP-NEXT:    buffer_gl1_inv
1975; GFX10-WGP-NEXT:    s_endpgm
1976;
1977; GFX10-CU-LABEL: flat_system_acquire_monotonic_cmpxchg:
1978; GFX10-CU:       ; %bb.0: ; %entry
1979; GFX10-CU-NEXT:    s_clause 0x1
1980; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1981; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1982; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1983; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
1984; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
1985; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1986; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
1987; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1988; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
1989; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1990; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1991; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1992; GFX10-CU-NEXT:    buffer_gl0_inv
1993; GFX10-CU-NEXT:    buffer_gl1_inv
1994; GFX10-CU-NEXT:    s_endpgm
1995;
1996; SKIP-CACHE-INV-LABEL: flat_system_acquire_monotonic_cmpxchg:
1997; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1998; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
1999; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2000; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2001; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2002; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2003; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2004; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2005; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2006; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2007; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2008; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2009; SKIP-CACHE-INV-NEXT:    s_endpgm
2010;
2011; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_monotonic_cmpxchg:
2012; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2013; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2014; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2015; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2016; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2017; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2018; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2019; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2020; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
2021; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2022; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2023;
2024; GFX90A-TGSPLIT-LABEL: flat_system_acquire_monotonic_cmpxchg:
2025; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2026; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2027; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2028; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2029; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2030; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2031; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2032; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2033; GFX90A-TGSPLIT-NEXT:    buffer_invl2
2034; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2035; GFX90A-TGSPLIT-NEXT:    s_endpgm
2036;
2037; GFX940-NOTTGSPLIT-LABEL: flat_system_acquire_monotonic_cmpxchg:
2038; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2039; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2040; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2041; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2042; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2043; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2044; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
2045; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2046; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
2047; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2048;
2049; GFX940-TGSPLIT-LABEL: flat_system_acquire_monotonic_cmpxchg:
2050; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2051; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2052; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2053; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2054; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2055; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2056; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
2057; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2058; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
2059; GFX940-TGSPLIT-NEXT:    s_endpgm
2060    i32* %out, i32 %in, i32 %old) {
2061entry:
2062  %gep = getelementptr i32, i32* %out, i32 4
2063  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acquire monotonic
2064  ret void
2065}
2066
2067define amdgpu_kernel void @flat_system_release_monotonic_cmpxchg(
2068; GFX7-LABEL: flat_system_release_monotonic_cmpxchg:
2069; GFX7:       ; %bb.0: ; %entry
2070; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2071; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2072; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2073; GFX7-NEXT:    s_add_u32 s0, s0, 16
2074; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2075; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2076; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2077; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2078; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2079; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2080; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2081; GFX7-NEXT:    s_endpgm
2082;
2083; GFX10-WGP-LABEL: flat_system_release_monotonic_cmpxchg:
2084; GFX10-WGP:       ; %bb.0: ; %entry
2085; GFX10-WGP-NEXT:    s_clause 0x1
2086; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2087; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2088; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2089; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2090; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2091; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2092; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2093; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2094; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2095; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2096; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2097; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2098; GFX10-WGP-NEXT:    s_endpgm
2099;
2100; GFX10-CU-LABEL: flat_system_release_monotonic_cmpxchg:
2101; GFX10-CU:       ; %bb.0: ; %entry
2102; GFX10-CU-NEXT:    s_clause 0x1
2103; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2104; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2105; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2106; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2107; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2108; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2109; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2110; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2111; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2112; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2113; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2114; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2115; GFX10-CU-NEXT:    s_endpgm
2116;
2117; SKIP-CACHE-INV-LABEL: flat_system_release_monotonic_cmpxchg:
2118; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2119; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2120; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2121; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2122; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2123; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2124; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2125; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2126; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2127; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2128; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2129; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2130; SKIP-CACHE-INV-NEXT:    s_endpgm
2131;
2132; GFX90A-NOTTGSPLIT-LABEL: flat_system_release_monotonic_cmpxchg:
2133; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2134; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2135; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2136; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2137; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2138; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2139; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
2140; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2141; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2142; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2143;
2144; GFX90A-TGSPLIT-LABEL: flat_system_release_monotonic_cmpxchg:
2145; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2146; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2147; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2148; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2149; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2150; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2151; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
2152; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2153; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2154; GFX90A-TGSPLIT-NEXT:    s_endpgm
2155;
2156; GFX940-NOTTGSPLIT-LABEL: flat_system_release_monotonic_cmpxchg:
2157; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2158; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2159; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2160; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2161; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2162; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2163; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
2164; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2165; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
2166; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2167;
2168; GFX940-TGSPLIT-LABEL: flat_system_release_monotonic_cmpxchg:
2169; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2170; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2171; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2172; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2173; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2174; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2175; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
2176; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2177; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
2178; GFX940-TGSPLIT-NEXT:    s_endpgm
2179    i32* %out, i32 %in, i32 %old) {
2180entry:
2181  %gep = getelementptr i32, i32* %out, i32 4
2182  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in release monotonic
2183  ret void
2184}
2185
2186define amdgpu_kernel void @flat_system_acq_rel_monotonic_cmpxchg(
2187; GFX7-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
2188; GFX7:       ; %bb.0: ; %entry
2189; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2190; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2191; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2192; GFX7-NEXT:    s_add_u32 s0, s0, 16
2193; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2194; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2195; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2196; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2197; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2198; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2199; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2200; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2201; GFX7-NEXT:    buffer_wbinvl1_vol
2202; GFX7-NEXT:    s_endpgm
2203;
2204; GFX10-WGP-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
2205; GFX10-WGP:       ; %bb.0: ; %entry
2206; GFX10-WGP-NEXT:    s_clause 0x1
2207; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2208; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2209; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2210; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2211; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2212; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2213; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2214; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2215; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2216; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2217; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2218; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2219; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2220; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2221; GFX10-WGP-NEXT:    buffer_gl0_inv
2222; GFX10-WGP-NEXT:    buffer_gl1_inv
2223; GFX10-WGP-NEXT:    s_endpgm
2224;
2225; GFX10-CU-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
2226; GFX10-CU:       ; %bb.0: ; %entry
2227; GFX10-CU-NEXT:    s_clause 0x1
2228; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2229; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2230; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2231; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2232; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2233; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2234; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2235; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2236; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2237; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2238; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2239; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2240; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2241; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2242; GFX10-CU-NEXT:    buffer_gl0_inv
2243; GFX10-CU-NEXT:    buffer_gl1_inv
2244; GFX10-CU-NEXT:    s_endpgm
2245;
2246; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
2247; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2248; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2249; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2250; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2251; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2252; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2253; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2254; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2255; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2256; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2257; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2258; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2259; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2260; SKIP-CACHE-INV-NEXT:    s_endpgm
2261;
2262; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
2263; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2264; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2265; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2266; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2267; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2268; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2269; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
2270; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2271; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2272; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2273; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
2274; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2275; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2276;
2277; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
2278; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2279; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2280; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2281; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2282; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2283; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2284; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
2285; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2286; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2287; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2288; GFX90A-TGSPLIT-NEXT:    buffer_invl2
2289; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2290; GFX90A-TGSPLIT-NEXT:    s_endpgm
2291;
2292; GFX940-NOTTGSPLIT-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
2293; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2294; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2295; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2296; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2297; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2298; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2299; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
2300; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2301; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
2302; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2303; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
2304; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2305;
2306; GFX940-TGSPLIT-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
2307; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2308; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2309; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2310; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2311; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2312; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2313; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
2314; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2315; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
2316; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2317; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
2318; GFX940-TGSPLIT-NEXT:    s_endpgm
2319    i32* %out, i32 %in, i32 %old) {
2320entry:
2321  %gep = getelementptr i32, i32* %out, i32 4
2322  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acq_rel monotonic
2323  ret void
2324}
2325
2326define amdgpu_kernel void @flat_system_seq_cst_monotonic_cmpxchg(
2327; GFX7-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
2328; GFX7:       ; %bb.0: ; %entry
2329; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2330; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2331; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2332; GFX7-NEXT:    s_add_u32 s0, s0, 16
2333; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2334; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2335; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2336; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2337; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2338; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2339; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2340; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2341; GFX7-NEXT:    buffer_wbinvl1_vol
2342; GFX7-NEXT:    s_endpgm
2343;
2344; GFX10-WGP-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
2345; GFX10-WGP:       ; %bb.0: ; %entry
2346; GFX10-WGP-NEXT:    s_clause 0x1
2347; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2348; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2349; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2350; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2351; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2352; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2353; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2354; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2355; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2356; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2357; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2358; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2359; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2360; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2361; GFX10-WGP-NEXT:    buffer_gl0_inv
2362; GFX10-WGP-NEXT:    buffer_gl1_inv
2363; GFX10-WGP-NEXT:    s_endpgm
2364;
2365; GFX10-CU-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
2366; GFX10-CU:       ; %bb.0: ; %entry
2367; GFX10-CU-NEXT:    s_clause 0x1
2368; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2369; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2370; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2371; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2372; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2373; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2374; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2375; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2376; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2377; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2378; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2379; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2380; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2381; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2382; GFX10-CU-NEXT:    buffer_gl0_inv
2383; GFX10-CU-NEXT:    buffer_gl1_inv
2384; GFX10-CU-NEXT:    s_endpgm
2385;
2386; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
2387; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2388; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2389; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2390; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2391; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2392; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2393; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2394; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2395; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2396; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2397; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2398; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2399; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2400; SKIP-CACHE-INV-NEXT:    s_endpgm
2401;
2402; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
2403; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2404; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2405; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2406; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2407; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2408; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2409; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
2410; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2411; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2412; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2413; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
2414; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2415; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2416;
2417; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
2418; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2419; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2420; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2421; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2422; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2423; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2424; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
2425; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2426; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2427; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2428; GFX90A-TGSPLIT-NEXT:    buffer_invl2
2429; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2430; GFX90A-TGSPLIT-NEXT:    s_endpgm
2431;
2432; GFX940-NOTTGSPLIT-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
2433; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2434; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2435; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2436; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2437; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2438; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2439; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
2440; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2441; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
2442; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2443; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
2444; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2445;
2446; GFX940-TGSPLIT-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
2447; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2448; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2449; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2450; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2451; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2452; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2453; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
2454; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2455; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
2456; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2457; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
2458; GFX940-TGSPLIT-NEXT:    s_endpgm
2459    i32* %out, i32 %in, i32 %old) {
2460entry:
2461  %gep = getelementptr i32, i32* %out, i32 4
2462  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst monotonic
2463  ret void
2464}
2465
2466define amdgpu_kernel void @flat_system_monotonic_acquire_cmpxchg(
2467; GFX7-LABEL: flat_system_monotonic_acquire_cmpxchg:
2468; GFX7:       ; %bb.0: ; %entry
2469; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2470; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2471; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2472; GFX7-NEXT:    s_add_u32 s0, s0, 16
2473; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2474; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2475; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2476; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2477; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2478; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2479; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2480; GFX7-NEXT:    buffer_wbinvl1_vol
2481; GFX7-NEXT:    s_endpgm
2482;
2483; GFX10-WGP-LABEL: flat_system_monotonic_acquire_cmpxchg:
2484; GFX10-WGP:       ; %bb.0: ; %entry
2485; GFX10-WGP-NEXT:    s_clause 0x1
2486; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2487; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2488; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2489; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2490; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2491; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2492; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2493; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2494; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2495; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2496; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2497; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2498; GFX10-WGP-NEXT:    buffer_gl0_inv
2499; GFX10-WGP-NEXT:    buffer_gl1_inv
2500; GFX10-WGP-NEXT:    s_endpgm
2501;
2502; GFX10-CU-LABEL: flat_system_monotonic_acquire_cmpxchg:
2503; GFX10-CU:       ; %bb.0: ; %entry
2504; GFX10-CU-NEXT:    s_clause 0x1
2505; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2506; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2507; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2508; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2509; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2510; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2511; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2512; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2513; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2514; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2515; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2516; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2517; GFX10-CU-NEXT:    buffer_gl0_inv
2518; GFX10-CU-NEXT:    buffer_gl1_inv
2519; GFX10-CU-NEXT:    s_endpgm
2520;
2521; SKIP-CACHE-INV-LABEL: flat_system_monotonic_acquire_cmpxchg:
2522; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2523; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2524; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2525; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2526; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2527; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2528; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2529; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2530; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2531; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2532; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2533; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2534; SKIP-CACHE-INV-NEXT:    s_endpgm
2535;
2536; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_acquire_cmpxchg:
2537; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2538; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2539; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2540; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2541; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2542; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2543; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2544; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2545; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
2546; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2547; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2548;
2549; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_acquire_cmpxchg:
2550; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2551; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2552; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2553; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2554; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2555; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2556; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2557; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2558; GFX90A-TGSPLIT-NEXT:    buffer_invl2
2559; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2560; GFX90A-TGSPLIT-NEXT:    s_endpgm
2561;
2562; GFX940-NOTTGSPLIT-LABEL: flat_system_monotonic_acquire_cmpxchg:
2563; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2564; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2565; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2566; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2567; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2568; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2569; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
2570; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2571; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
2572; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2573;
2574; GFX940-TGSPLIT-LABEL: flat_system_monotonic_acquire_cmpxchg:
2575; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2576; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2577; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2578; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2579; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2580; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2581; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
2582; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2583; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
2584; GFX940-TGSPLIT-NEXT:    s_endpgm
2585    i32* %out, i32 %in, i32 %old) {
2586entry:
2587  %gep = getelementptr i32, i32* %out, i32 4
2588  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in monotonic acquire
2589  ret void
2590}
2591
2592define amdgpu_kernel void @flat_system_acquire_acquire_cmpxchg(
2593; GFX7-LABEL: flat_system_acquire_acquire_cmpxchg:
2594; GFX7:       ; %bb.0: ; %entry
2595; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2596; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2597; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2598; GFX7-NEXT:    s_add_u32 s0, s0, 16
2599; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2600; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2601; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2602; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2603; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2604; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2605; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2606; GFX7-NEXT:    buffer_wbinvl1_vol
2607; GFX7-NEXT:    s_endpgm
2608;
2609; GFX10-WGP-LABEL: flat_system_acquire_acquire_cmpxchg:
2610; GFX10-WGP:       ; %bb.0: ; %entry
2611; GFX10-WGP-NEXT:    s_clause 0x1
2612; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2613; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2614; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2615; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2616; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2617; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2618; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2619; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2620; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2621; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2622; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2623; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2624; GFX10-WGP-NEXT:    buffer_gl0_inv
2625; GFX10-WGP-NEXT:    buffer_gl1_inv
2626; GFX10-WGP-NEXT:    s_endpgm
2627;
2628; GFX10-CU-LABEL: flat_system_acquire_acquire_cmpxchg:
2629; GFX10-CU:       ; %bb.0: ; %entry
2630; GFX10-CU-NEXT:    s_clause 0x1
2631; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2632; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2633; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2634; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2635; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2636; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2637; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2638; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2639; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2640; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2641; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2642; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2643; GFX10-CU-NEXT:    buffer_gl0_inv
2644; GFX10-CU-NEXT:    buffer_gl1_inv
2645; GFX10-CU-NEXT:    s_endpgm
2646;
2647; SKIP-CACHE-INV-LABEL: flat_system_acquire_acquire_cmpxchg:
2648; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2649; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2650; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2651; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2652; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2653; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2654; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2655; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2656; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2657; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2658; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2659; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2660; SKIP-CACHE-INV-NEXT:    s_endpgm
2661;
2662; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_acquire_cmpxchg:
2663; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2664; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2665; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2666; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2667; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2668; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2669; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2670; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2671; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
2672; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2673; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2674;
2675; GFX90A-TGSPLIT-LABEL: flat_system_acquire_acquire_cmpxchg:
2676; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2677; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2678; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2679; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2680; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2681; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2682; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2683; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2684; GFX90A-TGSPLIT-NEXT:    buffer_invl2
2685; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2686; GFX90A-TGSPLIT-NEXT:    s_endpgm
2687;
2688; GFX940-NOTTGSPLIT-LABEL: flat_system_acquire_acquire_cmpxchg:
2689; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2690; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2691; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2692; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2693; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2694; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2695; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
2696; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2697; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
2698; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2699;
2700; GFX940-TGSPLIT-LABEL: flat_system_acquire_acquire_cmpxchg:
2701; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2702; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2703; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2704; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2705; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2706; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2707; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
2708; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2709; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
2710; GFX940-TGSPLIT-NEXT:    s_endpgm
2711    i32* %out, i32 %in, i32 %old) {
2712entry:
2713  %gep = getelementptr i32, i32* %out, i32 4
2714  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acquire acquire
2715  ret void
2716}
2717
2718define amdgpu_kernel void @flat_system_release_acquire_cmpxchg(
2719; GFX7-LABEL: flat_system_release_acquire_cmpxchg:
2720; GFX7:       ; %bb.0: ; %entry
2721; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2722; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2723; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2724; GFX7-NEXT:    s_add_u32 s0, s0, 16
2725; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2726; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2727; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2728; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2729; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2730; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2731; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2732; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2733; GFX7-NEXT:    buffer_wbinvl1_vol
2734; GFX7-NEXT:    s_endpgm
2735;
2736; GFX10-WGP-LABEL: flat_system_release_acquire_cmpxchg:
2737; GFX10-WGP:       ; %bb.0: ; %entry
2738; GFX10-WGP-NEXT:    s_clause 0x1
2739; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2740; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2741; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2742; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2743; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2744; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2745; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2746; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2747; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2748; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2749; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2750; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2751; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2752; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2753; GFX10-WGP-NEXT:    buffer_gl0_inv
2754; GFX10-WGP-NEXT:    buffer_gl1_inv
2755; GFX10-WGP-NEXT:    s_endpgm
2756;
2757; GFX10-CU-LABEL: flat_system_release_acquire_cmpxchg:
2758; GFX10-CU:       ; %bb.0: ; %entry
2759; GFX10-CU-NEXT:    s_clause 0x1
2760; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2761; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2762; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2763; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2764; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2765; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2766; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2767; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2768; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2769; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2770; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2771; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2772; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2773; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2774; GFX10-CU-NEXT:    buffer_gl0_inv
2775; GFX10-CU-NEXT:    buffer_gl1_inv
2776; GFX10-CU-NEXT:    s_endpgm
2777;
2778; SKIP-CACHE-INV-LABEL: flat_system_release_acquire_cmpxchg:
2779; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2780; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2781; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2782; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2783; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2784; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2785; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2786; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2787; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2788; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2789; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2790; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2791; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2792; SKIP-CACHE-INV-NEXT:    s_endpgm
2793;
2794; GFX90A-NOTTGSPLIT-LABEL: flat_system_release_acquire_cmpxchg:
2795; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2796; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2797; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2798; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2799; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2800; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2801; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
2802; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2803; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2804; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2805; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
2806; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2807; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2808;
2809; GFX90A-TGSPLIT-LABEL: flat_system_release_acquire_cmpxchg:
2810; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2811; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2812; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2813; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2814; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2815; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2816; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
2817; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2818; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2819; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2820; GFX90A-TGSPLIT-NEXT:    buffer_invl2
2821; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2822; GFX90A-TGSPLIT-NEXT:    s_endpgm
2823;
2824; GFX940-NOTTGSPLIT-LABEL: flat_system_release_acquire_cmpxchg:
2825; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2826; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2827; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2828; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2829; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2830; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2831; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
2832; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2833; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
2834; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2835; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
2836; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2837;
2838; GFX940-TGSPLIT-LABEL: flat_system_release_acquire_cmpxchg:
2839; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2840; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2841; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2842; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2843; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2844; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2845; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
2846; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2847; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
2848; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2849; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
2850; GFX940-TGSPLIT-NEXT:    s_endpgm
2851    i32* %out, i32 %in, i32 %old) {
2852entry:
2853  %gep = getelementptr i32, i32* %out, i32 4
2854  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in release acquire
2855  ret void
2856}
2857
2858define amdgpu_kernel void @flat_system_acq_rel_acquire_cmpxchg(
2859; GFX7-LABEL: flat_system_acq_rel_acquire_cmpxchg:
2860; GFX7:       ; %bb.0: ; %entry
2861; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2862; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2863; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2864; GFX7-NEXT:    s_add_u32 s0, s0, 16
2865; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2866; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2867; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2868; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2869; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2870; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2871; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2872; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2873; GFX7-NEXT:    buffer_wbinvl1_vol
2874; GFX7-NEXT:    s_endpgm
2875;
2876; GFX10-WGP-LABEL: flat_system_acq_rel_acquire_cmpxchg:
2877; GFX10-WGP:       ; %bb.0: ; %entry
2878; GFX10-WGP-NEXT:    s_clause 0x1
2879; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2880; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2881; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2882; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
2883; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
2884; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2885; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
2886; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2887; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
2888; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2889; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2890; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2891; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2892; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2893; GFX10-WGP-NEXT:    buffer_gl0_inv
2894; GFX10-WGP-NEXT:    buffer_gl1_inv
2895; GFX10-WGP-NEXT:    s_endpgm
2896;
2897; GFX10-CU-LABEL: flat_system_acq_rel_acquire_cmpxchg:
2898; GFX10-CU:       ; %bb.0: ; %entry
2899; GFX10-CU-NEXT:    s_clause 0x1
2900; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2901; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2902; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2903; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
2904; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
2905; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2906; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
2907; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2908; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
2909; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2910; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2911; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2912; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2913; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2914; GFX10-CU-NEXT:    buffer_gl0_inv
2915; GFX10-CU-NEXT:    buffer_gl1_inv
2916; GFX10-CU-NEXT:    s_endpgm
2917;
2918; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_acquire_cmpxchg:
2919; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2920; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2921; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
2922; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2923; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
2924; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
2925; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
2926; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
2927; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
2928; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
2929; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2930; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2931; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2932; SKIP-CACHE-INV-NEXT:    s_endpgm
2933;
2934; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_acquire_cmpxchg:
2935; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2936; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2937; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2938; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2939; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2940; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2941; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
2942; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2943; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2944; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2945; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
2946; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2947; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2948;
2949; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_acquire_cmpxchg:
2950; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2951; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2952; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2953; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2954; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2955; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2956; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
2957; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2958; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2959; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2960; GFX90A-TGSPLIT-NEXT:    buffer_invl2
2961; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2962; GFX90A-TGSPLIT-NEXT:    s_endpgm
2963;
2964; GFX940-NOTTGSPLIT-LABEL: flat_system_acq_rel_acquire_cmpxchg:
2965; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
2966; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2967; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2968; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2969; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2970; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2971; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
2972; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2973; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
2974; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2975; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
2976; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
2977;
2978; GFX940-TGSPLIT-LABEL: flat_system_acq_rel_acquire_cmpxchg:
2979; GFX940-TGSPLIT:       ; %bb.0: ; %entry
2980; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
2981; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
2982; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2983; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
2984; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
2985; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
2986; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2987; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
2988; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2989; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
2990; GFX940-TGSPLIT-NEXT:    s_endpgm
2991    i32* %out, i32 %in, i32 %old) {
2992entry:
2993  %gep = getelementptr i32, i32* %out, i32 4
2994  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acq_rel acquire
2995  ret void
2996}
2997
2998define amdgpu_kernel void @flat_system_seq_cst_acquire_cmpxchg(
2999; GFX7-LABEL: flat_system_seq_cst_acquire_cmpxchg:
3000; GFX7:       ; %bb.0: ; %entry
3001; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3002; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3003; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3004; GFX7-NEXT:    s_add_u32 s0, s0, 16
3005; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3006; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3007; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3008; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3009; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3010; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3011; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3012; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3013; GFX7-NEXT:    buffer_wbinvl1_vol
3014; GFX7-NEXT:    s_endpgm
3015;
3016; GFX10-WGP-LABEL: flat_system_seq_cst_acquire_cmpxchg:
3017; GFX10-WGP:       ; %bb.0: ; %entry
3018; GFX10-WGP-NEXT:    s_clause 0x1
3019; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3020; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3021; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3022; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3023; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3024; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3025; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3026; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3027; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3028; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3029; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3030; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3031; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3032; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3033; GFX10-WGP-NEXT:    buffer_gl0_inv
3034; GFX10-WGP-NEXT:    buffer_gl1_inv
3035; GFX10-WGP-NEXT:    s_endpgm
3036;
3037; GFX10-CU-LABEL: flat_system_seq_cst_acquire_cmpxchg:
3038; GFX10-CU:       ; %bb.0: ; %entry
3039; GFX10-CU-NEXT:    s_clause 0x1
3040; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3041; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3042; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3043; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3044; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3045; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3046; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3047; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3048; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3049; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3050; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3051; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3052; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3053; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3054; GFX10-CU-NEXT:    buffer_gl0_inv
3055; GFX10-CU-NEXT:    buffer_gl1_inv
3056; GFX10-CU-NEXT:    s_endpgm
3057;
3058; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_acquire_cmpxchg:
3059; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3060; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3061; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3062; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3063; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3064; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3065; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3066; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3067; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3068; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3069; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3070; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3071; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3072; SKIP-CACHE-INV-NEXT:    s_endpgm
3073;
3074; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_acquire_cmpxchg:
3075; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3076; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3077; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3078; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3079; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3080; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3081; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
3082; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3083; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3084; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3085; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
3086; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3087; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3088;
3089; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_acquire_cmpxchg:
3090; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3091; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3092; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3093; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3094; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3095; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3096; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
3097; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3098; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3099; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3100; GFX90A-TGSPLIT-NEXT:    buffer_invl2
3101; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3102; GFX90A-TGSPLIT-NEXT:    s_endpgm
3103;
3104; GFX940-NOTTGSPLIT-LABEL: flat_system_seq_cst_acquire_cmpxchg:
3105; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3106; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3107; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3108; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3109; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3110; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3111; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
3112; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3113; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
3114; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3115; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
3116; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3117;
3118; GFX940-TGSPLIT-LABEL: flat_system_seq_cst_acquire_cmpxchg:
3119; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3120; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3121; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3122; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3123; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3124; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3125; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
3126; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3127; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
3128; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3129; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
3130; GFX940-TGSPLIT-NEXT:    s_endpgm
3131    i32* %out, i32 %in, i32 %old) {
3132entry:
3133  %gep = getelementptr i32, i32* %out, i32 4
3134  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst acquire
3135  ret void
3136}
3137
3138define amdgpu_kernel void @flat_system_monotonic_seq_cst_cmpxchg(
3139; GFX7-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
3140; GFX7:       ; %bb.0: ; %entry
3141; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3142; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3143; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3144; GFX7-NEXT:    s_add_u32 s0, s0, 16
3145; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3146; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3147; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3148; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3149; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3150; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3151; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3152; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3153; GFX7-NEXT:    buffer_wbinvl1_vol
3154; GFX7-NEXT:    s_endpgm
3155;
3156; GFX10-WGP-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
3157; GFX10-WGP:       ; %bb.0: ; %entry
3158; GFX10-WGP-NEXT:    s_clause 0x1
3159; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3160; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3161; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3162; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3163; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3164; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3165; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3166; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3167; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3168; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3169; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3170; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3171; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3172; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3173; GFX10-WGP-NEXT:    buffer_gl0_inv
3174; GFX10-WGP-NEXT:    buffer_gl1_inv
3175; GFX10-WGP-NEXT:    s_endpgm
3176;
3177; GFX10-CU-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
3178; GFX10-CU:       ; %bb.0: ; %entry
3179; GFX10-CU-NEXT:    s_clause 0x1
3180; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3181; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3182; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3183; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3184; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3185; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3186; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3187; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3188; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3189; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3190; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3191; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3192; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3193; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3194; GFX10-CU-NEXT:    buffer_gl0_inv
3195; GFX10-CU-NEXT:    buffer_gl1_inv
3196; GFX10-CU-NEXT:    s_endpgm
3197;
3198; SKIP-CACHE-INV-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
3199; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3200; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3201; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3202; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3203; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3204; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3205; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3206; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3207; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3208; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3209; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3210; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3211; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3212; SKIP-CACHE-INV-NEXT:    s_endpgm
3213;
3214; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
3215; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3216; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3217; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3218; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3219; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3220; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3221; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
3222; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3223; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3224; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3225; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
3226; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3227; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3228;
3229; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
3230; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3231; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3232; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3233; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3234; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3235; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3236; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
3237; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3238; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3239; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3240; GFX90A-TGSPLIT-NEXT:    buffer_invl2
3241; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3242; GFX90A-TGSPLIT-NEXT:    s_endpgm
3243;
3244; GFX940-NOTTGSPLIT-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
3245; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3246; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3247; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3248; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3249; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3250; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3251; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
3252; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3253; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
3254; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3255; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
3256; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3257;
3258; GFX940-TGSPLIT-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
3259; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3260; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3261; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3262; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3263; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3264; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3265; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
3266; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3267; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
3268; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3269; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
3270; GFX940-TGSPLIT-NEXT:    s_endpgm
3271    i32* %out, i32 %in, i32 %old) {
3272entry:
3273  %gep = getelementptr i32, i32* %out, i32 4
3274  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in monotonic seq_cst
3275  ret void
3276}
3277
3278define amdgpu_kernel void @flat_system_acquire_seq_cst_cmpxchg(
3279; GFX7-LABEL: flat_system_acquire_seq_cst_cmpxchg:
3280; GFX7:       ; %bb.0: ; %entry
3281; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3282; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3283; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3284; GFX7-NEXT:    s_add_u32 s0, s0, 16
3285; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3286; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3287; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3288; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3289; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3290; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3291; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3292; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3293; GFX7-NEXT:    buffer_wbinvl1_vol
3294; GFX7-NEXT:    s_endpgm
3295;
3296; GFX10-WGP-LABEL: flat_system_acquire_seq_cst_cmpxchg:
3297; GFX10-WGP:       ; %bb.0: ; %entry
3298; GFX10-WGP-NEXT:    s_clause 0x1
3299; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3300; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3301; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3302; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3303; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3304; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3305; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3306; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3307; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3308; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3309; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3310; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3311; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3312; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3313; GFX10-WGP-NEXT:    buffer_gl0_inv
3314; GFX10-WGP-NEXT:    buffer_gl1_inv
3315; GFX10-WGP-NEXT:    s_endpgm
3316;
3317; GFX10-CU-LABEL: flat_system_acquire_seq_cst_cmpxchg:
3318; GFX10-CU:       ; %bb.0: ; %entry
3319; GFX10-CU-NEXT:    s_clause 0x1
3320; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3321; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3322; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3323; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3324; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3325; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3326; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3327; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3328; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3329; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3330; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3331; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3332; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3333; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3334; GFX10-CU-NEXT:    buffer_gl0_inv
3335; GFX10-CU-NEXT:    buffer_gl1_inv
3336; GFX10-CU-NEXT:    s_endpgm
3337;
3338; SKIP-CACHE-INV-LABEL: flat_system_acquire_seq_cst_cmpxchg:
3339; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3340; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3341; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3342; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3343; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3344; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3345; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3346; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3347; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3348; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3349; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3350; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3351; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3352; SKIP-CACHE-INV-NEXT:    s_endpgm
3353;
3354; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_seq_cst_cmpxchg:
3355; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3356; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3357; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3358; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3359; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3360; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3361; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
3362; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3363; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3364; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3365; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
3366; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3367; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3368;
3369; GFX90A-TGSPLIT-LABEL: flat_system_acquire_seq_cst_cmpxchg:
3370; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3371; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3372; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3373; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3374; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3375; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3376; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
3377; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3378; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3379; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3380; GFX90A-TGSPLIT-NEXT:    buffer_invl2
3381; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3382; GFX90A-TGSPLIT-NEXT:    s_endpgm
3383;
3384; GFX940-NOTTGSPLIT-LABEL: flat_system_acquire_seq_cst_cmpxchg:
3385; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3386; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3387; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3388; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3389; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3390; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3391; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
3392; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3393; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
3394; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3395; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
3396; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3397;
3398; GFX940-TGSPLIT-LABEL: flat_system_acquire_seq_cst_cmpxchg:
3399; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3400; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3401; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3402; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3403; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3404; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3405; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
3406; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3407; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
3408; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3409; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
3410; GFX940-TGSPLIT-NEXT:    s_endpgm
3411    i32* %out, i32 %in, i32 %old) {
3412entry:
3413  %gep = getelementptr i32, i32* %out, i32 4
3414  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acquire seq_cst
3415  ret void
3416}
3417
3418define amdgpu_kernel void @flat_system_release_seq_cst_cmpxchg(
3419; GFX7-LABEL: flat_system_release_seq_cst_cmpxchg:
3420; GFX7:       ; %bb.0: ; %entry
3421; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3422; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3423; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3424; GFX7-NEXT:    s_add_u32 s0, s0, 16
3425; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3426; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3427; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3428; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3429; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3430; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3431; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3432; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3433; GFX7-NEXT:    buffer_wbinvl1_vol
3434; GFX7-NEXT:    s_endpgm
3435;
3436; GFX10-WGP-LABEL: flat_system_release_seq_cst_cmpxchg:
3437; GFX10-WGP:       ; %bb.0: ; %entry
3438; GFX10-WGP-NEXT:    s_clause 0x1
3439; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3440; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3441; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3442; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3443; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3444; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3445; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3446; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3447; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3448; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3449; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3450; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3451; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3452; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3453; GFX10-WGP-NEXT:    buffer_gl0_inv
3454; GFX10-WGP-NEXT:    buffer_gl1_inv
3455; GFX10-WGP-NEXT:    s_endpgm
3456;
3457; GFX10-CU-LABEL: flat_system_release_seq_cst_cmpxchg:
3458; GFX10-CU:       ; %bb.0: ; %entry
3459; GFX10-CU-NEXT:    s_clause 0x1
3460; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3461; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3462; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3463; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3464; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3465; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3466; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3467; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3468; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3469; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3470; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3471; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3472; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3473; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3474; GFX10-CU-NEXT:    buffer_gl0_inv
3475; GFX10-CU-NEXT:    buffer_gl1_inv
3476; GFX10-CU-NEXT:    s_endpgm
3477;
3478; SKIP-CACHE-INV-LABEL: flat_system_release_seq_cst_cmpxchg:
3479; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3480; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3481; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3482; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3483; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3484; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3485; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3486; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3487; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3488; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3489; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3490; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3491; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3492; SKIP-CACHE-INV-NEXT:    s_endpgm
3493;
3494; GFX90A-NOTTGSPLIT-LABEL: flat_system_release_seq_cst_cmpxchg:
3495; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3496; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3497; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3498; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3499; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3500; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3501; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
3502; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3503; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3504; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3505; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
3506; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3507; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3508;
3509; GFX90A-TGSPLIT-LABEL: flat_system_release_seq_cst_cmpxchg:
3510; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3511; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3512; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3513; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3514; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3515; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3516; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
3517; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3518; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3519; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3520; GFX90A-TGSPLIT-NEXT:    buffer_invl2
3521; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3522; GFX90A-TGSPLIT-NEXT:    s_endpgm
3523;
3524; GFX940-NOTTGSPLIT-LABEL: flat_system_release_seq_cst_cmpxchg:
3525; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3526; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3527; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3528; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3529; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3530; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3531; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
3532; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3533; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
3534; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3535; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
3536; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3537;
3538; GFX940-TGSPLIT-LABEL: flat_system_release_seq_cst_cmpxchg:
3539; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3540; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3541; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3542; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3543; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3544; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3545; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
3546; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3547; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
3548; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3549; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
3550; GFX940-TGSPLIT-NEXT:    s_endpgm
3551    i32* %out, i32 %in, i32 %old) {
3552entry:
3553  %gep = getelementptr i32, i32* %out, i32 4
3554  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in release seq_cst
3555  ret void
3556}
3557
3558define amdgpu_kernel void @flat_system_acq_rel_seq_cst_cmpxchg(
3559; GFX7-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
3560; GFX7:       ; %bb.0: ; %entry
3561; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3562; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3563; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3564; GFX7-NEXT:    s_add_u32 s0, s0, 16
3565; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3566; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3567; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3568; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3569; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3570; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3571; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3572; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3573; GFX7-NEXT:    buffer_wbinvl1_vol
3574; GFX7-NEXT:    s_endpgm
3575;
3576; GFX10-WGP-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
3577; GFX10-WGP:       ; %bb.0: ; %entry
3578; GFX10-WGP-NEXT:    s_clause 0x1
3579; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3580; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3581; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3582; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3583; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3584; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3585; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3586; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3587; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3588; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3589; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3590; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3591; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3592; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3593; GFX10-WGP-NEXT:    buffer_gl0_inv
3594; GFX10-WGP-NEXT:    buffer_gl1_inv
3595; GFX10-WGP-NEXT:    s_endpgm
3596;
3597; GFX10-CU-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
3598; GFX10-CU:       ; %bb.0: ; %entry
3599; GFX10-CU-NEXT:    s_clause 0x1
3600; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3601; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3602; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3603; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3604; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3605; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3606; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3607; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3608; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3609; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3610; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3611; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3612; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3613; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3614; GFX10-CU-NEXT:    buffer_gl0_inv
3615; GFX10-CU-NEXT:    buffer_gl1_inv
3616; GFX10-CU-NEXT:    s_endpgm
3617;
3618; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
3619; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3620; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3621; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3622; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3623; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3624; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3625; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3626; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3627; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3628; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3629; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3630; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3631; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3632; SKIP-CACHE-INV-NEXT:    s_endpgm
3633;
3634; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
3635; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3636; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3637; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3638; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3639; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3640; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3641; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
3642; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3643; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3644; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3645; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
3646; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3647; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3648;
3649; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
3650; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3651; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3652; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3653; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3654; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3655; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3656; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
3657; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3658; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3659; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3660; GFX90A-TGSPLIT-NEXT:    buffer_invl2
3661; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3662; GFX90A-TGSPLIT-NEXT:    s_endpgm
3663;
3664; GFX940-NOTTGSPLIT-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
3665; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3666; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3667; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3668; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3669; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3670; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3671; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
3672; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3673; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
3674; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3675; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
3676; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3677;
3678; GFX940-TGSPLIT-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
3679; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3680; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3681; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3682; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3683; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3684; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3685; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
3686; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3687; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
3688; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3689; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
3690; GFX940-TGSPLIT-NEXT:    s_endpgm
3691    i32* %out, i32 %in, i32 %old) {
3692entry:
3693  %gep = getelementptr i32, i32* %out, i32 4
3694  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acq_rel seq_cst
3695  ret void
3696}
3697
3698define amdgpu_kernel void @flat_system_seq_cst_seq_cst_cmpxchg(
3699; GFX7-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
3700; GFX7:       ; %bb.0: ; %entry
3701; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3702; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3703; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3704; GFX7-NEXT:    s_add_u32 s0, s0, 16
3705; GFX7-NEXT:    s_addc_u32 s1, s1, 0
3706; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3707; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3708; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3709; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3710; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3711; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3712; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3713; GFX7-NEXT:    buffer_wbinvl1_vol
3714; GFX7-NEXT:    s_endpgm
3715;
3716; GFX10-WGP-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
3717; GFX10-WGP:       ; %bb.0: ; %entry
3718; GFX10-WGP-NEXT:    s_clause 0x1
3719; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3720; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3721; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3722; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
3723; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
3724; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3725; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3726; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3727; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3728; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3729; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3730; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3731; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3732; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3733; GFX10-WGP-NEXT:    buffer_gl0_inv
3734; GFX10-WGP-NEXT:    buffer_gl1_inv
3735; GFX10-WGP-NEXT:    s_endpgm
3736;
3737; GFX10-CU-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
3738; GFX10-CU:       ; %bb.0: ; %entry
3739; GFX10-CU-NEXT:    s_clause 0x1
3740; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3741; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3742; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3743; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
3744; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
3745; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3746; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3747; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3748; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3749; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3750; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3751; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3752; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3753; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3754; GFX10-CU-NEXT:    buffer_gl0_inv
3755; GFX10-CU-NEXT:    buffer_gl1_inv
3756; GFX10-CU-NEXT:    s_endpgm
3757;
3758; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
3759; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3760; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3761; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3762; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3763; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
3764; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
3765; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3766; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3767; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3768; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3769; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3770; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
3771; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3772; SKIP-CACHE-INV-NEXT:    s_endpgm
3773;
3774; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
3775; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3776; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3777; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3778; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3779; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3780; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3781; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
3782; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3783; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3784; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3785; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
3786; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3787; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3788;
3789; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
3790; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3791; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3792; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3793; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3794; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3795; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3796; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
3797; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3798; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3799; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3800; GFX90A-TGSPLIT-NEXT:    buffer_invl2
3801; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3802; GFX90A-TGSPLIT-NEXT:    s_endpgm
3803;
3804; GFX940-NOTTGSPLIT-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
3805; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3806; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3807; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3808; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3809; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3810; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3811; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
3812; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3813; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
3814; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3815; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
3816; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3817;
3818; GFX940-TGSPLIT-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
3819; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3820; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3821; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3822; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3823; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3824; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3825; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
3826; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3827; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
3828; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3829; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
3830; GFX940-TGSPLIT-NEXT:    s_endpgm
3831    i32* %out, i32 %in, i32 %old) {
3832entry:
3833  %gep = getelementptr i32, i32* %out, i32 4
3834  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst seq_cst
3835  ret void
3836}
3837
3838define amdgpu_kernel void @flat_system_monotonic_monotonic_ret_cmpxchg(
3839; GFX7-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
3840; GFX7:       ; %bb.0: ; %entry
3841; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3842; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3843; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3844; GFX7-NEXT:    s_add_u32 s4, s0, 16
3845; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3846; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3847; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3848; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3849; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3850; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3851; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3852; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3853; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3854; GFX7-NEXT:    flat_store_dword v[0:1], v2
3855; GFX7-NEXT:    s_endpgm
3856;
3857; GFX10-WGP-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
3858; GFX10-WGP:       ; %bb.0: ; %entry
3859; GFX10-WGP-NEXT:    s_clause 0x1
3860; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3861; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3862; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3863; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3864; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3865; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3866; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3867; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
3868; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
3869; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3870; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3871; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3872; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3873; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
3874; GFX10-WGP-NEXT:    s_endpgm
3875;
3876; GFX10-CU-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
3877; GFX10-CU:       ; %bb.0: ; %entry
3878; GFX10-CU-NEXT:    s_clause 0x1
3879; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3880; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3881; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3882; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
3883; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
3884; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
3885; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
3886; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
3887; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
3888; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3889; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3890; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3891; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3892; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
3893; GFX10-CU-NEXT:    s_endpgm
3894;
3895; SKIP-CACHE-INV-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
3896; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3897; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3898; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
3899; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3900; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
3901; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
3902; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
3903; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
3904; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
3905; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
3906; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3907; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
3908; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
3909; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3910; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
3911; SKIP-CACHE-INV-NEXT:    s_endpgm
3912;
3913; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
3914; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3915; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3916; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3917; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3918; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3919; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3920; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3921; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3922; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3923; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3924;
3925; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
3926; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3927; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3928; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3929; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3930; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3931; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3932; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3933; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3934; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3935; GFX90A-TGSPLIT-NEXT:    s_endpgm
3936;
3937; GFX940-NOTTGSPLIT-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
3938; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
3939; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3940; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3941; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3942; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3943; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3944; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
3945; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3946; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3947; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
3948;
3949; GFX940-TGSPLIT-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
3950; GFX940-TGSPLIT:       ; %bb.0: ; %entry
3951; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
3952; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
3953; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3954; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
3955; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
3956; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
3957; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3958; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
3959; GFX940-TGSPLIT-NEXT:    s_endpgm
3960    i32* %out, i32 %in, i32 %old) {
3961entry:
3962  %gep = getelementptr i32, i32* %out, i32 4
3963  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in monotonic monotonic
3964  %val0 = extractvalue { i32, i1 } %val, 0
3965  store i32 %val0, i32* %out, align 4
3966  ret void
3967}
3968
3969define amdgpu_kernel void @flat_system_acquire_monotonic_ret_cmpxchg(
3970; GFX7-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
3971; GFX7:       ; %bb.0: ; %entry
3972; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3973; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3974; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3975; GFX7-NEXT:    s_add_u32 s4, s0, 16
3976; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3977; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3978; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3979; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3980; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3981; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3982; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3983; GFX7-NEXT:    buffer_wbinvl1_vol
3984; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3985; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3986; GFX7-NEXT:    flat_store_dword v[0:1], v2
3987; GFX7-NEXT:    s_endpgm
3988;
3989; GFX10-WGP-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
3990; GFX10-WGP:       ; %bb.0: ; %entry
3991; GFX10-WGP-NEXT:    s_clause 0x1
3992; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3993; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3994; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3995; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
3996; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
3997; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
3998; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
3999; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4000; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4001; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4002; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4003; GFX10-WGP-NEXT:    buffer_gl0_inv
4004; GFX10-WGP-NEXT:    buffer_gl1_inv
4005; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4006; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4007; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4008; GFX10-WGP-NEXT:    s_endpgm
4009;
4010; GFX10-CU-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
4011; GFX10-CU:       ; %bb.0: ; %entry
4012; GFX10-CU-NEXT:    s_clause 0x1
4013; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4014; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4015; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4016; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4017; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4018; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4019; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4020; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4021; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4022; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4023; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4024; GFX10-CU-NEXT:    buffer_gl0_inv
4025; GFX10-CU-NEXT:    buffer_gl1_inv
4026; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4027; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4028; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4029; GFX10-CU-NEXT:    s_endpgm
4030;
4031; SKIP-CACHE-INV-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
4032; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4033; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4034; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4035; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4036; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4037; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4038; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4039; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4040; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4041; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4042; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4043; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4044; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4045; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4046; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4047; SKIP-CACHE-INV-NEXT:    s_endpgm
4048;
4049; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
4050; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4051; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4052; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4053; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4054; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4055; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4056; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4057; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4058; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
4059; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4060; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4061; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4062;
4063; GFX90A-TGSPLIT-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
4064; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4065; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4066; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4067; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4068; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4069; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4070; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4071; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4072; GFX90A-TGSPLIT-NEXT:    buffer_invl2
4073; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4074; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4075; GFX90A-TGSPLIT-NEXT:    s_endpgm
4076;
4077; GFX940-NOTTGSPLIT-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
4078; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4079; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4080; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4081; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4082; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4083; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4084; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
4085; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4086; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
4087; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4088; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4089;
4090; GFX940-TGSPLIT-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
4091; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4092; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4093; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4094; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4095; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4096; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4097; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
4098; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4099; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
4100; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4101; GFX940-TGSPLIT-NEXT:    s_endpgm
4102    i32* %out, i32 %in, i32 %old) {
4103entry:
4104  %gep = getelementptr i32, i32* %out, i32 4
4105  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acquire monotonic
4106  %val0 = extractvalue { i32, i1 } %val, 0
4107  store i32 %val0, i32* %out, align 4
4108  ret void
4109}
4110
4111define amdgpu_kernel void @flat_system_release_monotonic_ret_cmpxchg(
4112; GFX7-LABEL: flat_system_release_monotonic_ret_cmpxchg:
4113; GFX7:       ; %bb.0: ; %entry
4114; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4115; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4116; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4117; GFX7-NEXT:    s_add_u32 s4, s0, 16
4118; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4119; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4120; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4121; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4122; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4123; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4124; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4125; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4126; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4127; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4128; GFX7-NEXT:    flat_store_dword v[0:1], v2
4129; GFX7-NEXT:    s_endpgm
4130;
4131; GFX10-WGP-LABEL: flat_system_release_monotonic_ret_cmpxchg:
4132; GFX10-WGP:       ; %bb.0: ; %entry
4133; GFX10-WGP-NEXT:    s_clause 0x1
4134; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4135; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4136; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4137; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4138; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4139; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4140; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4141; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4142; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4143; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4144; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4145; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4146; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4147; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4148; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4149; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4150; GFX10-WGP-NEXT:    s_endpgm
4151;
4152; GFX10-CU-LABEL: flat_system_release_monotonic_ret_cmpxchg:
4153; GFX10-CU:       ; %bb.0: ; %entry
4154; GFX10-CU-NEXT:    s_clause 0x1
4155; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4156; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4157; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4158; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4159; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4160; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4161; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4162; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4163; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4164; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4165; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4166; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4167; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4168; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4169; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4170; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4171; GFX10-CU-NEXT:    s_endpgm
4172;
4173; SKIP-CACHE-INV-LABEL: flat_system_release_monotonic_ret_cmpxchg:
4174; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4175; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4176; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4177; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4178; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4179; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4180; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4181; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4182; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4183; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4184; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4185; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4186; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4187; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4188; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4189; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4190; SKIP-CACHE-INV-NEXT:    s_endpgm
4191;
4192; GFX90A-NOTTGSPLIT-LABEL: flat_system_release_monotonic_ret_cmpxchg:
4193; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4194; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4195; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4196; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4197; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4198; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4199; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
4200; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4201; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4202; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4203; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4204; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4205;
4206; GFX90A-TGSPLIT-LABEL: flat_system_release_monotonic_ret_cmpxchg:
4207; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4208; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4209; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4210; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4211; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4212; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4213; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
4214; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4215; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4216; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4217; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4218; GFX90A-TGSPLIT-NEXT:    s_endpgm
4219;
4220; GFX940-NOTTGSPLIT-LABEL: flat_system_release_monotonic_ret_cmpxchg:
4221; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4222; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4223; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4224; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4225; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4226; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4227; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
4228; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4229; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
4230; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4231; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4232; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4233;
4234; GFX940-TGSPLIT-LABEL: flat_system_release_monotonic_ret_cmpxchg:
4235; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4236; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4237; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4238; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4239; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4240; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4241; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
4242; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4243; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
4244; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4245; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4246; GFX940-TGSPLIT-NEXT:    s_endpgm
4247    i32* %out, i32 %in, i32 %old) {
4248entry:
4249  %gep = getelementptr i32, i32* %out, i32 4
4250  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in release monotonic
4251  %val0 = extractvalue { i32, i1 } %val, 0
4252  store i32 %val0, i32* %out, align 4
4253  ret void
4254}
4255
4256define amdgpu_kernel void @flat_system_acq_rel_monotonic_ret_cmpxchg(
4257; GFX7-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
4258; GFX7:       ; %bb.0: ; %entry
4259; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4260; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4261; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4262; GFX7-NEXT:    s_add_u32 s4, s0, 16
4263; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4264; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4265; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4266; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4267; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4268; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4269; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4270; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4271; GFX7-NEXT:    buffer_wbinvl1_vol
4272; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4273; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4274; GFX7-NEXT:    flat_store_dword v[0:1], v2
4275; GFX7-NEXT:    s_endpgm
4276;
4277; GFX10-WGP-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
4278; GFX10-WGP:       ; %bb.0: ; %entry
4279; GFX10-WGP-NEXT:    s_clause 0x1
4280; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4281; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4282; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4283; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4284; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4285; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4286; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4287; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4288; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4289; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4290; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4291; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4292; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4293; GFX10-WGP-NEXT:    buffer_gl0_inv
4294; GFX10-WGP-NEXT:    buffer_gl1_inv
4295; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4296; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4297; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4298; GFX10-WGP-NEXT:    s_endpgm
4299;
4300; GFX10-CU-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
4301; GFX10-CU:       ; %bb.0: ; %entry
4302; GFX10-CU-NEXT:    s_clause 0x1
4303; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4304; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4305; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4306; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4307; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4308; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4309; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4310; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4311; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4312; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4313; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4314; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4315; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4316; GFX10-CU-NEXT:    buffer_gl0_inv
4317; GFX10-CU-NEXT:    buffer_gl1_inv
4318; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4319; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4320; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4321; GFX10-CU-NEXT:    s_endpgm
4322;
4323; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
4324; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4325; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4326; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4327; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4328; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4329; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4330; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4331; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4332; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4333; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4334; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4335; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4336; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4337; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4338; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4339; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4340; SKIP-CACHE-INV-NEXT:    s_endpgm
4341;
4342; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
4343; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4344; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4345; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4346; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4347; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4348; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4349; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
4350; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4351; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4352; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4353; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
4354; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4355; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4356; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4357;
4358; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
4359; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4360; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4361; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4362; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4363; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4364; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4365; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
4366; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4367; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4368; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4369; GFX90A-TGSPLIT-NEXT:    buffer_invl2
4370; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4371; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4372; GFX90A-TGSPLIT-NEXT:    s_endpgm
4373;
4374; GFX940-NOTTGSPLIT-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
4375; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4376; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4377; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4378; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4379; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4380; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4381; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
4382; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4383; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
4384; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4385; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
4386; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4387; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4388;
4389; GFX940-TGSPLIT-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
4390; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4391; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4392; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4393; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4394; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4395; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4396; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
4397; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4398; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
4399; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4400; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
4401; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4402; GFX940-TGSPLIT-NEXT:    s_endpgm
4403    i32* %out, i32 %in, i32 %old) {
4404entry:
4405  %gep = getelementptr i32, i32* %out, i32 4
4406  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acq_rel monotonic
4407  %val0 = extractvalue { i32, i1 } %val, 0
4408  store i32 %val0, i32* %out, align 4
4409  ret void
4410}
4411
4412define amdgpu_kernel void @flat_system_seq_cst_monotonic_ret_cmpxchg(
4413; GFX7-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
4414; GFX7:       ; %bb.0: ; %entry
4415; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4416; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4417; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4418; GFX7-NEXT:    s_add_u32 s4, s0, 16
4419; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4420; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4421; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4422; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4423; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4424; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4425; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4426; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4427; GFX7-NEXT:    buffer_wbinvl1_vol
4428; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4429; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4430; GFX7-NEXT:    flat_store_dword v[0:1], v2
4431; GFX7-NEXT:    s_endpgm
4432;
4433; GFX10-WGP-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
4434; GFX10-WGP:       ; %bb.0: ; %entry
4435; GFX10-WGP-NEXT:    s_clause 0x1
4436; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4437; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4438; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4439; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4440; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4441; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4442; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4443; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4444; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4445; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4446; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4447; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4448; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4449; GFX10-WGP-NEXT:    buffer_gl0_inv
4450; GFX10-WGP-NEXT:    buffer_gl1_inv
4451; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4452; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4453; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4454; GFX10-WGP-NEXT:    s_endpgm
4455;
4456; GFX10-CU-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
4457; GFX10-CU:       ; %bb.0: ; %entry
4458; GFX10-CU-NEXT:    s_clause 0x1
4459; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4460; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4461; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4462; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4463; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4464; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4465; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4466; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4467; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4468; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4469; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4470; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4471; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4472; GFX10-CU-NEXT:    buffer_gl0_inv
4473; GFX10-CU-NEXT:    buffer_gl1_inv
4474; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4475; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4476; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4477; GFX10-CU-NEXT:    s_endpgm
4478;
4479; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
4480; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4481; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4482; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4483; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4484; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4485; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4486; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4487; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4488; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4489; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4490; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4491; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4492; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4493; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4494; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4495; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4496; SKIP-CACHE-INV-NEXT:    s_endpgm
4497;
4498; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
4499; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4500; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4501; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4502; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4503; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4504; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4505; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
4506; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4507; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4508; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4509; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
4510; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4511; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4512; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4513;
4514; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
4515; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4516; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4517; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4518; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4519; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4520; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4521; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
4522; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4523; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4524; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4525; GFX90A-TGSPLIT-NEXT:    buffer_invl2
4526; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4527; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4528; GFX90A-TGSPLIT-NEXT:    s_endpgm
4529;
4530; GFX940-NOTTGSPLIT-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
4531; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4532; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4533; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4534; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4535; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4536; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4537; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
4538; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4539; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
4540; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4541; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
4542; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4543; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4544;
4545; GFX940-TGSPLIT-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
4546; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4547; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4548; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4549; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4550; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4551; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4552; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
4553; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4554; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
4555; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4556; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
4557; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4558; GFX940-TGSPLIT-NEXT:    s_endpgm
4559    i32* %out, i32 %in, i32 %old) {
4560entry:
4561  %gep = getelementptr i32, i32* %out, i32 4
4562  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst monotonic
4563  %val0 = extractvalue { i32, i1 } %val, 0
4564  store i32 %val0, i32* %out, align 4
4565  ret void
4566}
4567
4568define amdgpu_kernel void @flat_system_monotonic_acquire_ret_cmpxchg(
4569; GFX7-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
4570; GFX7:       ; %bb.0: ; %entry
4571; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4572; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4573; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4574; GFX7-NEXT:    s_add_u32 s4, s0, 16
4575; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4576; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4577; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4578; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4579; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4580; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4581; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4582; GFX7-NEXT:    buffer_wbinvl1_vol
4583; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4584; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4585; GFX7-NEXT:    flat_store_dword v[0:1], v2
4586; GFX7-NEXT:    s_endpgm
4587;
4588; GFX10-WGP-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
4589; GFX10-WGP:       ; %bb.0: ; %entry
4590; GFX10-WGP-NEXT:    s_clause 0x1
4591; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4592; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4593; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4594; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4595; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4596; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4597; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4598; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4599; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4600; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4601; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4602; GFX10-WGP-NEXT:    buffer_gl0_inv
4603; GFX10-WGP-NEXT:    buffer_gl1_inv
4604; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4605; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4606; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4607; GFX10-WGP-NEXT:    s_endpgm
4608;
4609; GFX10-CU-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
4610; GFX10-CU:       ; %bb.0: ; %entry
4611; GFX10-CU-NEXT:    s_clause 0x1
4612; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4613; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4614; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4615; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4616; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4617; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4618; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4619; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4620; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4621; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4622; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4623; GFX10-CU-NEXT:    buffer_gl0_inv
4624; GFX10-CU-NEXT:    buffer_gl1_inv
4625; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4626; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4627; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4628; GFX10-CU-NEXT:    s_endpgm
4629;
4630; SKIP-CACHE-INV-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
4631; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4632; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4633; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4634; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4635; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4636; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4637; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4638; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4639; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4640; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4641; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4642; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4643; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4644; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4645; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4646; SKIP-CACHE-INV-NEXT:    s_endpgm
4647;
4648; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
4649; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4650; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4651; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4652; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4653; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4654; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4655; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4656; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4657; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
4658; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4659; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4660; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4661;
4662; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
4663; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4664; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4665; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4666; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4667; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4668; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4669; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4670; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4671; GFX90A-TGSPLIT-NEXT:    buffer_invl2
4672; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4673; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4674; GFX90A-TGSPLIT-NEXT:    s_endpgm
4675;
4676; GFX940-NOTTGSPLIT-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
4677; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4678; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4679; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4680; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4681; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4682; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4683; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
4684; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4685; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
4686; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4687; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4688;
4689; GFX940-TGSPLIT-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
4690; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4691; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4692; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4693; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4694; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4695; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4696; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
4697; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4698; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
4699; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4700; GFX940-TGSPLIT-NEXT:    s_endpgm
4701    i32* %out, i32 %in, i32 %old) {
4702entry:
4703  %gep = getelementptr i32, i32* %out, i32 4
4704  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in monotonic acquire
4705  %val0 = extractvalue { i32, i1 } %val, 0
4706  store i32 %val0, i32* %out, align 4
4707  ret void
4708}
4709
4710define amdgpu_kernel void @flat_system_acquire_acquire_ret_cmpxchg(
4711; GFX7-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
4712; GFX7:       ; %bb.0: ; %entry
4713; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4714; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4715; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4716; GFX7-NEXT:    s_add_u32 s4, s0, 16
4717; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4718; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4719; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4720; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4721; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4722; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4723; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4724; GFX7-NEXT:    buffer_wbinvl1_vol
4725; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4726; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4727; GFX7-NEXT:    flat_store_dword v[0:1], v2
4728; GFX7-NEXT:    s_endpgm
4729;
4730; GFX10-WGP-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
4731; GFX10-WGP:       ; %bb.0: ; %entry
4732; GFX10-WGP-NEXT:    s_clause 0x1
4733; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4734; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4735; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4736; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4737; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4738; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4739; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4740; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4741; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4742; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4743; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4744; GFX10-WGP-NEXT:    buffer_gl0_inv
4745; GFX10-WGP-NEXT:    buffer_gl1_inv
4746; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4747; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4748; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4749; GFX10-WGP-NEXT:    s_endpgm
4750;
4751; GFX10-CU-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
4752; GFX10-CU:       ; %bb.0: ; %entry
4753; GFX10-CU-NEXT:    s_clause 0x1
4754; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4755; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4756; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4757; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4758; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4759; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4760; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4761; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4762; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4763; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4764; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4765; GFX10-CU-NEXT:    buffer_gl0_inv
4766; GFX10-CU-NEXT:    buffer_gl1_inv
4767; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4768; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4769; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4770; GFX10-CU-NEXT:    s_endpgm
4771;
4772; SKIP-CACHE-INV-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
4773; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4774; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4775; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4776; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4777; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4778; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4779; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4780; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4781; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4782; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4783; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4784; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4785; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4786; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4787; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4788; SKIP-CACHE-INV-NEXT:    s_endpgm
4789;
4790; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
4791; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4792; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4793; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4794; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4795; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4796; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4797; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4798; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4799; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
4800; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4801; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4802; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4803;
4804; GFX90A-TGSPLIT-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
4805; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4806; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4807; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4808; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4809; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4810; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4811; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4812; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4813; GFX90A-TGSPLIT-NEXT:    buffer_invl2
4814; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4815; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4816; GFX90A-TGSPLIT-NEXT:    s_endpgm
4817;
4818; GFX940-NOTTGSPLIT-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
4819; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4820; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4821; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4822; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4823; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4824; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4825; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
4826; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4827; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
4828; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4829; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4830;
4831; GFX940-TGSPLIT-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
4832; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4833; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4834; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4835; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4836; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4837; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4838; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
4839; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4840; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
4841; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4842; GFX940-TGSPLIT-NEXT:    s_endpgm
4843    i32* %out, i32 %in, i32 %old) {
4844entry:
4845  %gep = getelementptr i32, i32* %out, i32 4
4846  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acquire acquire
4847  %val0 = extractvalue { i32, i1 } %val, 0
4848  store i32 %val0, i32* %out, align 4
4849  ret void
4850}
4851
4852define amdgpu_kernel void @flat_system_release_acquire_ret_cmpxchg(
4853; GFX7-LABEL: flat_system_release_acquire_ret_cmpxchg:
4854; GFX7:       ; %bb.0: ; %entry
4855; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4856; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4857; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4858; GFX7-NEXT:    s_add_u32 s4, s0, 16
4859; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4860; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4861; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4862; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4863; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4864; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4865; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4866; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4867; GFX7-NEXT:    buffer_wbinvl1_vol
4868; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4869; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4870; GFX7-NEXT:    flat_store_dword v[0:1], v2
4871; GFX7-NEXT:    s_endpgm
4872;
4873; GFX10-WGP-LABEL: flat_system_release_acquire_ret_cmpxchg:
4874; GFX10-WGP:       ; %bb.0: ; %entry
4875; GFX10-WGP-NEXT:    s_clause 0x1
4876; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4877; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4878; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4879; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
4880; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
4881; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
4882; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
4883; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
4884; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
4885; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4886; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4887; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4888; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4889; GFX10-WGP-NEXT:    buffer_gl0_inv
4890; GFX10-WGP-NEXT:    buffer_gl1_inv
4891; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4892; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4893; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
4894; GFX10-WGP-NEXT:    s_endpgm
4895;
4896; GFX10-CU-LABEL: flat_system_release_acquire_ret_cmpxchg:
4897; GFX10-CU:       ; %bb.0: ; %entry
4898; GFX10-CU-NEXT:    s_clause 0x1
4899; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4900; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4901; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4902; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
4903; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
4904; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
4905; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
4906; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
4907; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
4908; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4909; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4910; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4911; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4912; GFX10-CU-NEXT:    buffer_gl0_inv
4913; GFX10-CU-NEXT:    buffer_gl1_inv
4914; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4915; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4916; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
4917; GFX10-CU-NEXT:    s_endpgm
4918;
4919; SKIP-CACHE-INV-LABEL: flat_system_release_acquire_ret_cmpxchg:
4920; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4921; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4922; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
4923; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4924; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
4925; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
4926; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4927; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
4928; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
4929; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
4930; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4931; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4932; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4933; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
4934; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
4935; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
4936; SKIP-CACHE-INV-NEXT:    s_endpgm
4937;
4938; GFX90A-NOTTGSPLIT-LABEL: flat_system_release_acquire_ret_cmpxchg:
4939; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4940; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4941; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4942; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4943; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4944; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4945; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
4946; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4947; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4948; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4949; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
4950; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4951; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4952; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4953;
4954; GFX90A-TGSPLIT-LABEL: flat_system_release_acquire_ret_cmpxchg:
4955; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4956; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4957; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4958; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4959; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4960; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4961; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
4962; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4963; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4964; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4965; GFX90A-TGSPLIT-NEXT:    buffer_invl2
4966; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4967; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4968; GFX90A-TGSPLIT-NEXT:    s_endpgm
4969;
4970; GFX940-NOTTGSPLIT-LABEL: flat_system_release_acquire_ret_cmpxchg:
4971; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
4972; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4973; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4974; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4975; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4976; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4977; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
4978; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4979; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
4980; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4981; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
4982; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4983; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
4984;
4985; GFX940-TGSPLIT-LABEL: flat_system_release_acquire_ret_cmpxchg:
4986; GFX940-TGSPLIT:       ; %bb.0: ; %entry
4987; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
4988; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
4989; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4990; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
4991; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
4992; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
4993; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4994; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
4995; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4996; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
4997; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
4998; GFX940-TGSPLIT-NEXT:    s_endpgm
4999    i32* %out, i32 %in, i32 %old) {
5000entry:
5001  %gep = getelementptr i32, i32* %out, i32 4
5002  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in release acquire
5003  %val0 = extractvalue { i32, i1 } %val, 0
5004  store i32 %val0, i32* %out, align 4
5005  ret void
5006}
5007
5008define amdgpu_kernel void @flat_system_acq_rel_acquire_ret_cmpxchg(
5009; GFX7-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
5010; GFX7:       ; %bb.0: ; %entry
5011; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5012; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5013; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5014; GFX7-NEXT:    s_add_u32 s4, s0, 16
5015; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5016; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5017; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5018; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5019; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5020; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5021; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5022; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5023; GFX7-NEXT:    buffer_wbinvl1_vol
5024; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5025; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5026; GFX7-NEXT:    flat_store_dword v[0:1], v2
5027; GFX7-NEXT:    s_endpgm
5028;
5029; GFX10-WGP-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
5030; GFX10-WGP:       ; %bb.0: ; %entry
5031; GFX10-WGP-NEXT:    s_clause 0x1
5032; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5033; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5034; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5035; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5036; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5037; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5038; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5039; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5040; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5041; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5042; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5043; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5044; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5045; GFX10-WGP-NEXT:    buffer_gl0_inv
5046; GFX10-WGP-NEXT:    buffer_gl1_inv
5047; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5048; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5049; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5050; GFX10-WGP-NEXT:    s_endpgm
5051;
5052; GFX10-CU-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
5053; GFX10-CU:       ; %bb.0: ; %entry
5054; GFX10-CU-NEXT:    s_clause 0x1
5055; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5056; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5057; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5058; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5059; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5060; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5061; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5062; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5063; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5064; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5065; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5066; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5067; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5068; GFX10-CU-NEXT:    buffer_gl0_inv
5069; GFX10-CU-NEXT:    buffer_gl1_inv
5070; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5071; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5072; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5073; GFX10-CU-NEXT:    s_endpgm
5074;
5075; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
5076; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5077; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5078; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5079; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5080; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5081; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5082; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5083; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5084; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5085; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5086; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5087; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5088; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5089; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5090; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5091; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5092; SKIP-CACHE-INV-NEXT:    s_endpgm
5093;
5094; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
5095; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5096; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5097; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5098; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5099; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5100; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5101; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
5102; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5103; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5104; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5105; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
5106; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5107; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5108; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5109;
5110; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
5111; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5112; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5113; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5114; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5115; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5116; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5117; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
5118; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5119; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5120; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5121; GFX90A-TGSPLIT-NEXT:    buffer_invl2
5122; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5123; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5124; GFX90A-TGSPLIT-NEXT:    s_endpgm
5125;
5126; GFX940-NOTTGSPLIT-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
5127; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5128; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5129; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5130; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5131; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5132; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5133; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
5134; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5135; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
5136; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5137; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
5138; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5139; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5140;
5141; GFX940-TGSPLIT-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
5142; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5143; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5144; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5145; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5146; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5147; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5148; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
5149; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5150; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
5151; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5152; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
5153; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5154; GFX940-TGSPLIT-NEXT:    s_endpgm
5155    i32* %out, i32 %in, i32 %old) {
5156entry:
5157  %gep = getelementptr i32, i32* %out, i32 4
5158  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acq_rel acquire
5159  %val0 = extractvalue { i32, i1 } %val, 0
5160  store i32 %val0, i32* %out, align 4
5161  ret void
5162}
5163
5164define amdgpu_kernel void @flat_system_seq_cst_acquire_ret_cmpxchg(
5165; GFX7-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
5166; GFX7:       ; %bb.0: ; %entry
5167; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5168; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5169; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5170; GFX7-NEXT:    s_add_u32 s4, s0, 16
5171; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5172; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5173; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5174; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5175; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5176; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5177; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5178; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5179; GFX7-NEXT:    buffer_wbinvl1_vol
5180; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5181; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5182; GFX7-NEXT:    flat_store_dword v[0:1], v2
5183; GFX7-NEXT:    s_endpgm
5184;
5185; GFX10-WGP-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
5186; GFX10-WGP:       ; %bb.0: ; %entry
5187; GFX10-WGP-NEXT:    s_clause 0x1
5188; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5189; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5190; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5191; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5192; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5193; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5194; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5195; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5196; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5197; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5198; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5199; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5200; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5201; GFX10-WGP-NEXT:    buffer_gl0_inv
5202; GFX10-WGP-NEXT:    buffer_gl1_inv
5203; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5204; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5205; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5206; GFX10-WGP-NEXT:    s_endpgm
5207;
5208; GFX10-CU-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
5209; GFX10-CU:       ; %bb.0: ; %entry
5210; GFX10-CU-NEXT:    s_clause 0x1
5211; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5212; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5213; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5214; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5215; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5216; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5217; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5218; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5219; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5220; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5221; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5222; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5223; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5224; GFX10-CU-NEXT:    buffer_gl0_inv
5225; GFX10-CU-NEXT:    buffer_gl1_inv
5226; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5227; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5228; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5229; GFX10-CU-NEXT:    s_endpgm
5230;
5231; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
5232; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5233; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5234; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5235; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5236; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5237; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5238; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5239; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5240; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5241; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5242; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5243; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5244; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5245; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5246; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5247; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5248; SKIP-CACHE-INV-NEXT:    s_endpgm
5249;
5250; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
5251; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5252; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5253; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5254; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5255; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5256; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5257; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
5258; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5259; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5260; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5261; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
5262; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5263; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5264; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5265;
5266; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
5267; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5268; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5269; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5270; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5271; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5272; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5273; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
5274; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5275; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5276; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5277; GFX90A-TGSPLIT-NEXT:    buffer_invl2
5278; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5279; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5280; GFX90A-TGSPLIT-NEXT:    s_endpgm
5281;
5282; GFX940-NOTTGSPLIT-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
5283; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5284; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5285; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5286; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5287; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5288; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5289; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
5290; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5291; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
5292; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5293; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
5294; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5295; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5296;
5297; GFX940-TGSPLIT-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
5298; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5299; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5300; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5301; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5302; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5303; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5304; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
5305; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5306; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
5307; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5308; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
5309; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5310; GFX940-TGSPLIT-NEXT:    s_endpgm
5311    i32* %out, i32 %in, i32 %old) {
5312entry:
5313  %gep = getelementptr i32, i32* %out, i32 4
5314  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst acquire
5315  %val0 = extractvalue { i32, i1 } %val, 0
5316  store i32 %val0, i32* %out, align 4
5317  ret void
5318}
5319
5320define amdgpu_kernel void @flat_system_monotonic_seq_cst_ret_cmpxchg(
5321; GFX7-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
5322; GFX7:       ; %bb.0: ; %entry
5323; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5324; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5325; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5326; GFX7-NEXT:    s_add_u32 s4, s0, 16
5327; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5328; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5329; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5330; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5331; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5332; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5333; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5334; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5335; GFX7-NEXT:    buffer_wbinvl1_vol
5336; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5337; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5338; GFX7-NEXT:    flat_store_dword v[0:1], v2
5339; GFX7-NEXT:    s_endpgm
5340;
5341; GFX10-WGP-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
5342; GFX10-WGP:       ; %bb.0: ; %entry
5343; GFX10-WGP-NEXT:    s_clause 0x1
5344; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5345; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5346; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5347; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5348; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5349; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5350; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5351; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5352; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5353; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5354; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5355; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5356; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5357; GFX10-WGP-NEXT:    buffer_gl0_inv
5358; GFX10-WGP-NEXT:    buffer_gl1_inv
5359; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5360; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5361; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5362; GFX10-WGP-NEXT:    s_endpgm
5363;
5364; GFX10-CU-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
5365; GFX10-CU:       ; %bb.0: ; %entry
5366; GFX10-CU-NEXT:    s_clause 0x1
5367; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5368; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5369; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5370; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5371; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5372; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5373; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5374; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5375; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5376; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5377; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5378; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5379; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5380; GFX10-CU-NEXT:    buffer_gl0_inv
5381; GFX10-CU-NEXT:    buffer_gl1_inv
5382; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5383; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5384; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5385; GFX10-CU-NEXT:    s_endpgm
5386;
5387; SKIP-CACHE-INV-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
5388; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5389; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5390; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5391; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5392; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5393; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5394; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5395; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5396; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5397; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5398; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5399; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5400; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5401; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5402; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5403; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5404; SKIP-CACHE-INV-NEXT:    s_endpgm
5405;
5406; GFX90A-NOTTGSPLIT-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
5407; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5408; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5409; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5410; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5411; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5412; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5413; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
5414; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5415; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5416; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5417; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
5418; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5419; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5420; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5421;
5422; GFX90A-TGSPLIT-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
5423; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5424; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5425; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5426; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5427; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5428; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5429; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
5430; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5431; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5432; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5433; GFX90A-TGSPLIT-NEXT:    buffer_invl2
5434; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5435; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5436; GFX90A-TGSPLIT-NEXT:    s_endpgm
5437;
5438; GFX940-NOTTGSPLIT-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
5439; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5440; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5441; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5442; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5443; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5444; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5445; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
5446; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5447; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
5448; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5449; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
5450; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5451; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5452;
5453; GFX940-TGSPLIT-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
5454; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5455; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5456; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5457; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5458; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5459; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5460; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
5461; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5462; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
5463; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5464; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
5465; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5466; GFX940-TGSPLIT-NEXT:    s_endpgm
5467    i32* %out, i32 %in, i32 %old) {
5468entry:
5469  %gep = getelementptr i32, i32* %out, i32 4
5470  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in monotonic seq_cst
5471  %val0 = extractvalue { i32, i1 } %val, 0
5472  store i32 %val0, i32* %out, align 4
5473  ret void
5474}
5475
5476define amdgpu_kernel void @flat_system_acquire_seq_cst_ret_cmpxchg(
5477; GFX7-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
5478; GFX7:       ; %bb.0: ; %entry
5479; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5480; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5481; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5482; GFX7-NEXT:    s_add_u32 s4, s0, 16
5483; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5484; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5485; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5486; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5487; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5488; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5489; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5490; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5491; GFX7-NEXT:    buffer_wbinvl1_vol
5492; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5493; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5494; GFX7-NEXT:    flat_store_dword v[0:1], v2
5495; GFX7-NEXT:    s_endpgm
5496;
5497; GFX10-WGP-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
5498; GFX10-WGP:       ; %bb.0: ; %entry
5499; GFX10-WGP-NEXT:    s_clause 0x1
5500; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5501; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5502; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5503; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5504; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5505; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5506; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5507; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5508; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5509; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5510; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5511; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5512; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5513; GFX10-WGP-NEXT:    buffer_gl0_inv
5514; GFX10-WGP-NEXT:    buffer_gl1_inv
5515; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5516; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5517; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5518; GFX10-WGP-NEXT:    s_endpgm
5519;
5520; GFX10-CU-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
5521; GFX10-CU:       ; %bb.0: ; %entry
5522; GFX10-CU-NEXT:    s_clause 0x1
5523; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5524; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5525; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5526; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5527; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5528; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5529; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5530; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5531; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5532; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5533; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5534; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5535; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5536; GFX10-CU-NEXT:    buffer_gl0_inv
5537; GFX10-CU-NEXT:    buffer_gl1_inv
5538; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5539; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5540; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5541; GFX10-CU-NEXT:    s_endpgm
5542;
5543; SKIP-CACHE-INV-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
5544; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5545; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5546; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5547; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5548; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5549; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5550; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5551; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5552; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5553; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5554; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5555; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5556; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5557; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5558; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5559; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5560; SKIP-CACHE-INV-NEXT:    s_endpgm
5561;
5562; GFX90A-NOTTGSPLIT-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
5563; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5564; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5565; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5566; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5567; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5568; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5569; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
5570; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5571; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5572; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5573; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
5574; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5575; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5576; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5577;
5578; GFX90A-TGSPLIT-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
5579; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5580; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5581; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5582; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5583; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5584; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5585; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
5586; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5587; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5588; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5589; GFX90A-TGSPLIT-NEXT:    buffer_invl2
5590; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5591; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5592; GFX90A-TGSPLIT-NEXT:    s_endpgm
5593;
5594; GFX940-NOTTGSPLIT-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
5595; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5596; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5597; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5598; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5599; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5600; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5601; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
5602; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5603; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
5604; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5605; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
5606; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5607; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5608;
5609; GFX940-TGSPLIT-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
5610; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5611; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5612; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5613; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5614; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5615; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5616; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
5617; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5618; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
5619; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5620; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
5621; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5622; GFX940-TGSPLIT-NEXT:    s_endpgm
5623    i32* %out, i32 %in, i32 %old) {
5624entry:
5625  %gep = getelementptr i32, i32* %out, i32 4
5626  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acquire seq_cst
5627  %val0 = extractvalue { i32, i1 } %val, 0
5628  store i32 %val0, i32* %out, align 4
5629  ret void
5630}
5631
5632define amdgpu_kernel void @flat_system_release_seq_cst_ret_cmpxchg(
5633; GFX7-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
5634; GFX7:       ; %bb.0: ; %entry
5635; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5636; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5637; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5638; GFX7-NEXT:    s_add_u32 s4, s0, 16
5639; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5640; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5641; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5642; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5643; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5644; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5645; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5646; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5647; GFX7-NEXT:    buffer_wbinvl1_vol
5648; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5649; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5650; GFX7-NEXT:    flat_store_dword v[0:1], v2
5651; GFX7-NEXT:    s_endpgm
5652;
5653; GFX10-WGP-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
5654; GFX10-WGP:       ; %bb.0: ; %entry
5655; GFX10-WGP-NEXT:    s_clause 0x1
5656; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5657; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5658; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5659; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5660; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5661; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5662; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5663; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5664; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5665; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5666; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5667; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5668; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5669; GFX10-WGP-NEXT:    buffer_gl0_inv
5670; GFX10-WGP-NEXT:    buffer_gl1_inv
5671; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5672; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5673; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5674; GFX10-WGP-NEXT:    s_endpgm
5675;
5676; GFX10-CU-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
5677; GFX10-CU:       ; %bb.0: ; %entry
5678; GFX10-CU-NEXT:    s_clause 0x1
5679; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5680; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5681; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5682; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5683; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5684; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5685; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5686; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5687; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5688; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5689; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5690; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5691; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5692; GFX10-CU-NEXT:    buffer_gl0_inv
5693; GFX10-CU-NEXT:    buffer_gl1_inv
5694; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5695; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5696; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5697; GFX10-CU-NEXT:    s_endpgm
5698;
5699; SKIP-CACHE-INV-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
5700; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5701; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5702; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5703; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5704; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5705; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5706; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5707; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5708; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5709; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5710; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5711; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5712; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5713; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5714; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5715; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5716; SKIP-CACHE-INV-NEXT:    s_endpgm
5717;
5718; GFX90A-NOTTGSPLIT-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
5719; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5720; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5721; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5722; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5723; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5724; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5725; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
5726; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5727; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5728; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5729; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
5730; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5731; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5732; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5733;
5734; GFX90A-TGSPLIT-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
5735; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5736; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5737; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5738; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5739; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5740; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5741; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
5742; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5743; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5744; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5745; GFX90A-TGSPLIT-NEXT:    buffer_invl2
5746; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5747; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5748; GFX90A-TGSPLIT-NEXT:    s_endpgm
5749;
5750; GFX940-NOTTGSPLIT-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
5751; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5752; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5753; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5754; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5755; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5756; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5757; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
5758; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5759; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
5760; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5761; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
5762; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5763; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5764;
5765; GFX940-TGSPLIT-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
5766; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5767; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5768; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5769; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5770; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5771; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5772; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
5773; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5774; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
5775; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5776; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
5777; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5778; GFX940-TGSPLIT-NEXT:    s_endpgm
5779    i32* %out, i32 %in, i32 %old) {
5780entry:
5781  %gep = getelementptr i32, i32* %out, i32 4
5782  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in release seq_cst
5783  %val0 = extractvalue { i32, i1 } %val, 0
5784  store i32 %val0, i32* %out, align 4
5785  ret void
5786}
5787
5788define amdgpu_kernel void @flat_system_acq_rel_seq_cst_ret_cmpxchg(
5789; GFX7-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
5790; GFX7:       ; %bb.0: ; %entry
5791; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5792; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5793; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5794; GFX7-NEXT:    s_add_u32 s4, s0, 16
5795; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5796; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5797; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5798; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5799; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5800; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5801; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5802; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5803; GFX7-NEXT:    buffer_wbinvl1_vol
5804; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5805; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5806; GFX7-NEXT:    flat_store_dword v[0:1], v2
5807; GFX7-NEXT:    s_endpgm
5808;
5809; GFX10-WGP-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
5810; GFX10-WGP:       ; %bb.0: ; %entry
5811; GFX10-WGP-NEXT:    s_clause 0x1
5812; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5813; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5814; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5815; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5816; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5817; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5818; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5819; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5820; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5821; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5822; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5823; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5824; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5825; GFX10-WGP-NEXT:    buffer_gl0_inv
5826; GFX10-WGP-NEXT:    buffer_gl1_inv
5827; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5828; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5829; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5830; GFX10-WGP-NEXT:    s_endpgm
5831;
5832; GFX10-CU-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
5833; GFX10-CU:       ; %bb.0: ; %entry
5834; GFX10-CU-NEXT:    s_clause 0x1
5835; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5836; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5837; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5838; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5839; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5840; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5841; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5842; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5843; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
5844; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5845; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5846; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5847; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5848; GFX10-CU-NEXT:    buffer_gl0_inv
5849; GFX10-CU-NEXT:    buffer_gl1_inv
5850; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
5851; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
5852; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
5853; GFX10-CU-NEXT:    s_endpgm
5854;
5855; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
5856; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5857; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5858; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
5859; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5860; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
5861; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
5862; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5863; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
5864; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
5865; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
5866; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5867; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5868; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5869; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
5870; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
5871; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
5872; SKIP-CACHE-INV-NEXT:    s_endpgm
5873;
5874; GFX90A-NOTTGSPLIT-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
5875; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5876; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5877; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5878; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5879; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5880; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5881; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
5882; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5883; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5884; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5885; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
5886; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5887; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5888; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5889;
5890; GFX90A-TGSPLIT-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
5891; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5892; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5893; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5894; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5895; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5896; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5897; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
5898; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5899; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5900; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5901; GFX90A-TGSPLIT-NEXT:    buffer_invl2
5902; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5903; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5904; GFX90A-TGSPLIT-NEXT:    s_endpgm
5905;
5906; GFX940-NOTTGSPLIT-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
5907; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
5908; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5909; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5910; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5911; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5912; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5913; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
5914; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5915; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
5916; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5917; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
5918; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5919; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
5920;
5921; GFX940-TGSPLIT-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
5922; GFX940-TGSPLIT:       ; %bb.0: ; %entry
5923; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
5924; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
5925; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5926; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
5927; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
5928; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
5929; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5930; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
5931; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5932; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
5933; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
5934; GFX940-TGSPLIT-NEXT:    s_endpgm
5935    i32* %out, i32 %in, i32 %old) {
5936entry:
5937  %gep = getelementptr i32, i32* %out, i32 4
5938  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acq_rel seq_cst
5939  %val0 = extractvalue { i32, i1 } %val, 0
5940  store i32 %val0, i32* %out, align 4
5941  ret void
5942}
5943
5944define amdgpu_kernel void @flat_system_seq_cst_seq_cst_ret_cmpxchg(
5945; GFX7-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
5946; GFX7:       ; %bb.0: ; %entry
5947; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5948; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
5949; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5950; GFX7-NEXT:    s_add_u32 s4, s0, 16
5951; GFX7-NEXT:    s_addc_u32 s5, s1, 0
5952; GFX7-NEXT:    v_mov_b32_e32 v0, s4
5953; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5954; GFX7-NEXT:    v_mov_b32_e32 v1, s5
5955; GFX7-NEXT:    v_mov_b32_e32 v3, s3
5956; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5957; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5958; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5959; GFX7-NEXT:    buffer_wbinvl1_vol
5960; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5961; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5962; GFX7-NEXT:    flat_store_dword v[0:1], v2
5963; GFX7-NEXT:    s_endpgm
5964;
5965; GFX10-WGP-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
5966; GFX10-WGP:       ; %bb.0: ; %entry
5967; GFX10-WGP-NEXT:    s_clause 0x1
5968; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5969; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5970; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5971; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
5972; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
5973; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
5974; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
5975; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
5976; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
5977; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5978; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5979; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5980; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
5981; GFX10-WGP-NEXT:    buffer_gl0_inv
5982; GFX10-WGP-NEXT:    buffer_gl1_inv
5983; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
5984; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
5985; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
5986; GFX10-WGP-NEXT:    s_endpgm
5987;
5988; GFX10-CU-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
5989; GFX10-CU:       ; %bb.0: ; %entry
5990; GFX10-CU-NEXT:    s_clause 0x1
5991; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5992; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
5993; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5994; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
5995; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
5996; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
5997; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
5998; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
5999; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
6000; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6001; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6002; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6003; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6004; GFX10-CU-NEXT:    buffer_gl0_inv
6005; GFX10-CU-NEXT:    buffer_gl1_inv
6006; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6007; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6008; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6009; GFX10-CU-NEXT:    s_endpgm
6010;
6011; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
6012; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6013; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6014; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
6015; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6016; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
6017; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
6018; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
6019; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
6020; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
6021; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6022; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6023; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6024; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6025; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6026; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6027; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6028; SKIP-CACHE-INV-NEXT:    s_endpgm
6029;
6030; GFX90A-NOTTGSPLIT-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
6031; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6032; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6033; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6034; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6035; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6036; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6037; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
6038; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6039; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6040; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6041; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
6042; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6043; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6044; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6045;
6046; GFX90A-TGSPLIT-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
6047; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6048; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6049; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6050; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6051; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6052; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
6053; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
6054; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6055; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6056; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6057; GFX90A-TGSPLIT-NEXT:    buffer_invl2
6058; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6059; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6060; GFX90A-TGSPLIT-NEXT:    s_endpgm
6061;
6062; GFX940-NOTTGSPLIT-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
6063; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6064; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6065; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6066; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6067; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6068; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6069; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
6070; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6071; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
6072; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6073; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
6074; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6075; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6076;
6077; GFX940-TGSPLIT-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
6078; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6079; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
6080; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
6081; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6082; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6083; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
6084; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
6085; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6086; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
6087; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6088; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
6089; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6090; GFX940-TGSPLIT-NEXT:    s_endpgm
6091    i32* %out, i32 %in, i32 %old) {
6092entry:
6093  %gep = getelementptr i32, i32* %out, i32 4
6094  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst seq_cst
6095  %val0 = extractvalue { i32, i1 } %val, 0
6096  store i32 %val0, i32* %out, align 4
6097  ret void
6098}
6099
6100define amdgpu_kernel void @flat_system_one_as_unordered_load(
6101; GFX7-LABEL: flat_system_one_as_unordered_load:
6102; GFX7:       ; %bb.0: ; %entry
6103; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6104; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6105; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6106; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6107; GFX7-NEXT:    flat_load_dword v2, v[0:1]
6108; GFX7-NEXT:    v_mov_b32_e32 v0, s2
6109; GFX7-NEXT:    v_mov_b32_e32 v1, s3
6110; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6111; GFX7-NEXT:    flat_store_dword v[0:1], v2
6112; GFX7-NEXT:    s_endpgm
6113;
6114; GFX10-WGP-LABEL: flat_system_one_as_unordered_load:
6115; GFX10-WGP:       ; %bb.0: ; %entry
6116; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6117; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6118; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6119; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6120; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
6121; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
6122; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
6123; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6124; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6125; GFX10-WGP-NEXT:    s_endpgm
6126;
6127; GFX10-CU-LABEL: flat_system_one_as_unordered_load:
6128; GFX10-CU:       ; %bb.0: ; %entry
6129; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6130; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6131; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6132; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6133; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
6134; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
6135; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
6136; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6137; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6138; GFX10-CU-NEXT:    s_endpgm
6139;
6140; SKIP-CACHE-INV-LABEL: flat_system_one_as_unordered_load:
6141; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6142; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6143; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6144; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6145; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6146; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
6147; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6148; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6149; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6150; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6151; SKIP-CACHE-INV-NEXT:    s_endpgm
6152;
6153; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_unordered_load:
6154; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6155; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6156; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6157; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6158; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6159; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6160; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6161; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6162; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6163; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6164; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6165;
6166; GFX90A-TGSPLIT-LABEL: flat_system_one_as_unordered_load:
6167; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6168; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6169; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6170; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6171; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6172; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6173; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6174; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6175; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6176; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6177; GFX90A-TGSPLIT-NEXT:    s_endpgm
6178;
6179; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_unordered_load:
6180; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6181; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6182; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6183; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6184; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6185; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6186; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6187; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6188; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6189; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6190; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6191;
6192; GFX940-TGSPLIT-LABEL: flat_system_one_as_unordered_load:
6193; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6194; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6195; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6196; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6197; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6198; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1]
6199; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6200; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6201; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6202; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6203; GFX940-TGSPLIT-NEXT:    s_endpgm
6204    i32* %in, i32* %out) {
6205entry:
6206  %val = load atomic i32, i32* %in syncscope("one-as") unordered, align 4
6207  store i32 %val, i32* %out
6208  ret void
6209}
6210
6211define amdgpu_kernel void @flat_system_one_as_monotonic_load(
6212; GFX7-LABEL: flat_system_one_as_monotonic_load:
6213; GFX7:       ; %bb.0: ; %entry
6214; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6215; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6216; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6217; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6218; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
6219; GFX7-NEXT:    v_mov_b32_e32 v0, s2
6220; GFX7-NEXT:    v_mov_b32_e32 v1, s3
6221; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6222; GFX7-NEXT:    flat_store_dword v[0:1], v2
6223; GFX7-NEXT:    s_endpgm
6224;
6225; GFX10-WGP-LABEL: flat_system_one_as_monotonic_load:
6226; GFX10-WGP:       ; %bb.0: ; %entry
6227; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6228; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6229; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6230; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6231; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
6232; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
6233; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
6234; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6235; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6236; GFX10-WGP-NEXT:    s_endpgm
6237;
6238; GFX10-CU-LABEL: flat_system_one_as_monotonic_load:
6239; GFX10-CU:       ; %bb.0: ; %entry
6240; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6241; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6242; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6243; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6244; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
6245; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
6246; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
6247; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6248; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6249; GFX10-CU-NEXT:    s_endpgm
6250;
6251; SKIP-CACHE-INV-LABEL: flat_system_one_as_monotonic_load:
6252; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6253; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6254; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6255; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6256; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6257; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
6258; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6259; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6260; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6261; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6262; SKIP-CACHE-INV-NEXT:    s_endpgm
6263;
6264; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_load:
6265; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6266; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6267; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6268; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6269; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6270; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
6271; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6272; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6273; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6274; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6275; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6276;
6277; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_load:
6278; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6279; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6280; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6281; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6282; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6283; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
6284; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6285; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6286; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6287; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6288; GFX90A-TGSPLIT-NEXT:    s_endpgm
6289;
6290; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_load:
6291; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6292; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6293; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6294; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6295; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6296; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0 sc1
6297; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6298; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6299; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
6300; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6301; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6302;
6303; GFX940-TGSPLIT-LABEL: flat_system_one_as_monotonic_load:
6304; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6305; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6306; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6307; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6308; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6309; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0 sc1
6310; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6311; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6312; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6313; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6314; GFX940-TGSPLIT-NEXT:    s_endpgm
6315    i32* %in, i32* %out) {
6316entry:
6317  %val = load atomic i32, i32* %in syncscope("one-as") monotonic, align 4
6318  store i32 %val, i32* %out
6319  ret void
6320}
6321
6322define amdgpu_kernel void @flat_system_one_as_acquire_load(
6323; GFX7-LABEL: flat_system_one_as_acquire_load:
6324; GFX7:       ; %bb.0: ; %entry
6325; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6326; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6327; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6328; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6329; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
6330; GFX7-NEXT:    s_waitcnt vmcnt(0)
6331; GFX7-NEXT:    buffer_wbinvl1_vol
6332; GFX7-NEXT:    v_mov_b32_e32 v0, s2
6333; GFX7-NEXT:    v_mov_b32_e32 v1, s3
6334; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6335; GFX7-NEXT:    flat_store_dword v[0:1], v2
6336; GFX7-NEXT:    s_endpgm
6337;
6338; GFX10-WGP-LABEL: flat_system_one_as_acquire_load:
6339; GFX10-WGP:       ; %bb.0: ; %entry
6340; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6341; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6342; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6343; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6344; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
6345; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6346; GFX10-WGP-NEXT:    buffer_gl0_inv
6347; GFX10-WGP-NEXT:    buffer_gl1_inv
6348; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
6349; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
6350; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6351; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6352; GFX10-WGP-NEXT:    s_endpgm
6353;
6354; GFX10-CU-LABEL: flat_system_one_as_acquire_load:
6355; GFX10-CU:       ; %bb.0: ; %entry
6356; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6357; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6358; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6359; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6360; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
6361; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6362; GFX10-CU-NEXT:    buffer_gl0_inv
6363; GFX10-CU-NEXT:    buffer_gl1_inv
6364; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
6365; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
6366; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6367; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6368; GFX10-CU-NEXT:    s_endpgm
6369;
6370; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_load:
6371; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6372; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6373; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6374; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6375; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6376; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
6377; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6378; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6379; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6380; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6381; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6382; SKIP-CACHE-INV-NEXT:    s_endpgm
6383;
6384; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_load:
6385; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6386; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6387; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6388; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6389; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6390; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
6391; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6392; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
6393; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6394; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6395; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6396; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6397; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6398; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6399;
6400; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_load:
6401; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6402; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6403; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6404; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6405; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6406; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
6407; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6408; GFX90A-TGSPLIT-NEXT:    buffer_invl2
6409; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6410; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6411; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6412; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6413; GFX90A-TGSPLIT-NEXT:    s_endpgm
6414;
6415; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_load:
6416; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6417; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6418; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6419; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6420; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6421; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0 sc1
6422; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6423; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
6424; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6425; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6426; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6427; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6428; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6429;
6430; GFX940-TGSPLIT-LABEL: flat_system_one_as_acquire_load:
6431; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6432; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6433; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6434; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6435; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6436; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0 sc1
6437; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6438; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
6439; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6440; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6441; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6442; GFX940-TGSPLIT-NEXT:    s_endpgm
6443    i32* %in, i32* %out) {
6444entry:
6445  %val = load atomic i32, i32* %in syncscope("one-as") acquire, align 4
6446  store i32 %val, i32* %out
6447  ret void
6448}
6449
6450define amdgpu_kernel void @flat_system_one_as_seq_cst_load(
6451; GFX7-LABEL: flat_system_one_as_seq_cst_load:
6452; GFX7:       ; %bb.0: ; %entry
6453; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6454; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6455; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6456; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6457; GFX7-NEXT:    s_waitcnt vmcnt(0)
6458; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
6459; GFX7-NEXT:    s_waitcnt vmcnt(0)
6460; GFX7-NEXT:    buffer_wbinvl1_vol
6461; GFX7-NEXT:    v_mov_b32_e32 v0, s2
6462; GFX7-NEXT:    v_mov_b32_e32 v1, s3
6463; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6464; GFX7-NEXT:    flat_store_dword v[0:1], v2
6465; GFX7-NEXT:    s_endpgm
6466;
6467; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_load:
6468; GFX10-WGP:       ; %bb.0: ; %entry
6469; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6470; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6471; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6472; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6473; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6474; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6475; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
6476; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6477; GFX10-WGP-NEXT:    buffer_gl0_inv
6478; GFX10-WGP-NEXT:    buffer_gl1_inv
6479; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
6480; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
6481; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6482; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6483; GFX10-WGP-NEXT:    s_endpgm
6484;
6485; GFX10-CU-LABEL: flat_system_one_as_seq_cst_load:
6486; GFX10-CU:       ; %bb.0: ; %entry
6487; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6488; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6489; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6490; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6491; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6492; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6493; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
6494; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6495; GFX10-CU-NEXT:    buffer_gl0_inv
6496; GFX10-CU-NEXT:    buffer_gl1_inv
6497; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
6498; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
6499; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6500; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6501; GFX10-CU-NEXT:    s_endpgm
6502;
6503; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_load:
6504; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6505; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6506; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6507; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6508; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6509; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6510; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
6511; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6512; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6513; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6514; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6515; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6516; SKIP-CACHE-INV-NEXT:    s_endpgm
6517;
6518; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_load:
6519; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6520; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6521; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6522; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6523; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6524; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6525; GFX90A-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
6526; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6527; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
6528; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6529; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6530; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6531; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6532; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6533; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6534;
6535; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_load:
6536; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6537; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
6538; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6539; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6540; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6541; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6542; GFX90A-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] glc
6543; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6544; GFX90A-TGSPLIT-NEXT:    buffer_invl2
6545; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6546; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6547; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6548; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6549; GFX90A-TGSPLIT-NEXT:    s_endpgm
6550;
6551; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_load:
6552; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6553; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6554; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6555; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6556; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6557; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6558; GFX940-NOTTGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0 sc1
6559; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6560; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
6561; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6562; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6563; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6564; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6565; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6566;
6567; GFX940-TGSPLIT-LABEL: flat_system_one_as_seq_cst_load:
6568; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6569; GFX940-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
6570; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6571; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s0
6572; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s1
6573; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6574; GFX940-TGSPLIT-NEXT:    flat_load_dword v2, v[0:1] sc0 sc1
6575; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6576; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
6577; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v0, s2
6578; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s3
6579; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6580; GFX940-TGSPLIT-NEXT:    s_endpgm
6581    i32* %in, i32* %out) {
6582entry:
6583  %val = load atomic i32, i32* %in syncscope("one-as") seq_cst, align 4
6584  store i32 %val, i32* %out
6585  ret void
6586}
6587
6588define amdgpu_kernel void @flat_system_one_as_unordered_store(
6589; GFX7-LABEL: flat_system_one_as_unordered_store:
6590; GFX7:       ; %bb.0: ; %entry
6591; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
6592; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
6593; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6594; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6595; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6596; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6597; GFX7-NEXT:    flat_store_dword v[0:1], v2
6598; GFX7-NEXT:    s_endpgm
6599;
6600; GFX10-WGP-LABEL: flat_system_one_as_unordered_store:
6601; GFX10-WGP:       ; %bb.0: ; %entry
6602; GFX10-WGP-NEXT:    s_clause 0x1
6603; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6604; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
6605; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6606; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6607; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6608; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6609; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6610; GFX10-WGP-NEXT:    s_endpgm
6611;
6612; GFX10-CU-LABEL: flat_system_one_as_unordered_store:
6613; GFX10-CU:       ; %bb.0: ; %entry
6614; GFX10-CU-NEXT:    s_clause 0x1
6615; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6616; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
6617; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6618; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6619; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6620; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6621; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6622; GFX10-CU-NEXT:    s_endpgm
6623;
6624; SKIP-CACHE-INV-LABEL: flat_system_one_as_unordered_store:
6625; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6626; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
6627; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
6628; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6629; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6630; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6631; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6632; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6633; SKIP-CACHE-INV-NEXT:    s_endpgm
6634;
6635; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_unordered_store:
6636; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6637; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6638; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
6639; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6640; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6641; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6642; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6643; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6644;
6645; GFX90A-TGSPLIT-LABEL: flat_system_one_as_unordered_store:
6646; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6647; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6648; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
6649; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6650; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6651; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6652; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6653; GFX90A-TGSPLIT-NEXT:    s_endpgm
6654;
6655; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_unordered_store:
6656; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6657; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
6658; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
6659; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6660; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6661; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
6662; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6663; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6664;
6665; GFX940-TGSPLIT-LABEL: flat_system_one_as_unordered_store:
6666; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6667; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
6668; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
6669; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6670; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6671; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
6672; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6673; GFX940-TGSPLIT-NEXT:    s_endpgm
6674    i32 %in, i32* %out) {
6675entry:
6676  store atomic i32 %in, i32* %out syncscope("one-as") unordered, align 4
6677  ret void
6678}
6679
6680define amdgpu_kernel void @flat_system_one_as_monotonic_store(
6681; GFX7-LABEL: flat_system_one_as_monotonic_store:
6682; GFX7:       ; %bb.0: ; %entry
6683; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
6684; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
6685; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6686; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6687; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6688; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6689; GFX7-NEXT:    flat_store_dword v[0:1], v2
6690; GFX7-NEXT:    s_endpgm
6691;
6692; GFX10-WGP-LABEL: flat_system_one_as_monotonic_store:
6693; GFX10-WGP:       ; %bb.0: ; %entry
6694; GFX10-WGP-NEXT:    s_clause 0x1
6695; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6696; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
6697; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6698; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6699; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6700; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6701; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6702; GFX10-WGP-NEXT:    s_endpgm
6703;
6704; GFX10-CU-LABEL: flat_system_one_as_monotonic_store:
6705; GFX10-CU:       ; %bb.0: ; %entry
6706; GFX10-CU-NEXT:    s_clause 0x1
6707; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6708; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
6709; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6710; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6711; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6712; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6713; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6714; GFX10-CU-NEXT:    s_endpgm
6715;
6716; SKIP-CACHE-INV-LABEL: flat_system_one_as_monotonic_store:
6717; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6718; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
6719; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
6720; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6721; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6722; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6723; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6724; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6725; SKIP-CACHE-INV-NEXT:    s_endpgm
6726;
6727; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_store:
6728; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6729; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6730; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
6731; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6732; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6733; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6734; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6735; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6736;
6737; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_store:
6738; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6739; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6740; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
6741; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6742; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6743; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6744; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6745; GFX90A-TGSPLIT-NEXT:    s_endpgm
6746;
6747; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_store:
6748; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6749; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
6750; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
6751; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6752; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6753; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
6754; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
6755; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6756;
6757; GFX940-TGSPLIT-LABEL: flat_system_one_as_monotonic_store:
6758; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6759; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
6760; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
6761; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6762; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6763; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
6764; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
6765; GFX940-TGSPLIT-NEXT:    s_endpgm
6766    i32 %in, i32* %out) {
6767entry:
6768  store atomic i32 %in, i32* %out syncscope("one-as") monotonic, align 4
6769  ret void
6770}
6771
6772define amdgpu_kernel void @flat_system_one_as_release_store(
6773; GFX7-LABEL: flat_system_one_as_release_store:
6774; GFX7:       ; %bb.0: ; %entry
6775; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
6776; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
6777; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6778; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6779; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6780; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6781; GFX7-NEXT:    s_waitcnt vmcnt(0)
6782; GFX7-NEXT:    flat_store_dword v[0:1], v2
6783; GFX7-NEXT:    s_endpgm
6784;
6785; GFX10-WGP-LABEL: flat_system_one_as_release_store:
6786; GFX10-WGP:       ; %bb.0: ; %entry
6787; GFX10-WGP-NEXT:    s_clause 0x1
6788; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6789; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
6790; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6791; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6792; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6793; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6794; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6795; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6796; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6797; GFX10-WGP-NEXT:    s_endpgm
6798;
6799; GFX10-CU-LABEL: flat_system_one_as_release_store:
6800; GFX10-CU:       ; %bb.0: ; %entry
6801; GFX10-CU-NEXT:    s_clause 0x1
6802; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6803; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
6804; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6805; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6806; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6807; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6808; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6809; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6810; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6811; GFX10-CU-NEXT:    s_endpgm
6812;
6813; SKIP-CACHE-INV-LABEL: flat_system_one_as_release_store:
6814; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6815; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
6816; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
6817; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6818; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6819; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6820; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6821; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6822; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6823; SKIP-CACHE-INV-NEXT:    s_endpgm
6824;
6825; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_release_store:
6826; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6827; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6828; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
6829; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6830; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6831; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6832; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
6833; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6834; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6835; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6836;
6837; GFX90A-TGSPLIT-LABEL: flat_system_one_as_release_store:
6838; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6839; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6840; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
6841; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6842; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6843; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6844; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
6845; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6846; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6847; GFX90A-TGSPLIT-NEXT:    s_endpgm
6848;
6849; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_release_store:
6850; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6851; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
6852; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
6853; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6854; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6855; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
6856; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
6857; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6858; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
6859; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6860;
6861; GFX940-TGSPLIT-LABEL: flat_system_one_as_release_store:
6862; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6863; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
6864; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
6865; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6866; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6867; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
6868; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
6869; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6870; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
6871; GFX940-TGSPLIT-NEXT:    s_endpgm
6872    i32 %in, i32* %out) {
6873entry:
6874  store atomic i32 %in, i32* %out syncscope("one-as") release, align 4
6875  ret void
6876}
6877
6878define amdgpu_kernel void @flat_system_one_as_seq_cst_store(
6879; GFX7-LABEL: flat_system_one_as_seq_cst_store:
6880; GFX7:       ; %bb.0: ; %entry
6881; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
6882; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
6883; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6884; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6885; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6886; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6887; GFX7-NEXT:    s_waitcnt vmcnt(0)
6888; GFX7-NEXT:    flat_store_dword v[0:1], v2
6889; GFX7-NEXT:    s_endpgm
6890;
6891; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_store:
6892; GFX10-WGP:       ; %bb.0: ; %entry
6893; GFX10-WGP-NEXT:    s_clause 0x1
6894; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6895; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
6896; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6897; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6898; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6899; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
6900; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6901; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6902; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
6903; GFX10-WGP-NEXT:    s_endpgm
6904;
6905; GFX10-CU-LABEL: flat_system_one_as_seq_cst_store:
6906; GFX10-CU:       ; %bb.0: ; %entry
6907; GFX10-CU-NEXT:    s_clause 0x1
6908; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6909; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
6910; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6911; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6912; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6913; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
6914; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6915; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6916; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
6917; GFX10-CU-NEXT:    s_endpgm
6918;
6919; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_store:
6920; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6921; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
6922; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
6923; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6924; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
6925; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
6926; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
6927; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6928; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
6929; SKIP-CACHE-INV-NEXT:    s_endpgm
6930;
6931; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_store:
6932; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6933; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6934; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
6935; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6936; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6937; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6938; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
6939; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6940; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6941; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6942;
6943; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_store:
6944; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6945; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6946; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
6947; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6948; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6949; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
6950; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
6951; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6952; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
6953; GFX90A-TGSPLIT-NEXT:    s_endpgm
6954;
6955; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_store:
6956; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
6957; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
6958; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
6959; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6960; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6961; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
6962; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
6963; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6964; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
6965; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
6966;
6967; GFX940-TGSPLIT-LABEL: flat_system_one_as_seq_cst_store:
6968; GFX940-TGSPLIT:       ; %bb.0: ; %entry
6969; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
6970; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x0
6971; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6972; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
6973; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
6974; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
6975; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6976; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2 sc0 sc1
6977; GFX940-TGSPLIT-NEXT:    s_endpgm
6978    i32 %in, i32* %out) {
6979entry:
6980  store atomic i32 %in, i32* %out syncscope("one-as") seq_cst, align 4
6981  ret void
6982}
6983
6984define amdgpu_kernel void @flat_system_one_as_monotonic_atomicrmw(
6985; GFX7-LABEL: flat_system_one_as_monotonic_atomicrmw:
6986; GFX7:       ; %bb.0: ; %entry
6987; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6988; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
6989; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6990; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6991; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6992; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6993; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
6994; GFX7-NEXT:    s_endpgm
6995;
6996; GFX10-WGP-LABEL: flat_system_one_as_monotonic_atomicrmw:
6997; GFX10-WGP:       ; %bb.0: ; %entry
6998; GFX10-WGP-NEXT:    s_clause 0x1
6999; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7000; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7001; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7002; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7003; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7004; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7005; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
7006; GFX10-WGP-NEXT:    s_endpgm
7007;
7008; GFX10-CU-LABEL: flat_system_one_as_monotonic_atomicrmw:
7009; GFX10-CU:       ; %bb.0: ; %entry
7010; GFX10-CU-NEXT:    s_clause 0x1
7011; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7012; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7013; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7014; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7015; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7016; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7017; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
7018; GFX10-CU-NEXT:    s_endpgm
7019;
7020; SKIP-CACHE-INV-LABEL: flat_system_one_as_monotonic_atomicrmw:
7021; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7022; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7023; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
7024; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7025; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7026; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7027; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7028; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
7029; SKIP-CACHE-INV-NEXT:    s_endpgm
7030;
7031; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_atomicrmw:
7032; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7033; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7034; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7035; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7036; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7037; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7038; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7039; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7040;
7041; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_atomicrmw:
7042; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7043; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7044; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7045; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7046; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7047; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7048; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7049; GFX90A-TGSPLIT-NEXT:    s_endpgm
7050;
7051; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_atomicrmw:
7052; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7053; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7054; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7055; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7056; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7057; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7058; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
7059; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7060;
7061; GFX940-TGSPLIT-LABEL: flat_system_one_as_monotonic_atomicrmw:
7062; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7063; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7064; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7065; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7066; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7067; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7068; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
7069; GFX940-TGSPLIT-NEXT:    s_endpgm
7070    i32* %out, i32 %in) {
7071entry:
7072  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") monotonic
7073  ret void
7074}
7075
7076define amdgpu_kernel void @flat_system_one_as_acquire_atomicrmw(
7077; GFX7-LABEL: flat_system_one_as_acquire_atomicrmw:
7078; GFX7:       ; %bb.0: ; %entry
7079; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7080; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7081; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7082; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7083; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7084; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7085; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
7086; GFX7-NEXT:    s_waitcnt vmcnt(0)
7087; GFX7-NEXT:    buffer_wbinvl1_vol
7088; GFX7-NEXT:    s_endpgm
7089;
7090; GFX10-WGP-LABEL: flat_system_one_as_acquire_atomicrmw:
7091; GFX10-WGP:       ; %bb.0: ; %entry
7092; GFX10-WGP-NEXT:    s_clause 0x1
7093; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7094; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7095; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7096; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7097; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7098; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7099; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
7100; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7101; GFX10-WGP-NEXT:    buffer_gl0_inv
7102; GFX10-WGP-NEXT:    buffer_gl1_inv
7103; GFX10-WGP-NEXT:    s_endpgm
7104;
7105; GFX10-CU-LABEL: flat_system_one_as_acquire_atomicrmw:
7106; GFX10-CU:       ; %bb.0: ; %entry
7107; GFX10-CU-NEXT:    s_clause 0x1
7108; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7109; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7110; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7111; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7112; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7113; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7114; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
7115; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7116; GFX10-CU-NEXT:    buffer_gl0_inv
7117; GFX10-CU-NEXT:    buffer_gl1_inv
7118; GFX10-CU-NEXT:    s_endpgm
7119;
7120; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_atomicrmw:
7121; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7122; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7123; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
7124; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7125; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7126; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7127; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7128; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
7129; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7130; SKIP-CACHE-INV-NEXT:    s_endpgm
7131;
7132; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_atomicrmw:
7133; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7134; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7135; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7136; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7137; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7138; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7139; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7140; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7141; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
7142; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7143; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7144;
7145; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_atomicrmw:
7146; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7147; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7148; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7149; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7150; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7151; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7152; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7153; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7154; GFX90A-TGSPLIT-NEXT:    buffer_invl2
7155; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7156; GFX90A-TGSPLIT-NEXT:    s_endpgm
7157;
7158; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_atomicrmw:
7159; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7160; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7161; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7162; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7163; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7164; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7165; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
7166; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7167; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
7168; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7169;
7170; GFX940-TGSPLIT-LABEL: flat_system_one_as_acquire_atomicrmw:
7171; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7172; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7173; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7174; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7175; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7176; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7177; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
7178; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7179; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
7180; GFX940-TGSPLIT-NEXT:    s_endpgm
7181    i32* %out, i32 %in) {
7182entry:
7183  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") acquire
7184  ret void
7185}
7186
7187define amdgpu_kernel void @flat_system_one_as_release_atomicrmw(
7188; GFX7-LABEL: flat_system_one_as_release_atomicrmw:
7189; GFX7:       ; %bb.0: ; %entry
7190; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7191; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7192; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7193; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7194; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7195; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7196; GFX7-NEXT:    s_waitcnt vmcnt(0)
7197; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
7198; GFX7-NEXT:    s_endpgm
7199;
7200; GFX10-WGP-LABEL: flat_system_one_as_release_atomicrmw:
7201; GFX10-WGP:       ; %bb.0: ; %entry
7202; GFX10-WGP-NEXT:    s_clause 0x1
7203; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7204; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7205; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7206; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7207; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7208; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7209; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7210; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7211; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
7212; GFX10-WGP-NEXT:    s_endpgm
7213;
7214; GFX10-CU-LABEL: flat_system_one_as_release_atomicrmw:
7215; GFX10-CU:       ; %bb.0: ; %entry
7216; GFX10-CU-NEXT:    s_clause 0x1
7217; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7218; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7219; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7220; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7221; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7222; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7223; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7224; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7225; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
7226; GFX10-CU-NEXT:    s_endpgm
7227;
7228; SKIP-CACHE-INV-LABEL: flat_system_one_as_release_atomicrmw:
7229; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7230; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7231; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
7232; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7233; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7234; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7235; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7236; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7237; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
7238; SKIP-CACHE-INV-NEXT:    s_endpgm
7239;
7240; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_release_atomicrmw:
7241; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7242; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7243; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7244; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7245; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7246; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7247; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
7248; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7249; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7250; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7251;
7252; GFX90A-TGSPLIT-LABEL: flat_system_one_as_release_atomicrmw:
7253; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7254; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7255; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7256; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7257; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7258; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7259; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
7260; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7261; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7262; GFX90A-TGSPLIT-NEXT:    s_endpgm
7263;
7264; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_release_atomicrmw:
7265; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7266; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7267; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7268; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7269; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7270; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7271; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
7272; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7273; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
7274; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7275;
7276; GFX940-TGSPLIT-LABEL: flat_system_one_as_release_atomicrmw:
7277; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7278; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7279; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7280; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7281; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7282; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7283; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
7284; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7285; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
7286; GFX940-TGSPLIT-NEXT:    s_endpgm
7287    i32* %out, i32 %in) {
7288entry:
7289  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") release
7290  ret void
7291}
7292
7293define amdgpu_kernel void @flat_system_one_as_acq_rel_atomicrmw(
7294; GFX7-LABEL: flat_system_one_as_acq_rel_atomicrmw:
7295; GFX7:       ; %bb.0: ; %entry
7296; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7297; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7298; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7299; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7300; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7301; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7302; GFX7-NEXT:    s_waitcnt vmcnt(0)
7303; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
7304; GFX7-NEXT:    s_waitcnt vmcnt(0)
7305; GFX7-NEXT:    buffer_wbinvl1_vol
7306; GFX7-NEXT:    s_endpgm
7307;
7308; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_atomicrmw:
7309; GFX10-WGP:       ; %bb.0: ; %entry
7310; GFX10-WGP-NEXT:    s_clause 0x1
7311; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7312; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7313; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7314; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7315; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7316; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7317; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7318; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7319; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
7320; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7321; GFX10-WGP-NEXT:    buffer_gl0_inv
7322; GFX10-WGP-NEXT:    buffer_gl1_inv
7323; GFX10-WGP-NEXT:    s_endpgm
7324;
7325; GFX10-CU-LABEL: flat_system_one_as_acq_rel_atomicrmw:
7326; GFX10-CU:       ; %bb.0: ; %entry
7327; GFX10-CU-NEXT:    s_clause 0x1
7328; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7329; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7330; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7331; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7332; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7333; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7334; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7335; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7336; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
7337; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7338; GFX10-CU-NEXT:    buffer_gl0_inv
7339; GFX10-CU-NEXT:    buffer_gl1_inv
7340; GFX10-CU-NEXT:    s_endpgm
7341;
7342; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_atomicrmw:
7343; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7344; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7345; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
7346; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7347; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7348; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7349; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7350; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7351; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
7352; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7353; SKIP-CACHE-INV-NEXT:    s_endpgm
7354;
7355; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_atomicrmw:
7356; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7357; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7358; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7359; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7360; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7361; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7362; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
7363; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7364; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7365; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7366; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
7367; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7368; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7369;
7370; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_atomicrmw:
7371; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7372; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7373; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7374; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7375; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7376; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7377; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
7378; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7379; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7380; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7381; GFX90A-TGSPLIT-NEXT:    buffer_invl2
7382; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7383; GFX90A-TGSPLIT-NEXT:    s_endpgm
7384;
7385; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_atomicrmw:
7386; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7387; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7388; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7389; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7390; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7391; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7392; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
7393; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7394; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
7395; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7396; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
7397; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7398;
7399; GFX940-TGSPLIT-LABEL: flat_system_one_as_acq_rel_atomicrmw:
7400; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7401; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7402; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7403; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7404; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7405; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7406; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
7407; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7408; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
7409; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7410; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
7411; GFX940-TGSPLIT-NEXT:    s_endpgm
7412    i32* %out, i32 %in) {
7413entry:
7414  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") acq_rel
7415  ret void
7416}
7417
7418define amdgpu_kernel void @flat_system_one_as_seq_cst_atomicrmw(
7419; GFX7-LABEL: flat_system_one_as_seq_cst_atomicrmw:
7420; GFX7:       ; %bb.0: ; %entry
7421; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7422; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7423; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7424; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7425; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7426; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7427; GFX7-NEXT:    s_waitcnt vmcnt(0)
7428; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
7429; GFX7-NEXT:    s_waitcnt vmcnt(0)
7430; GFX7-NEXT:    buffer_wbinvl1_vol
7431; GFX7-NEXT:    s_endpgm
7432;
7433; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_atomicrmw:
7434; GFX10-WGP:       ; %bb.0: ; %entry
7435; GFX10-WGP-NEXT:    s_clause 0x1
7436; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7437; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7438; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7439; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7440; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7441; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7442; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7443; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7444; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
7445; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7446; GFX10-WGP-NEXT:    buffer_gl0_inv
7447; GFX10-WGP-NEXT:    buffer_gl1_inv
7448; GFX10-WGP-NEXT:    s_endpgm
7449;
7450; GFX10-CU-LABEL: flat_system_one_as_seq_cst_atomicrmw:
7451; GFX10-CU:       ; %bb.0: ; %entry
7452; GFX10-CU-NEXT:    s_clause 0x1
7453; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7454; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7455; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7456; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7457; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7458; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7459; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7460; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7461; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
7462; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7463; GFX10-CU-NEXT:    buffer_gl0_inv
7464; GFX10-CU-NEXT:    buffer_gl1_inv
7465; GFX10-CU-NEXT:    s_endpgm
7466;
7467; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_atomicrmw:
7468; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7469; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7470; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
7471; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7472; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7473; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7474; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7475; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7476; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
7477; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7478; SKIP-CACHE-INV-NEXT:    s_endpgm
7479;
7480; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_atomicrmw:
7481; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7482; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7483; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7484; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7485; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7486; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7487; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
7488; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7489; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7490; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7491; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
7492; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7493; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7494;
7495; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_atomicrmw:
7496; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7497; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7498; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7499; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7500; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7501; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7502; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
7503; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7504; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2
7505; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7506; GFX90A-TGSPLIT-NEXT:    buffer_invl2
7507; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7508; GFX90A-TGSPLIT-NEXT:    s_endpgm
7509;
7510; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_atomicrmw:
7511; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7512; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7513; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7514; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7515; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7516; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7517; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
7518; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7519; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
7520; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7521; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
7522; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7523;
7524; GFX940-TGSPLIT-LABEL: flat_system_one_as_seq_cst_atomicrmw:
7525; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7526; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7527; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7528; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7529; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7530; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7531; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
7532; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7533; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v[0:1], v2 sc1
7534; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7535; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
7536; GFX940-TGSPLIT-NEXT:    s_endpgm
7537    i32* %out, i32 %in) {
7538entry:
7539  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") seq_cst
7540  ret void
7541}
7542
7543define amdgpu_kernel void @flat_system_one_as_acquire_ret_atomicrmw(
7544; GFX7-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
7545; GFX7:       ; %bb.0: ; %entry
7546; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7547; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7548; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7549; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7550; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7551; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7552; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7553; GFX7-NEXT:    s_waitcnt vmcnt(0)
7554; GFX7-NEXT:    buffer_wbinvl1_vol
7555; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7556; GFX7-NEXT:    flat_store_dword v[0:1], v2
7557; GFX7-NEXT:    s_endpgm
7558;
7559; GFX10-WGP-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
7560; GFX10-WGP:       ; %bb.0: ; %entry
7561; GFX10-WGP-NEXT:    s_clause 0x1
7562; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7563; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7564; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7565; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7566; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7567; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7568; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7569; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7570; GFX10-WGP-NEXT:    buffer_gl0_inv
7571; GFX10-WGP-NEXT:    buffer_gl1_inv
7572; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7573; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7574; GFX10-WGP-NEXT:    s_endpgm
7575;
7576; GFX10-CU-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
7577; GFX10-CU:       ; %bb.0: ; %entry
7578; GFX10-CU-NEXT:    s_clause 0x1
7579; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7580; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7581; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7582; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7583; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7584; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7585; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7586; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7587; GFX10-CU-NEXT:    buffer_gl0_inv
7588; GFX10-CU-NEXT:    buffer_gl1_inv
7589; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7590; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7591; GFX10-CU-NEXT:    s_endpgm
7592;
7593; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
7594; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7595; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7596; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
7597; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7598; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7599; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7600; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7601; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7602; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7603; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7604; SKIP-CACHE-INV-NEXT:    s_endpgm
7605;
7606; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
7607; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7608; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7609; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7610; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7611; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7612; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7613; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7614; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7615; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
7616; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7617; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7618; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7619; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7620;
7621; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
7622; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7623; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7624; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7625; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7626; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7627; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7628; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7629; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7630; GFX90A-TGSPLIT-NEXT:    buffer_invl2
7631; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7632; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7633; GFX90A-TGSPLIT-NEXT:    s_endpgm
7634;
7635; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
7636; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7637; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7638; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7639; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7640; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7641; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7642; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0 sc1
7643; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7644; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
7645; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7646; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7647; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7648;
7649; GFX940-TGSPLIT-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
7650; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7651; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7652; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7653; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7654; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7655; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7656; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0 sc1
7657; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7658; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
7659; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7660; GFX940-TGSPLIT-NEXT:    s_endpgm
7661    i32* %out, i32 %in) {
7662entry:
7663  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") acquire
7664  store i32 %val, i32* %out, align 4
7665  ret void
7666}
7667
7668define amdgpu_kernel void @flat_system_one_as_acq_rel_ret_atomicrmw(
7669; GFX7-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
7670; GFX7:       ; %bb.0: ; %entry
7671; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7672; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7673; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7674; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7675; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7676; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7677; GFX7-NEXT:    s_waitcnt vmcnt(0)
7678; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7679; GFX7-NEXT:    s_waitcnt vmcnt(0)
7680; GFX7-NEXT:    buffer_wbinvl1_vol
7681; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7682; GFX7-NEXT:    flat_store_dword v[0:1], v2
7683; GFX7-NEXT:    s_endpgm
7684;
7685; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
7686; GFX10-WGP:       ; %bb.0: ; %entry
7687; GFX10-WGP-NEXT:    s_clause 0x1
7688; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7689; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7690; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7691; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7692; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7693; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7694; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7695; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7696; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7697; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7698; GFX10-WGP-NEXT:    buffer_gl0_inv
7699; GFX10-WGP-NEXT:    buffer_gl1_inv
7700; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7701; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7702; GFX10-WGP-NEXT:    s_endpgm
7703;
7704; GFX10-CU-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
7705; GFX10-CU:       ; %bb.0: ; %entry
7706; GFX10-CU-NEXT:    s_clause 0x1
7707; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7708; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7709; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7710; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7711; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7712; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7713; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7714; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7715; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7716; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7717; GFX10-CU-NEXT:    buffer_gl0_inv
7718; GFX10-CU-NEXT:    buffer_gl1_inv
7719; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7720; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7721; GFX10-CU-NEXT:    s_endpgm
7722;
7723; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
7724; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7725; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7726; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
7727; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7728; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7729; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7730; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7731; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7732; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7733; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7734; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7735; SKIP-CACHE-INV-NEXT:    s_endpgm
7736;
7737; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
7738; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7739; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7740; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7741; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7742; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7743; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7744; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
7745; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7746; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7747; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7748; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
7749; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7750; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7751; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7752; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7753;
7754; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
7755; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7756; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7757; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7758; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7759; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7760; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7761; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
7762; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7763; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7764; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7765; GFX90A-TGSPLIT-NEXT:    buffer_invl2
7766; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7767; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7768; GFX90A-TGSPLIT-NEXT:    s_endpgm
7769;
7770; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
7771; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7772; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7773; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7774; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7775; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7776; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7777; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
7778; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7779; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0 sc1
7780; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7781; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
7782; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7783; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7784; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7785;
7786; GFX940-TGSPLIT-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
7787; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7788; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7789; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7790; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7791; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7792; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7793; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
7794; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7795; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0 sc1
7796; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7797; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
7798; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7799; GFX940-TGSPLIT-NEXT:    s_endpgm
7800    i32* %out, i32 %in) {
7801entry:
7802  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") acq_rel
7803  store i32 %val, i32* %out, align 4
7804  ret void
7805}
7806
7807define amdgpu_kernel void @flat_system_one_as_seq_cst_ret_atomicrmw(
7808; GFX7-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
7809; GFX7:       ; %bb.0: ; %entry
7810; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7811; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
7812; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7813; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7814; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7815; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7816; GFX7-NEXT:    s_waitcnt vmcnt(0)
7817; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7818; GFX7-NEXT:    s_waitcnt vmcnt(0)
7819; GFX7-NEXT:    buffer_wbinvl1_vol
7820; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7821; GFX7-NEXT:    flat_store_dword v[0:1], v2
7822; GFX7-NEXT:    s_endpgm
7823;
7824; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
7825; GFX10-WGP:       ; %bb.0: ; %entry
7826; GFX10-WGP-NEXT:    s_clause 0x1
7827; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7828; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
7829; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7830; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7831; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7832; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7833; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7834; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7835; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7836; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7837; GFX10-WGP-NEXT:    buffer_gl0_inv
7838; GFX10-WGP-NEXT:    buffer_gl1_inv
7839; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7840; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
7841; GFX10-WGP-NEXT:    s_endpgm
7842;
7843; GFX10-CU-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
7844; GFX10-CU:       ; %bb.0: ; %entry
7845; GFX10-CU-NEXT:    s_clause 0x1
7846; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7847; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
7848; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7849; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7850; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7851; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7852; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7853; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7854; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7855; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7856; GFX10-CU-NEXT:    buffer_gl0_inv
7857; GFX10-CU-NEXT:    buffer_gl1_inv
7858; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7859; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
7860; GFX10-CU-NEXT:    s_endpgm
7861;
7862; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
7863; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7864; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7865; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x2
7866; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7867; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7868; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
7869; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
7870; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7871; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7872; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
7873; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
7874; SKIP-CACHE-INV-NEXT:    s_endpgm
7875;
7876; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
7877; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7878; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7879; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7880; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7881; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7882; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7883; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
7884; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7885; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7886; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7887; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
7888; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7889; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7890; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7891; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7892;
7893; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
7894; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7895; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7896; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
7897; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7898; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7899; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s2
7900; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
7901; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7902; GFX90A-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
7903; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7904; GFX90A-TGSPLIT-NEXT:    buffer_invl2
7905; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7906; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7907; GFX90A-TGSPLIT-NEXT:    s_endpgm
7908;
7909; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
7910; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
7911; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7912; GFX940-NOTTGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7913; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7914; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7915; GFX940-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7916; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
7917; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7918; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0 sc1
7919; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7920; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
7921; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7922; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7923; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
7924;
7925; GFX940-TGSPLIT-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
7926; GFX940-TGSPLIT:       ; %bb.0: ; %entry
7927; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7928; GFX940-TGSPLIT-NEXT:    s_load_dword s4, s[0:1], 0x8
7929; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7930; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
7931; GFX940-TGSPLIT-NEXT:    v_mov_b32_e32 v2, s4
7932; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
7933; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7934; GFX940-TGSPLIT-NEXT:    flat_atomic_swap v2, v[0:1], v2 sc0 sc1
7935; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7936; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
7937; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
7938; GFX940-TGSPLIT-NEXT:    s_endpgm
7939    i32* %out, i32 %in) {
7940entry:
7941  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") seq_cst
7942  store i32 %val, i32* %out, align 4
7943  ret void
7944}
7945
7946define amdgpu_kernel void @flat_system_one_as_monotonic_monotonic_cmpxchg(
7947; GFX7-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
7948; GFX7:       ; %bb.0: ; %entry
7949; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7950; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7951; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7952; GFX7-NEXT:    s_add_u32 s0, s0, 16
7953; GFX7-NEXT:    s_addc_u32 s1, s1, 0
7954; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7955; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7956; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7957; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7958; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7959; GFX7-NEXT:    s_endpgm
7960;
7961; GFX10-WGP-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
7962; GFX10-WGP:       ; %bb.0: ; %entry
7963; GFX10-WGP-NEXT:    s_clause 0x1
7964; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7965; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7966; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7967; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
7968; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
7969; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7970; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
7971; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7972; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
7973; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7974; GFX10-WGP-NEXT:    s_endpgm
7975;
7976; GFX10-CU-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
7977; GFX10-CU:       ; %bb.0: ; %entry
7978; GFX10-CU-NEXT:    s_clause 0x1
7979; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7980; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7981; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7982; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
7983; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
7984; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7985; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
7986; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7987; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
7988; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7989; GFX10-CU-NEXT:    s_endpgm
7990;
7991; SKIP-CACHE-INV-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
7992; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7993; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
7994; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
7995; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7996; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
7997; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
7998; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
7999; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8000; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8001; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8002; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8003; SKIP-CACHE-INV-NEXT:    s_endpgm
8004;
8005; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
8006; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8007; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8008; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8009; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8010; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8011; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8012; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8013; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8014;
8015; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
8016; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8017; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8018; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8019; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8020; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8021; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8022; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8023; GFX90A-TGSPLIT-NEXT:    s_endpgm
8024;
8025; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
8026; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8027; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8028; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8029; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8030; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8031; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8032; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
8033; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8034;
8035; GFX940-TGSPLIT-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
8036; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8037; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8038; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8039; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8040; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8041; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8042; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
8043; GFX940-TGSPLIT-NEXT:    s_endpgm
8044    i32* %out, i32 %in, i32 %old) {
8045entry:
8046  %gep = getelementptr i32, i32* %out, i32 4
8047  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") monotonic monotonic
8048  ret void
8049}
8050
8051define amdgpu_kernel void @flat_system_one_as_acquire_monotonic_cmpxchg(
8052; GFX7-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
8053; GFX7:       ; %bb.0: ; %entry
8054; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8055; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8056; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8057; GFX7-NEXT:    s_add_u32 s0, s0, 16
8058; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8059; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8060; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8061; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8062; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8063; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8064; GFX7-NEXT:    s_waitcnt vmcnt(0)
8065; GFX7-NEXT:    buffer_wbinvl1_vol
8066; GFX7-NEXT:    s_endpgm
8067;
8068; GFX10-WGP-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
8069; GFX10-WGP:       ; %bb.0: ; %entry
8070; GFX10-WGP-NEXT:    s_clause 0x1
8071; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8072; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8073; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8074; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
8075; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
8076; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8077; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8078; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8079; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8080; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8081; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8082; GFX10-WGP-NEXT:    buffer_gl0_inv
8083; GFX10-WGP-NEXT:    buffer_gl1_inv
8084; GFX10-WGP-NEXT:    s_endpgm
8085;
8086; GFX10-CU-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
8087; GFX10-CU:       ; %bb.0: ; %entry
8088; GFX10-CU-NEXT:    s_clause 0x1
8089; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8090; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8091; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8092; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8093; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8094; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8095; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8096; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8097; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8098; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8099; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8100; GFX10-CU-NEXT:    buffer_gl0_inv
8101; GFX10-CU-NEXT:    buffer_gl1_inv
8102; GFX10-CU-NEXT:    s_endpgm
8103;
8104; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
8105; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8106; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8107; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8108; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8109; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
8110; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
8111; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8112; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8113; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8114; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8115; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8116; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8117; SKIP-CACHE-INV-NEXT:    s_endpgm
8118;
8119; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
8120; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8121; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8122; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8123; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8124; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8125; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8126; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8127; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8128; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
8129; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
8130; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8131;
8132; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
8133; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8134; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8135; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8136; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8137; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8138; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8139; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8140; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8141; GFX90A-TGSPLIT-NEXT:    buffer_invl2
8142; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8143; GFX90A-TGSPLIT-NEXT:    s_endpgm
8144;
8145; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
8146; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8147; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8148; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8149; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8150; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8151; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8152; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
8153; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8154; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
8155; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8156;
8157; GFX940-TGSPLIT-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
8158; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8159; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8160; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8161; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8162; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8163; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8164; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
8165; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8166; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
8167; GFX940-TGSPLIT-NEXT:    s_endpgm
8168    i32* %out, i32 %in, i32 %old) {
8169entry:
8170  %gep = getelementptr i32, i32* %out, i32 4
8171  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acquire monotonic
8172  ret void
8173}
8174
8175define amdgpu_kernel void @flat_system_one_as_release_monotonic_cmpxchg(
8176; GFX7-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
8177; GFX7:       ; %bb.0: ; %entry
8178; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8179; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8180; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8181; GFX7-NEXT:    s_add_u32 s0, s0, 16
8182; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8183; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8184; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8185; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8186; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8187; GFX7-NEXT:    s_waitcnt vmcnt(0)
8188; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8189; GFX7-NEXT:    s_endpgm
8190;
8191; GFX10-WGP-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
8192; GFX10-WGP:       ; %bb.0: ; %entry
8193; GFX10-WGP-NEXT:    s_clause 0x1
8194; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8195; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8196; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8197; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
8198; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
8199; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8200; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8201; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8202; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8203; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8204; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8205; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8206; GFX10-WGP-NEXT:    s_endpgm
8207;
8208; GFX10-CU-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
8209; GFX10-CU:       ; %bb.0: ; %entry
8210; GFX10-CU-NEXT:    s_clause 0x1
8211; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8212; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8213; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8214; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8215; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8216; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8217; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8218; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8219; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8220; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8221; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8222; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8223; GFX10-CU-NEXT:    s_endpgm
8224;
8225; SKIP-CACHE-INV-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
8226; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8227; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8228; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8229; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8230; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
8231; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
8232; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8233; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8234; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8235; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8236; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8237; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8238; SKIP-CACHE-INV-NEXT:    s_endpgm
8239;
8240; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
8241; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8242; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8243; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8244; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8245; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8246; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8247; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
8248; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8249; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8250; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8251;
8252; GFX90A-TGSPLIT-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
8253; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8254; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8255; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8256; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8257; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8258; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8259; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
8260; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8261; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8262; GFX90A-TGSPLIT-NEXT:    s_endpgm
8263;
8264; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
8265; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8266; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8267; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8268; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8269; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8270; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8271; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
8272; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8273; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
8274; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8275;
8276; GFX940-TGSPLIT-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
8277; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8278; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8279; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8280; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8281; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8282; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8283; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
8284; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8285; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
8286; GFX940-TGSPLIT-NEXT:    s_endpgm
8287    i32* %out, i32 %in, i32 %old) {
8288entry:
8289  %gep = getelementptr i32, i32* %out, i32 4
8290  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") release monotonic
8291  ret void
8292}
8293
8294define amdgpu_kernel void @flat_system_one_as_acq_rel_monotonic_cmpxchg(
8295; GFX7-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
8296; GFX7:       ; %bb.0: ; %entry
8297; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8298; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8299; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8300; GFX7-NEXT:    s_add_u32 s0, s0, 16
8301; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8302; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8303; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8304; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8305; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8306; GFX7-NEXT:    s_waitcnt vmcnt(0)
8307; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8308; GFX7-NEXT:    s_waitcnt vmcnt(0)
8309; GFX7-NEXT:    buffer_wbinvl1_vol
8310; GFX7-NEXT:    s_endpgm
8311;
8312; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
8313; GFX10-WGP:       ; %bb.0: ; %entry
8314; GFX10-WGP-NEXT:    s_clause 0x1
8315; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8316; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8317; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8318; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
8319; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
8320; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8321; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8322; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8323; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8324; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8325; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8326; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8327; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8328; GFX10-WGP-NEXT:    buffer_gl0_inv
8329; GFX10-WGP-NEXT:    buffer_gl1_inv
8330; GFX10-WGP-NEXT:    s_endpgm
8331;
8332; GFX10-CU-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
8333; GFX10-CU:       ; %bb.0: ; %entry
8334; GFX10-CU-NEXT:    s_clause 0x1
8335; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8336; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8337; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8338; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8339; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8340; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8341; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8342; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8343; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8344; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8345; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8346; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8347; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8348; GFX10-CU-NEXT:    buffer_gl0_inv
8349; GFX10-CU-NEXT:    buffer_gl1_inv
8350; GFX10-CU-NEXT:    s_endpgm
8351;
8352; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
8353; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8354; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8355; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8356; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8357; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
8358; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
8359; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8360; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8361; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8362; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8363; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8364; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8365; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8366; SKIP-CACHE-INV-NEXT:    s_endpgm
8367;
8368; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
8369; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8370; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8371; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8372; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8373; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8374; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8375; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
8376; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8377; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8378; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8379; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
8380; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
8381; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8382;
8383; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
8384; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8385; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8386; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8387; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8388; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8389; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8390; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
8391; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8392; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8393; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8394; GFX90A-TGSPLIT-NEXT:    buffer_invl2
8395; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8396; GFX90A-TGSPLIT-NEXT:    s_endpgm
8397;
8398; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
8399; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8400; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8401; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8402; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8403; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8404; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8405; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
8406; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8407; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
8408; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8409; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
8410; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8411;
8412; GFX940-TGSPLIT-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
8413; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8414; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8415; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8416; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8417; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8418; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8419; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
8420; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8421; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
8422; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8423; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
8424; GFX940-TGSPLIT-NEXT:    s_endpgm
8425    i32* %out, i32 %in, i32 %old) {
8426entry:
8427  %gep = getelementptr i32, i32* %out, i32 4
8428  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel monotonic
8429  ret void
8430}
8431
8432define amdgpu_kernel void @flat_system_one_as_seq_cst_monotonic_cmpxchg(
8433; GFX7-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
8434; GFX7:       ; %bb.0: ; %entry
8435; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8436; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8437; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8438; GFX7-NEXT:    s_add_u32 s0, s0, 16
8439; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8440; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8441; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8442; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8443; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8444; GFX7-NEXT:    s_waitcnt vmcnt(0)
8445; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8446; GFX7-NEXT:    s_waitcnt vmcnt(0)
8447; GFX7-NEXT:    buffer_wbinvl1_vol
8448; GFX7-NEXT:    s_endpgm
8449;
8450; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
8451; GFX10-WGP:       ; %bb.0: ; %entry
8452; GFX10-WGP-NEXT:    s_clause 0x1
8453; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8454; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8455; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8456; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
8457; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
8458; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8459; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8460; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8461; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8462; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8463; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8464; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8465; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8466; GFX10-WGP-NEXT:    buffer_gl0_inv
8467; GFX10-WGP-NEXT:    buffer_gl1_inv
8468; GFX10-WGP-NEXT:    s_endpgm
8469;
8470; GFX10-CU-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
8471; GFX10-CU:       ; %bb.0: ; %entry
8472; GFX10-CU-NEXT:    s_clause 0x1
8473; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8474; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8475; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8476; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8477; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8478; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8479; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8480; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8481; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8482; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8483; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8484; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8485; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8486; GFX10-CU-NEXT:    buffer_gl0_inv
8487; GFX10-CU-NEXT:    buffer_gl1_inv
8488; GFX10-CU-NEXT:    s_endpgm
8489;
8490; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
8491; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8492; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8493; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8494; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8495; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
8496; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
8497; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8498; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8499; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8500; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8501; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8502; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8503; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8504; SKIP-CACHE-INV-NEXT:    s_endpgm
8505;
8506; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
8507; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8508; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8509; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8510; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8511; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8512; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8513; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
8514; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8515; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8516; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8517; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
8518; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
8519; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8520;
8521; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
8522; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8523; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8524; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8525; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8526; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8527; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8528; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
8529; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8530; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8531; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8532; GFX90A-TGSPLIT-NEXT:    buffer_invl2
8533; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8534; GFX90A-TGSPLIT-NEXT:    s_endpgm
8535;
8536; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
8537; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8538; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8539; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8540; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8541; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8542; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8543; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
8544; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8545; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
8546; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8547; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
8548; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8549;
8550; GFX940-TGSPLIT-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
8551; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8552; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8553; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8554; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8555; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8556; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8557; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
8558; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8559; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
8560; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8561; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
8562; GFX940-TGSPLIT-NEXT:    s_endpgm
8563    i32* %out, i32 %in, i32 %old) {
8564entry:
8565  %gep = getelementptr i32, i32* %out, i32 4
8566  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst monotonic
8567  ret void
8568}
8569
8570define amdgpu_kernel void @flat_system_one_as_monotonic_acquire_cmpxchg(
8571; GFX7-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
8572; GFX7:       ; %bb.0: ; %entry
8573; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8574; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8575; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8576; GFX7-NEXT:    s_add_u32 s0, s0, 16
8577; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8578; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8579; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8580; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8581; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8582; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8583; GFX7-NEXT:    s_waitcnt vmcnt(0)
8584; GFX7-NEXT:    buffer_wbinvl1_vol
8585; GFX7-NEXT:    s_endpgm
8586;
8587; GFX10-WGP-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
8588; GFX10-WGP:       ; %bb.0: ; %entry
8589; GFX10-WGP-NEXT:    s_clause 0x1
8590; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8591; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8592; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8593; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
8594; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
8595; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8596; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8597; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8598; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8599; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8600; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8601; GFX10-WGP-NEXT:    buffer_gl0_inv
8602; GFX10-WGP-NEXT:    buffer_gl1_inv
8603; GFX10-WGP-NEXT:    s_endpgm
8604;
8605; GFX10-CU-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
8606; GFX10-CU:       ; %bb.0: ; %entry
8607; GFX10-CU-NEXT:    s_clause 0x1
8608; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8609; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8610; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8611; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8612; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8613; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8614; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8615; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8616; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8617; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8618; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8619; GFX10-CU-NEXT:    buffer_gl0_inv
8620; GFX10-CU-NEXT:    buffer_gl1_inv
8621; GFX10-CU-NEXT:    s_endpgm
8622;
8623; SKIP-CACHE-INV-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
8624; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8625; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8626; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8627; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8628; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
8629; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
8630; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8631; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8632; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8633; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8634; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8635; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8636; SKIP-CACHE-INV-NEXT:    s_endpgm
8637;
8638; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
8639; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8640; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8641; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8642; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8643; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8644; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8645; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8646; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8647; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
8648; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
8649; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8650;
8651; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
8652; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8653; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8654; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8655; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8656; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8657; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8658; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8659; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8660; GFX90A-TGSPLIT-NEXT:    buffer_invl2
8661; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8662; GFX90A-TGSPLIT-NEXT:    s_endpgm
8663;
8664; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
8665; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8666; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8667; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8668; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8669; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8670; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8671; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
8672; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8673; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
8674; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8675;
8676; GFX940-TGSPLIT-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
8677; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8678; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8679; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8680; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8681; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8682; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8683; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
8684; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8685; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
8686; GFX940-TGSPLIT-NEXT:    s_endpgm
8687    i32* %out, i32 %in, i32 %old) {
8688entry:
8689  %gep = getelementptr i32, i32* %out, i32 4
8690  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") monotonic acquire
8691  ret void
8692}
8693
8694define amdgpu_kernel void @flat_system_one_as_acquire_acquire_cmpxchg(
8695; GFX7-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
8696; GFX7:       ; %bb.0: ; %entry
8697; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8698; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8699; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8700; GFX7-NEXT:    s_add_u32 s0, s0, 16
8701; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8702; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8703; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8704; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8705; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8706; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8707; GFX7-NEXT:    s_waitcnt vmcnt(0)
8708; GFX7-NEXT:    buffer_wbinvl1_vol
8709; GFX7-NEXT:    s_endpgm
8710;
8711; GFX10-WGP-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
8712; GFX10-WGP:       ; %bb.0: ; %entry
8713; GFX10-WGP-NEXT:    s_clause 0x1
8714; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8715; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8716; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8717; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
8718; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
8719; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8720; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8721; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8722; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8723; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8724; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8725; GFX10-WGP-NEXT:    buffer_gl0_inv
8726; GFX10-WGP-NEXT:    buffer_gl1_inv
8727; GFX10-WGP-NEXT:    s_endpgm
8728;
8729; GFX10-CU-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
8730; GFX10-CU:       ; %bb.0: ; %entry
8731; GFX10-CU-NEXT:    s_clause 0x1
8732; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8733; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8734; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8735; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8736; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8737; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8738; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8739; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8740; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8741; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8742; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8743; GFX10-CU-NEXT:    buffer_gl0_inv
8744; GFX10-CU-NEXT:    buffer_gl1_inv
8745; GFX10-CU-NEXT:    s_endpgm
8746;
8747; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
8748; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8749; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8750; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8751; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8752; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
8753; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
8754; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8755; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8756; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8757; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8758; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8759; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8760; SKIP-CACHE-INV-NEXT:    s_endpgm
8761;
8762; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
8763; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8764; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8765; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8766; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8767; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8768; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8769; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8770; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8771; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
8772; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
8773; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8774;
8775; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
8776; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8777; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8778; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8779; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8780; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8781; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8782; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8783; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8784; GFX90A-TGSPLIT-NEXT:    buffer_invl2
8785; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8786; GFX90A-TGSPLIT-NEXT:    s_endpgm
8787;
8788; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
8789; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8790; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8791; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8792; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8793; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8794; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8795; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
8796; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8797; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
8798; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8799;
8800; GFX940-TGSPLIT-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
8801; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8802; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8803; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8804; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8805; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8806; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8807; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
8808; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8809; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
8810; GFX940-TGSPLIT-NEXT:    s_endpgm
8811    i32* %out, i32 %in, i32 %old) {
8812entry:
8813  %gep = getelementptr i32, i32* %out, i32 4
8814  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acquire acquire
8815  ret void
8816}
8817
8818define amdgpu_kernel void @flat_system_one_as_release_acquire_cmpxchg(
8819; GFX7-LABEL: flat_system_one_as_release_acquire_cmpxchg:
8820; GFX7:       ; %bb.0: ; %entry
8821; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8822; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8823; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8824; GFX7-NEXT:    s_add_u32 s0, s0, 16
8825; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8826; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8827; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8828; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8829; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8830; GFX7-NEXT:    s_waitcnt vmcnt(0)
8831; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8832; GFX7-NEXT:    s_waitcnt vmcnt(0)
8833; GFX7-NEXT:    buffer_wbinvl1_vol
8834; GFX7-NEXT:    s_endpgm
8835;
8836; GFX10-WGP-LABEL: flat_system_one_as_release_acquire_cmpxchg:
8837; GFX10-WGP:       ; %bb.0: ; %entry
8838; GFX10-WGP-NEXT:    s_clause 0x1
8839; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8840; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8841; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8842; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
8843; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
8844; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8845; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8846; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8847; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8848; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8849; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8850; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8851; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8852; GFX10-WGP-NEXT:    buffer_gl0_inv
8853; GFX10-WGP-NEXT:    buffer_gl1_inv
8854; GFX10-WGP-NEXT:    s_endpgm
8855;
8856; GFX10-CU-LABEL: flat_system_one_as_release_acquire_cmpxchg:
8857; GFX10-CU:       ; %bb.0: ; %entry
8858; GFX10-CU-NEXT:    s_clause 0x1
8859; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8860; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8861; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8862; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
8863; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
8864; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8865; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
8866; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8867; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
8868; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8869; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8870; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8871; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8872; GFX10-CU-NEXT:    buffer_gl0_inv
8873; GFX10-CU-NEXT:    buffer_gl1_inv
8874; GFX10-CU-NEXT:    s_endpgm
8875;
8876; SKIP-CACHE-INV-LABEL: flat_system_one_as_release_acquire_cmpxchg:
8877; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8878; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8879; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
8880; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8881; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
8882; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
8883; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
8884; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
8885; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
8886; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
8887; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8888; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8889; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8890; SKIP-CACHE-INV-NEXT:    s_endpgm
8891;
8892; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_release_acquire_cmpxchg:
8893; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8894; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8895; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8896; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8897; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8898; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8899; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
8900; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8901; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8902; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8903; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
8904; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
8905; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8906;
8907; GFX90A-TGSPLIT-LABEL: flat_system_one_as_release_acquire_cmpxchg:
8908; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8909; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8910; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8911; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8912; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8913; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8914; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
8915; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8916; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8917; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8918; GFX90A-TGSPLIT-NEXT:    buffer_invl2
8919; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8920; GFX90A-TGSPLIT-NEXT:    s_endpgm
8921;
8922; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_release_acquire_cmpxchg:
8923; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
8924; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8925; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8926; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8927; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8928; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8929; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
8930; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8931; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
8932; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8933; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
8934; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
8935;
8936; GFX940-TGSPLIT-LABEL: flat_system_one_as_release_acquire_cmpxchg:
8937; GFX940-TGSPLIT:       ; %bb.0: ; %entry
8938; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
8939; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
8940; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8941; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
8942; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
8943; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
8944; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8945; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
8946; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8947; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
8948; GFX940-TGSPLIT-NEXT:    s_endpgm
8949    i32* %out, i32 %in, i32 %old) {
8950entry:
8951  %gep = getelementptr i32, i32* %out, i32 4
8952  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") release acquire
8953  ret void
8954}
8955
8956define amdgpu_kernel void @flat_system_one_as_acq_rel_acquire_cmpxchg(
8957; GFX7-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
8958; GFX7:       ; %bb.0: ; %entry
8959; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8960; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8961; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8962; GFX7-NEXT:    s_add_u32 s0, s0, 16
8963; GFX7-NEXT:    s_addc_u32 s1, s1, 0
8964; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8965; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8966; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8967; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8968; GFX7-NEXT:    s_waitcnt vmcnt(0)
8969; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8970; GFX7-NEXT:    s_waitcnt vmcnt(0)
8971; GFX7-NEXT:    buffer_wbinvl1_vol
8972; GFX7-NEXT:    s_endpgm
8973;
8974; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
8975; GFX10-WGP:       ; %bb.0: ; %entry
8976; GFX10-WGP-NEXT:    s_clause 0x1
8977; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8978; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8979; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8980; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
8981; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
8982; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8983; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
8984; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8985; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
8986; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8987; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8988; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
8989; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8990; GFX10-WGP-NEXT:    buffer_gl0_inv
8991; GFX10-WGP-NEXT:    buffer_gl1_inv
8992; GFX10-WGP-NEXT:    s_endpgm
8993;
8994; GFX10-CU-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
8995; GFX10-CU:       ; %bb.0: ; %entry
8996; GFX10-CU-NEXT:    s_clause 0x1
8997; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8998; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8999; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9000; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9001; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9002; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9003; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9004; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9005; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9006; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9007; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9008; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9009; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9010; GFX10-CU-NEXT:    buffer_gl0_inv
9011; GFX10-CU-NEXT:    buffer_gl1_inv
9012; GFX10-CU-NEXT:    s_endpgm
9013;
9014; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
9015; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9016; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9017; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9018; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9019; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9020; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9021; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9022; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9023; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9024; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9025; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9026; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9027; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9028; SKIP-CACHE-INV-NEXT:    s_endpgm
9029;
9030; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
9031; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9032; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9033; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9034; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9035; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9036; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9037; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
9038; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9039; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9040; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9041; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
9042; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9043; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9044;
9045; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
9046; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9047; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9048; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9049; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9050; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9051; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9052; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
9053; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9054; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9055; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9056; GFX90A-TGSPLIT-NEXT:    buffer_invl2
9057; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9058; GFX90A-TGSPLIT-NEXT:    s_endpgm
9059;
9060; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
9061; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9062; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9063; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9064; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9065; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9066; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9067; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
9068; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9069; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
9070; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9071; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
9072; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9073;
9074; GFX940-TGSPLIT-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
9075; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9076; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9077; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9078; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9079; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9080; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9081; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
9082; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9083; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
9084; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9085; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
9086; GFX940-TGSPLIT-NEXT:    s_endpgm
9087    i32* %out, i32 %in, i32 %old) {
9088entry:
9089  %gep = getelementptr i32, i32* %out, i32 4
9090  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel acquire
9091  ret void
9092}
9093
9094define amdgpu_kernel void @flat_system_one_as_seq_cst_acquire_cmpxchg(
9095; GFX7-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
9096; GFX7:       ; %bb.0: ; %entry
9097; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9098; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9099; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9100; GFX7-NEXT:    s_add_u32 s0, s0, 16
9101; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9102; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9103; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9104; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9105; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9106; GFX7-NEXT:    s_waitcnt vmcnt(0)
9107; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9108; GFX7-NEXT:    s_waitcnt vmcnt(0)
9109; GFX7-NEXT:    buffer_wbinvl1_vol
9110; GFX7-NEXT:    s_endpgm
9111;
9112; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
9113; GFX10-WGP:       ; %bb.0: ; %entry
9114; GFX10-WGP-NEXT:    s_clause 0x1
9115; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9116; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9117; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9118; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9119; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9120; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9121; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9122; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9123; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9124; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9125; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9126; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9127; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9128; GFX10-WGP-NEXT:    buffer_gl0_inv
9129; GFX10-WGP-NEXT:    buffer_gl1_inv
9130; GFX10-WGP-NEXT:    s_endpgm
9131;
9132; GFX10-CU-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
9133; GFX10-CU:       ; %bb.0: ; %entry
9134; GFX10-CU-NEXT:    s_clause 0x1
9135; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9136; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9137; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9138; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9139; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9140; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9141; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9142; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9143; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9144; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9145; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9146; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9147; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9148; GFX10-CU-NEXT:    buffer_gl0_inv
9149; GFX10-CU-NEXT:    buffer_gl1_inv
9150; GFX10-CU-NEXT:    s_endpgm
9151;
9152; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
9153; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9154; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9155; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9156; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9157; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9158; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9159; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9160; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9161; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9162; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9163; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9164; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9165; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9166; SKIP-CACHE-INV-NEXT:    s_endpgm
9167;
9168; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
9169; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9170; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9171; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9172; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9173; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9174; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9175; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
9176; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9177; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9178; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9179; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
9180; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9181; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9182;
9183; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
9184; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9185; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9186; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9187; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9188; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9189; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9190; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
9191; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9192; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9193; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9194; GFX90A-TGSPLIT-NEXT:    buffer_invl2
9195; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9196; GFX90A-TGSPLIT-NEXT:    s_endpgm
9197;
9198; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
9199; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9200; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9201; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9202; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9203; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9204; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9205; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
9206; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9207; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
9208; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9209; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
9210; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9211;
9212; GFX940-TGSPLIT-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
9213; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9214; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9215; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9216; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9217; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9218; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9219; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
9220; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9221; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
9222; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9223; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
9224; GFX940-TGSPLIT-NEXT:    s_endpgm
9225    i32* %out, i32 %in, i32 %old) {
9226entry:
9227  %gep = getelementptr i32, i32* %out, i32 4
9228  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst acquire
9229  ret void
9230}
9231
9232define amdgpu_kernel void @flat_system_one_as_monotonic_seq_cst_cmpxchg(
9233; GFX7-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
9234; GFX7:       ; %bb.0: ; %entry
9235; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9236; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9237; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9238; GFX7-NEXT:    s_add_u32 s0, s0, 16
9239; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9240; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9241; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9242; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9243; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9244; GFX7-NEXT:    s_waitcnt vmcnt(0)
9245; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9246; GFX7-NEXT:    s_waitcnt vmcnt(0)
9247; GFX7-NEXT:    buffer_wbinvl1_vol
9248; GFX7-NEXT:    s_endpgm
9249;
9250; GFX10-WGP-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
9251; GFX10-WGP:       ; %bb.0: ; %entry
9252; GFX10-WGP-NEXT:    s_clause 0x1
9253; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9254; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9255; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9256; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9257; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9258; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9259; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9260; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9261; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9262; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9263; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9264; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9265; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9266; GFX10-WGP-NEXT:    buffer_gl0_inv
9267; GFX10-WGP-NEXT:    buffer_gl1_inv
9268; GFX10-WGP-NEXT:    s_endpgm
9269;
9270; GFX10-CU-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
9271; GFX10-CU:       ; %bb.0: ; %entry
9272; GFX10-CU-NEXT:    s_clause 0x1
9273; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9274; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9275; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9276; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9277; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9278; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9279; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9280; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9281; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9282; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9283; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9284; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9285; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9286; GFX10-CU-NEXT:    buffer_gl0_inv
9287; GFX10-CU-NEXT:    buffer_gl1_inv
9288; GFX10-CU-NEXT:    s_endpgm
9289;
9290; SKIP-CACHE-INV-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
9291; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9292; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9293; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9294; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9295; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9296; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9297; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9298; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9299; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9300; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9301; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9302; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9303; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9304; SKIP-CACHE-INV-NEXT:    s_endpgm
9305;
9306; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
9307; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9308; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9309; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9310; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9311; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9312; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9313; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
9314; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9315; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9316; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9317; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
9318; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9319; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9320;
9321; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
9322; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9323; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9324; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9325; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9326; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9327; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9328; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
9329; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9330; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9331; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9332; GFX90A-TGSPLIT-NEXT:    buffer_invl2
9333; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9334; GFX90A-TGSPLIT-NEXT:    s_endpgm
9335;
9336; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
9337; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9338; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9339; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9340; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9341; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9342; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9343; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
9344; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9345; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
9346; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9347; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
9348; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9349;
9350; GFX940-TGSPLIT-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
9351; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9352; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9353; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9354; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9355; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9356; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9357; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
9358; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9359; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
9360; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9361; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
9362; GFX940-TGSPLIT-NEXT:    s_endpgm
9363    i32* %out, i32 %in, i32 %old) {
9364entry:
9365  %gep = getelementptr i32, i32* %out, i32 4
9366  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") monotonic seq_cst
9367  ret void
9368}
9369
9370define amdgpu_kernel void @flat_system_one_as_acquire_seq_cst_cmpxchg(
9371; GFX7-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
9372; GFX7:       ; %bb.0: ; %entry
9373; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9374; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9375; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9376; GFX7-NEXT:    s_add_u32 s0, s0, 16
9377; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9378; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9379; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9380; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9381; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9382; GFX7-NEXT:    s_waitcnt vmcnt(0)
9383; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9384; GFX7-NEXT:    s_waitcnt vmcnt(0)
9385; GFX7-NEXT:    buffer_wbinvl1_vol
9386; GFX7-NEXT:    s_endpgm
9387;
9388; GFX10-WGP-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
9389; GFX10-WGP:       ; %bb.0: ; %entry
9390; GFX10-WGP-NEXT:    s_clause 0x1
9391; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9392; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9393; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9394; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9395; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9396; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9397; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9398; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9399; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9400; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9401; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9402; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9403; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9404; GFX10-WGP-NEXT:    buffer_gl0_inv
9405; GFX10-WGP-NEXT:    buffer_gl1_inv
9406; GFX10-WGP-NEXT:    s_endpgm
9407;
9408; GFX10-CU-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
9409; GFX10-CU:       ; %bb.0: ; %entry
9410; GFX10-CU-NEXT:    s_clause 0x1
9411; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9412; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9413; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9414; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9415; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9416; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9417; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9418; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9419; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9420; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9421; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9422; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9423; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9424; GFX10-CU-NEXT:    buffer_gl0_inv
9425; GFX10-CU-NEXT:    buffer_gl1_inv
9426; GFX10-CU-NEXT:    s_endpgm
9427;
9428; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
9429; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9430; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9431; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9432; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9433; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9434; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9435; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9436; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9437; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9438; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9439; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9440; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9441; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9442; SKIP-CACHE-INV-NEXT:    s_endpgm
9443;
9444; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
9445; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9446; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9447; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9448; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9449; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9450; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9451; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
9452; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9453; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9454; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9455; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
9456; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9457; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9458;
9459; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
9460; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9461; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9462; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9463; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9464; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9465; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9466; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
9467; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9468; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9469; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9470; GFX90A-TGSPLIT-NEXT:    buffer_invl2
9471; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9472; GFX90A-TGSPLIT-NEXT:    s_endpgm
9473;
9474; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
9475; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9476; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9477; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9478; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9479; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9480; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9481; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
9482; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9483; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
9484; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9485; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
9486; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9487;
9488; GFX940-TGSPLIT-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
9489; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9490; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9491; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9492; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9493; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9494; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9495; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
9496; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9497; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
9498; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9499; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
9500; GFX940-TGSPLIT-NEXT:    s_endpgm
9501    i32* %out, i32 %in, i32 %old) {
9502entry:
9503  %gep = getelementptr i32, i32* %out, i32 4
9504  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acquire seq_cst
9505  ret void
9506}
9507
9508define amdgpu_kernel void @flat_system_one_as_release_seq_cst_cmpxchg(
9509; GFX7-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
9510; GFX7:       ; %bb.0: ; %entry
9511; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9512; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9513; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9514; GFX7-NEXT:    s_add_u32 s0, s0, 16
9515; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9516; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9517; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9518; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9519; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9520; GFX7-NEXT:    s_waitcnt vmcnt(0)
9521; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9522; GFX7-NEXT:    s_waitcnt vmcnt(0)
9523; GFX7-NEXT:    buffer_wbinvl1_vol
9524; GFX7-NEXT:    s_endpgm
9525;
9526; GFX10-WGP-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
9527; GFX10-WGP:       ; %bb.0: ; %entry
9528; GFX10-WGP-NEXT:    s_clause 0x1
9529; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9530; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9531; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9532; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9533; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9534; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9535; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9536; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9537; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9538; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9539; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9540; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9541; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9542; GFX10-WGP-NEXT:    buffer_gl0_inv
9543; GFX10-WGP-NEXT:    buffer_gl1_inv
9544; GFX10-WGP-NEXT:    s_endpgm
9545;
9546; GFX10-CU-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
9547; GFX10-CU:       ; %bb.0: ; %entry
9548; GFX10-CU-NEXT:    s_clause 0x1
9549; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9550; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9551; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9552; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9553; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9554; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9555; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9556; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9557; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9558; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9559; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9560; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9561; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9562; GFX10-CU-NEXT:    buffer_gl0_inv
9563; GFX10-CU-NEXT:    buffer_gl1_inv
9564; GFX10-CU-NEXT:    s_endpgm
9565;
9566; SKIP-CACHE-INV-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
9567; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9568; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9569; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9570; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9571; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9572; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9573; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9574; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9575; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9576; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9577; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9578; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9579; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9580; SKIP-CACHE-INV-NEXT:    s_endpgm
9581;
9582; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
9583; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9584; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9585; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9586; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9587; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9588; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9589; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
9590; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9591; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9592; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9593; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
9594; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9595; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9596;
9597; GFX90A-TGSPLIT-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
9598; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9599; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9600; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9601; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9602; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9603; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9604; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
9605; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9606; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9607; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9608; GFX90A-TGSPLIT-NEXT:    buffer_invl2
9609; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9610; GFX90A-TGSPLIT-NEXT:    s_endpgm
9611;
9612; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
9613; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9614; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9615; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9616; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9617; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9618; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9619; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
9620; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9621; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
9622; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9623; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
9624; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9625;
9626; GFX940-TGSPLIT-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
9627; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9628; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9629; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9630; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9631; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9632; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9633; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
9634; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9635; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
9636; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9637; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
9638; GFX940-TGSPLIT-NEXT:    s_endpgm
9639    i32* %out, i32 %in, i32 %old) {
9640entry:
9641  %gep = getelementptr i32, i32* %out, i32 4
9642  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") release seq_cst
9643  ret void
9644}
9645
9646define amdgpu_kernel void @flat_system_one_as_acq_rel_seq_cst_cmpxchg(
9647; GFX7-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
9648; GFX7:       ; %bb.0: ; %entry
9649; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9650; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9651; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9652; GFX7-NEXT:    s_add_u32 s0, s0, 16
9653; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9654; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9655; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9656; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9657; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9658; GFX7-NEXT:    s_waitcnt vmcnt(0)
9659; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9660; GFX7-NEXT:    s_waitcnt vmcnt(0)
9661; GFX7-NEXT:    buffer_wbinvl1_vol
9662; GFX7-NEXT:    s_endpgm
9663;
9664; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
9665; GFX10-WGP:       ; %bb.0: ; %entry
9666; GFX10-WGP-NEXT:    s_clause 0x1
9667; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9668; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9669; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9670; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9671; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9672; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9673; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9674; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9675; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9676; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9677; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9678; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9679; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9680; GFX10-WGP-NEXT:    buffer_gl0_inv
9681; GFX10-WGP-NEXT:    buffer_gl1_inv
9682; GFX10-WGP-NEXT:    s_endpgm
9683;
9684; GFX10-CU-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
9685; GFX10-CU:       ; %bb.0: ; %entry
9686; GFX10-CU-NEXT:    s_clause 0x1
9687; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9688; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9689; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9690; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9691; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9692; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9693; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9694; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9695; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9696; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9697; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9698; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9699; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9700; GFX10-CU-NEXT:    buffer_gl0_inv
9701; GFX10-CU-NEXT:    buffer_gl1_inv
9702; GFX10-CU-NEXT:    s_endpgm
9703;
9704; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
9705; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9706; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9707; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9708; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9709; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9710; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9711; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9712; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9713; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9714; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9715; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9716; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9717; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9718; SKIP-CACHE-INV-NEXT:    s_endpgm
9719;
9720; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
9721; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9722; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9723; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9724; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9725; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9726; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9727; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
9728; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9729; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9730; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9731; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
9732; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9733; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9734;
9735; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
9736; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9737; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9738; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9739; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9740; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9741; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9742; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
9743; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9744; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9745; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9746; GFX90A-TGSPLIT-NEXT:    buffer_invl2
9747; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9748; GFX90A-TGSPLIT-NEXT:    s_endpgm
9749;
9750; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
9751; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9752; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9753; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9754; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9755; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9756; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9757; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
9758; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9759; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
9760; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9761; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
9762; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9763;
9764; GFX940-TGSPLIT-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
9765; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9766; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9767; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9768; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9769; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9770; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9771; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
9772; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9773; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
9774; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9775; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
9776; GFX940-TGSPLIT-NEXT:    s_endpgm
9777    i32* %out, i32 %in, i32 %old) {
9778entry:
9779  %gep = getelementptr i32, i32* %out, i32 4
9780  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel seq_cst
9781  ret void
9782}
9783
9784define amdgpu_kernel void @flat_system_one_as_seq_cst_seq_cst_cmpxchg(
9785; GFX7-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
9786; GFX7:       ; %bb.0: ; %entry
9787; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9788; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9789; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9790; GFX7-NEXT:    s_add_u32 s0, s0, 16
9791; GFX7-NEXT:    s_addc_u32 s1, s1, 0
9792; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9793; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9794; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9795; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9796; GFX7-NEXT:    s_waitcnt vmcnt(0)
9797; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9798; GFX7-NEXT:    s_waitcnt vmcnt(0)
9799; GFX7-NEXT:    buffer_wbinvl1_vol
9800; GFX7-NEXT:    s_endpgm
9801;
9802; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
9803; GFX10-WGP:       ; %bb.0: ; %entry
9804; GFX10-WGP-NEXT:    s_clause 0x1
9805; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9806; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9807; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9808; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
9809; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
9810; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9811; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9812; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9813; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9814; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9815; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9816; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9817; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9818; GFX10-WGP-NEXT:    buffer_gl0_inv
9819; GFX10-WGP-NEXT:    buffer_gl1_inv
9820; GFX10-WGP-NEXT:    s_endpgm
9821;
9822; GFX10-CU-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
9823; GFX10-CU:       ; %bb.0: ; %entry
9824; GFX10-CU-NEXT:    s_clause 0x1
9825; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9826; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9827; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9828; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
9829; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
9830; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9831; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9832; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9833; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9834; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9835; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9836; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9837; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9838; GFX10-CU-NEXT:    buffer_gl0_inv
9839; GFX10-CU-NEXT:    buffer_gl1_inv
9840; GFX10-CU-NEXT:    s_endpgm
9841;
9842; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
9843; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9844; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9845; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9846; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9847; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
9848; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
9849; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9850; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9851; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9852; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9853; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9854; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
9855; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9856; SKIP-CACHE-INV-NEXT:    s_endpgm
9857;
9858; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
9859; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9860; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9861; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9862; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9863; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9864; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9865; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
9866; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9867; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9868; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9869; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
9870; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9871; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9872;
9873; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
9874; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9875; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9876; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9877; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9878; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9879; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9880; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
9881; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9882; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9883; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9884; GFX90A-TGSPLIT-NEXT:    buffer_invl2
9885; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9886; GFX90A-TGSPLIT-NEXT:    s_endpgm
9887;
9888; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
9889; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
9890; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9891; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9892; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9893; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9894; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9895; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
9896; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9897; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
9898; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9899; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
9900; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
9901;
9902; GFX940-TGSPLIT-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
9903; GFX940-TGSPLIT:       ; %bb.0: ; %entry
9904; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9905; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
9906; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9907; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
9908; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
9909; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
9910; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9911; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3] offset:16 sc1
9912; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9913; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
9914; GFX940-TGSPLIT-NEXT:    s_endpgm
9915    i32* %out, i32 %in, i32 %old) {
9916entry:
9917  %gep = getelementptr i32, i32* %out, i32 4
9918  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst seq_cst
9919  ret void
9920}
9921
9922define amdgpu_kernel void @flat_system_one_as_monotonic_monotonic_ret_cmpxchg(
9923; GFX7-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
9924; GFX7:       ; %bb.0: ; %entry
9925; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9926; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9927; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9928; GFX7-NEXT:    s_add_u32 s4, s0, 16
9929; GFX7-NEXT:    s_addc_u32 s5, s1, 0
9930; GFX7-NEXT:    v_mov_b32_e32 v0, s4
9931; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9932; GFX7-NEXT:    v_mov_b32_e32 v1, s5
9933; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9934; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9935; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9936; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9937; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9938; GFX7-NEXT:    flat_store_dword v[0:1], v2
9939; GFX7-NEXT:    s_endpgm
9940;
9941; GFX10-WGP-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
9942; GFX10-WGP:       ; %bb.0: ; %entry
9943; GFX10-WGP-NEXT:    s_clause 0x1
9944; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9945; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9946; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9947; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
9948; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
9949; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
9950; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
9951; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
9952; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
9953; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9954; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9955; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9956; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9957; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
9958; GFX10-WGP-NEXT:    s_endpgm
9959;
9960; GFX10-CU-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
9961; GFX10-CU:       ; %bb.0: ; %entry
9962; GFX10-CU-NEXT:    s_clause 0x1
9963; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9964; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9965; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9966; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
9967; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
9968; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
9969; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
9970; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
9971; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
9972; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9973; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9974; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9975; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9976; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
9977; GFX10-CU-NEXT:    s_endpgm
9978;
9979; SKIP-CACHE-INV-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
9980; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9981; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
9982; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
9983; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9984; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
9985; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
9986; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
9987; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
9988; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
9989; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
9990; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9991; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
9992; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
9993; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
9994; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
9995; SKIP-CACHE-INV-NEXT:    s_endpgm
9996;
9997; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
9998; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9999; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10000; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10001; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10002; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10003; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10004; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10005; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10006; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10007; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10008;
10009; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
10010; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10011; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10012; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10013; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10014; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10015; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10016; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10017; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10018; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10019; GFX90A-TGSPLIT-NEXT:    s_endpgm
10020;
10021; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
10022; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10023; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10024; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10025; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10026; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10027; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10028; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
10029; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10030; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10031; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10032;
10033; GFX940-TGSPLIT-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
10034; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10035; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10036; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10037; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10038; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10039; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10040; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
10041; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10042; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10043; GFX940-TGSPLIT-NEXT:    s_endpgm
10044    i32* %out, i32 %in, i32 %old) {
10045entry:
10046  %gep = getelementptr i32, i32* %out, i32 4
10047  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") monotonic monotonic
10048  %val0 = extractvalue { i32, i1 } %val, 0
10049  store i32 %val0, i32* %out, align 4
10050  ret void
10051}
10052
10053define amdgpu_kernel void @flat_system_one_as_acquire_monotonic_ret_cmpxchg(
10054; GFX7-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
10055; GFX7:       ; %bb.0: ; %entry
10056; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10057; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10058; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10059; GFX7-NEXT:    s_add_u32 s4, s0, 16
10060; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10061; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10062; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10063; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10064; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10065; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10066; GFX7-NEXT:    s_waitcnt vmcnt(0)
10067; GFX7-NEXT:    buffer_wbinvl1_vol
10068; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10069; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10070; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10071; GFX7-NEXT:    flat_store_dword v[0:1], v2
10072; GFX7-NEXT:    s_endpgm
10073;
10074; GFX10-WGP-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
10075; GFX10-WGP:       ; %bb.0: ; %entry
10076; GFX10-WGP-NEXT:    s_clause 0x1
10077; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10078; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10079; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10080; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
10081; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
10082; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
10083; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10084; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
10085; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10086; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10087; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10088; GFX10-WGP-NEXT:    buffer_gl0_inv
10089; GFX10-WGP-NEXT:    buffer_gl1_inv
10090; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10091; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10092; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10093; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
10094; GFX10-WGP-NEXT:    s_endpgm
10095;
10096; GFX10-CU-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
10097; GFX10-CU:       ; %bb.0: ; %entry
10098; GFX10-CU-NEXT:    s_clause 0x1
10099; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10100; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10101; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10102; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
10103; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
10104; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
10105; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10106; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
10107; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10108; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10109; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10110; GFX10-CU-NEXT:    buffer_gl0_inv
10111; GFX10-CU-NEXT:    buffer_gl1_inv
10112; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10113; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10114; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10115; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
10116; GFX10-CU-NEXT:    s_endpgm
10117;
10118; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
10119; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10120; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10121; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10122; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10123; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
10124; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
10125; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10126; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10127; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10128; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10129; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10130; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10131; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10132; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10133; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10134; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
10135; SKIP-CACHE-INV-NEXT:    s_endpgm
10136;
10137; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
10138; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10139; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10140; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10141; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10142; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10143; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10144; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10145; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10146; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
10147; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
10148; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10149; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10150; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10151;
10152; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
10153; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10154; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10155; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10156; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10157; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10158; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10159; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10160; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10161; GFX90A-TGSPLIT-NEXT:    buffer_invl2
10162; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10163; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10164; GFX90A-TGSPLIT-NEXT:    s_endpgm
10165;
10166; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
10167; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10168; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10169; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10170; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10171; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10172; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10173; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
10174; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10175; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
10176; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10177; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10178; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10179;
10180; GFX940-TGSPLIT-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
10181; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10182; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10183; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10184; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10185; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10186; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10187; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
10188; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10189; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
10190; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10191; GFX940-TGSPLIT-NEXT:    s_endpgm
10192    i32* %out, i32 %in, i32 %old) {
10193entry:
10194  %gep = getelementptr i32, i32* %out, i32 4
10195  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acquire monotonic
10196  %val0 = extractvalue { i32, i1 } %val, 0
10197  store i32 %val0, i32* %out, align 4
10198  ret void
10199}
10200
10201define amdgpu_kernel void @flat_system_one_as_release_monotonic_ret_cmpxchg(
10202; GFX7-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
10203; GFX7:       ; %bb.0: ; %entry
10204; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10205; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10206; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10207; GFX7-NEXT:    s_add_u32 s4, s0, 16
10208; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10209; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10210; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10211; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10212; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10213; GFX7-NEXT:    s_waitcnt vmcnt(0)
10214; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10215; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10216; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10217; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10218; GFX7-NEXT:    flat_store_dword v[0:1], v2
10219; GFX7-NEXT:    s_endpgm
10220;
10221; GFX10-WGP-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
10222; GFX10-WGP:       ; %bb.0: ; %entry
10223; GFX10-WGP-NEXT:    s_clause 0x1
10224; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10225; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10226; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10227; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
10228; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
10229; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
10230; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10231; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
10232; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10233; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10234; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10235; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10236; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10237; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10238; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10239; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
10240; GFX10-WGP-NEXT:    s_endpgm
10241;
10242; GFX10-CU-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
10243; GFX10-CU:       ; %bb.0: ; %entry
10244; GFX10-CU-NEXT:    s_clause 0x1
10245; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10246; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10247; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10248; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
10249; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
10250; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
10251; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10252; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
10253; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10254; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10255; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10256; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10257; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10258; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10259; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10260; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
10261; GFX10-CU-NEXT:    s_endpgm
10262;
10263; SKIP-CACHE-INV-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
10264; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10265; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10266; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10267; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10268; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
10269; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
10270; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10271; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10272; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10273; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10274; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10275; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10276; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10277; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10278; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10279; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
10280; SKIP-CACHE-INV-NEXT:    s_endpgm
10281;
10282; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
10283; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10284; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10285; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10286; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10287; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10288; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10289; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
10290; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10291; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10292; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10293; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10294; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10295;
10296; GFX90A-TGSPLIT-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
10297; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10298; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10299; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10300; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10301; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10302; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10303; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
10304; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10305; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10306; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10307; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10308; GFX90A-TGSPLIT-NEXT:    s_endpgm
10309;
10310; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
10311; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10312; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10313; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10314; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10315; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10316; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10317; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
10318; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10319; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
10320; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
10321; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10322; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10323;
10324; GFX940-TGSPLIT-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
10325; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10326; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10327; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10328; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10329; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10330; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10331; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
10332; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10333; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
10334; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10335; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10336; GFX940-TGSPLIT-NEXT:    s_endpgm
10337    i32* %out, i32 %in, i32 %old) {
10338entry:
10339  %gep = getelementptr i32, i32* %out, i32 4
10340  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") release monotonic
10341  %val0 = extractvalue { i32, i1 } %val, 0
10342  store i32 %val0, i32* %out, align 4
10343  ret void
10344}
10345
10346define amdgpu_kernel void @flat_system_one_as_acq_rel_monotonic_ret_cmpxchg(
10347; GFX7-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
10348; GFX7:       ; %bb.0: ; %entry
10349; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10350; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10351; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10352; GFX7-NEXT:    s_add_u32 s4, s0, 16
10353; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10354; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10355; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10356; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10357; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10358; GFX7-NEXT:    s_waitcnt vmcnt(0)
10359; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10360; GFX7-NEXT:    s_waitcnt vmcnt(0)
10361; GFX7-NEXT:    buffer_wbinvl1_vol
10362; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10363; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10364; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10365; GFX7-NEXT:    flat_store_dword v[0:1], v2
10366; GFX7-NEXT:    s_endpgm
10367;
10368; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
10369; GFX10-WGP:       ; %bb.0: ; %entry
10370; GFX10-WGP-NEXT:    s_clause 0x1
10371; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10372; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10373; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10374; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
10375; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
10376; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
10377; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10378; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
10379; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10380; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10381; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10382; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10383; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10384; GFX10-WGP-NEXT:    buffer_gl0_inv
10385; GFX10-WGP-NEXT:    buffer_gl1_inv
10386; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10387; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10388; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10389; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
10390; GFX10-WGP-NEXT:    s_endpgm
10391;
10392; GFX10-CU-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
10393; GFX10-CU:       ; %bb.0: ; %entry
10394; GFX10-CU-NEXT:    s_clause 0x1
10395; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10396; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10397; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10398; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
10399; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
10400; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
10401; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10402; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
10403; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10404; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10405; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10406; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10407; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10408; GFX10-CU-NEXT:    buffer_gl0_inv
10409; GFX10-CU-NEXT:    buffer_gl1_inv
10410; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10411; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10412; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10413; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
10414; GFX10-CU-NEXT:    s_endpgm
10415;
10416; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
10417; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10418; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10419; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10420; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10421; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
10422; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
10423; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10424; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10425; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10426; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10427; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10428; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10429; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10430; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10431; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10432; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10433; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
10434; SKIP-CACHE-INV-NEXT:    s_endpgm
10435;
10436; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
10437; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10438; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10439; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10440; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10441; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10442; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10443; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
10444; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10445; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10446; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10447; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
10448; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
10449; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10450; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10451; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10452;
10453; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
10454; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10455; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10456; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10457; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10458; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10459; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10460; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
10461; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10462; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10463; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10464; GFX90A-TGSPLIT-NEXT:    buffer_invl2
10465; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10466; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10467; GFX90A-TGSPLIT-NEXT:    s_endpgm
10468;
10469; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
10470; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10471; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10472; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10473; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10474; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10475; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10476; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
10477; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10478; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
10479; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10480; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
10481; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10482; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10483; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10484;
10485; GFX940-TGSPLIT-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
10486; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10487; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10488; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10489; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10490; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10491; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10492; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
10493; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10494; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
10495; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10496; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
10497; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10498; GFX940-TGSPLIT-NEXT:    s_endpgm
10499    i32* %out, i32 %in, i32 %old) {
10500entry:
10501  %gep = getelementptr i32, i32* %out, i32 4
10502  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel monotonic
10503  %val0 = extractvalue { i32, i1 } %val, 0
10504  store i32 %val0, i32* %out, align 4
10505  ret void
10506}
10507
10508define amdgpu_kernel void @flat_system_one_as_seq_cst_monotonic_ret_cmpxchg(
10509; GFX7-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
10510; GFX7:       ; %bb.0: ; %entry
10511; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10512; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10513; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10514; GFX7-NEXT:    s_add_u32 s4, s0, 16
10515; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10516; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10517; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10518; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10519; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10520; GFX7-NEXT:    s_waitcnt vmcnt(0)
10521; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10522; GFX7-NEXT:    s_waitcnt vmcnt(0)
10523; GFX7-NEXT:    buffer_wbinvl1_vol
10524; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10525; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10526; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10527; GFX7-NEXT:    flat_store_dword v[0:1], v2
10528; GFX7-NEXT:    s_endpgm
10529;
10530; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
10531; GFX10-WGP:       ; %bb.0: ; %entry
10532; GFX10-WGP-NEXT:    s_clause 0x1
10533; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10534; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10535; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10536; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
10537; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
10538; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
10539; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10540; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
10541; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10542; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10543; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
10544; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10545; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10546; GFX10-WGP-NEXT:    buffer_gl0_inv
10547; GFX10-WGP-NEXT:    buffer_gl1_inv
10548; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10549; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10550; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10551; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
10552; GFX10-WGP-NEXT:    s_endpgm
10553;
10554; GFX10-CU-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
10555; GFX10-CU:       ; %bb.0: ; %entry
10556; GFX10-CU-NEXT:    s_clause 0x1
10557; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10558; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10559; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10560; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
10561; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
10562; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
10563; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10564; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
10565; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10566; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10567; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
10568; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10569; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10570; GFX10-CU-NEXT:    buffer_gl0_inv
10571; GFX10-CU-NEXT:    buffer_gl1_inv
10572; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10573; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10574; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10575; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
10576; GFX10-CU-NEXT:    s_endpgm
10577;
10578; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
10579; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10580; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10581; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10582; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10583; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
10584; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
10585; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10586; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10587; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10588; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10589; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10590; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10591; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10592; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10593; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10594; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10595; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
10596; SKIP-CACHE-INV-NEXT:    s_endpgm
10597;
10598; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
10599; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10600; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10601; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10602; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10603; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10604; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10605; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
10606; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10607; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10608; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10609; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
10610; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
10611; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10612; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10613; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10614;
10615; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
10616; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10617; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10618; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10619; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10620; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10621; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10622; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
10623; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10624; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10625; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10626; GFX90A-TGSPLIT-NEXT:    buffer_invl2
10627; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10628; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10629; GFX90A-TGSPLIT-NEXT:    s_endpgm
10630;
10631; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
10632; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10633; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10634; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10635; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10636; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10637; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10638; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
10639; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10640; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
10641; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10642; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
10643; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10644; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10645; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10646;
10647; GFX940-TGSPLIT-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
10648; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10649; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10650; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10651; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10652; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10653; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10654; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
10655; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10656; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
10657; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10658; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
10659; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10660; GFX940-TGSPLIT-NEXT:    s_endpgm
10661    i32* %out, i32 %in, i32 %old) {
10662entry:
10663  %gep = getelementptr i32, i32* %out, i32 4
10664  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst monotonic
10665  %val0 = extractvalue { i32, i1 } %val, 0
10666  store i32 %val0, i32* %out, align 4
10667  ret void
10668}
10669
10670define amdgpu_kernel void @flat_system_one_as_monotonic_acquire_ret_cmpxchg(
10671; GFX7-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
10672; GFX7:       ; %bb.0: ; %entry
10673; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10674; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10675; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10676; GFX7-NEXT:    s_add_u32 s4, s0, 16
10677; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10678; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10679; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10680; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10681; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10682; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10683; GFX7-NEXT:    s_waitcnt vmcnt(0)
10684; GFX7-NEXT:    buffer_wbinvl1_vol
10685; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10686; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10687; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10688; GFX7-NEXT:    flat_store_dword v[0:1], v2
10689; GFX7-NEXT:    s_endpgm
10690;
10691; GFX10-WGP-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
10692; GFX10-WGP:       ; %bb.0: ; %entry
10693; GFX10-WGP-NEXT:    s_clause 0x1
10694; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10695; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10696; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10697; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
10698; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
10699; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
10700; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10701; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
10702; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10703; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10704; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10705; GFX10-WGP-NEXT:    buffer_gl0_inv
10706; GFX10-WGP-NEXT:    buffer_gl1_inv
10707; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10708; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10709; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10710; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
10711; GFX10-WGP-NEXT:    s_endpgm
10712;
10713; GFX10-CU-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
10714; GFX10-CU:       ; %bb.0: ; %entry
10715; GFX10-CU-NEXT:    s_clause 0x1
10716; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10717; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10718; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10719; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
10720; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
10721; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
10722; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10723; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
10724; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10725; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10726; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10727; GFX10-CU-NEXT:    buffer_gl0_inv
10728; GFX10-CU-NEXT:    buffer_gl1_inv
10729; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10730; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10731; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10732; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
10733; GFX10-CU-NEXT:    s_endpgm
10734;
10735; SKIP-CACHE-INV-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
10736; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10737; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10738; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10739; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10740; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
10741; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
10742; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10743; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10744; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10745; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10746; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10747; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10748; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10749; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10750; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10751; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
10752; SKIP-CACHE-INV-NEXT:    s_endpgm
10753;
10754; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
10755; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10756; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10757; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10758; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10759; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10760; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10761; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10762; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10763; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
10764; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
10765; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10766; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10767; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10768;
10769; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
10770; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10771; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10772; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10773; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10774; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10775; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10776; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10777; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10778; GFX90A-TGSPLIT-NEXT:    buffer_invl2
10779; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10780; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10781; GFX90A-TGSPLIT-NEXT:    s_endpgm
10782;
10783; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
10784; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10785; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10786; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10787; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10788; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10789; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10790; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
10791; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10792; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
10793; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10794; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10795; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10796;
10797; GFX940-TGSPLIT-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
10798; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10799; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10800; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10801; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10802; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10803; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10804; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
10805; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10806; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
10807; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10808; GFX940-TGSPLIT-NEXT:    s_endpgm
10809    i32* %out, i32 %in, i32 %old) {
10810entry:
10811  %gep = getelementptr i32, i32* %out, i32 4
10812  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") monotonic acquire
10813  %val0 = extractvalue { i32, i1 } %val, 0
10814  store i32 %val0, i32* %out, align 4
10815  ret void
10816}
10817
10818define amdgpu_kernel void @flat_system_one_as_acquire_acquire_ret_cmpxchg(
10819; GFX7-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
10820; GFX7:       ; %bb.0: ; %entry
10821; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10822; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10823; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10824; GFX7-NEXT:    s_add_u32 s4, s0, 16
10825; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10826; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10827; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10828; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10829; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10830; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10831; GFX7-NEXT:    s_waitcnt vmcnt(0)
10832; GFX7-NEXT:    buffer_wbinvl1_vol
10833; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10834; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10835; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10836; GFX7-NEXT:    flat_store_dword v[0:1], v2
10837; GFX7-NEXT:    s_endpgm
10838;
10839; GFX10-WGP-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
10840; GFX10-WGP:       ; %bb.0: ; %entry
10841; GFX10-WGP-NEXT:    s_clause 0x1
10842; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10843; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10844; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10845; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
10846; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
10847; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
10848; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10849; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
10850; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
10851; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10852; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
10853; GFX10-WGP-NEXT:    buffer_gl0_inv
10854; GFX10-WGP-NEXT:    buffer_gl1_inv
10855; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
10856; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
10857; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10858; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
10859; GFX10-WGP-NEXT:    s_endpgm
10860;
10861; GFX10-CU-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
10862; GFX10-CU:       ; %bb.0: ; %entry
10863; GFX10-CU-NEXT:    s_clause 0x1
10864; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10865; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10866; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10867; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
10868; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
10869; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
10870; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
10871; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
10872; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
10873; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10874; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
10875; GFX10-CU-NEXT:    buffer_gl0_inv
10876; GFX10-CU-NEXT:    buffer_gl1_inv
10877; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
10878; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
10879; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
10880; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
10881; GFX10-CU-NEXT:    s_endpgm
10882;
10883; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
10884; SKIP-CACHE-INV:       ; %bb.0: ; %entry
10885; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10886; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
10887; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10888; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
10889; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
10890; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
10891; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
10892; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
10893; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
10894; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10895; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
10896; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
10897; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
10898; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
10899; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
10900; SKIP-CACHE-INV-NEXT:    s_endpgm
10901;
10902; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
10903; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
10904; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10905; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10906; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10907; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10908; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10909; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10910; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10911; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
10912; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
10913; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10914; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10915; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
10916;
10917; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
10918; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
10919; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10920; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10921; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10922; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10923; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10924; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10925; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10926; GFX90A-TGSPLIT-NEXT:    buffer_invl2
10927; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
10928; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10929; GFX90A-TGSPLIT-NEXT:    s_endpgm
10930;
10931; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
10932; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
10933; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10934; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10935; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10936; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10937; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10938; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
10939; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10940; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
10941; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10942; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10943; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
10944;
10945; GFX940-TGSPLIT-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
10946; GFX940-TGSPLIT:       ; %bb.0: ; %entry
10947; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
10948; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
10949; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
10950; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
10951; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
10952; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
10953; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
10954; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
10955; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
10956; GFX940-TGSPLIT-NEXT:    s_endpgm
10957    i32* %out, i32 %in, i32 %old) {
10958entry:
10959  %gep = getelementptr i32, i32* %out, i32 4
10960  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acquire acquire
10961  %val0 = extractvalue { i32, i1 } %val, 0
10962  store i32 %val0, i32* %out, align 4
10963  ret void
10964}
10965
10966define amdgpu_kernel void @flat_system_one_as_release_acquire_ret_cmpxchg(
10967; GFX7-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
10968; GFX7:       ; %bb.0: ; %entry
10969; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10970; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
10971; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10972; GFX7-NEXT:    s_add_u32 s4, s0, 16
10973; GFX7-NEXT:    s_addc_u32 s5, s1, 0
10974; GFX7-NEXT:    v_mov_b32_e32 v0, s4
10975; GFX7-NEXT:    v_mov_b32_e32 v2, s2
10976; GFX7-NEXT:    v_mov_b32_e32 v1, s5
10977; GFX7-NEXT:    v_mov_b32_e32 v3, s3
10978; GFX7-NEXT:    s_waitcnt vmcnt(0)
10979; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10980; GFX7-NEXT:    s_waitcnt vmcnt(0)
10981; GFX7-NEXT:    buffer_wbinvl1_vol
10982; GFX7-NEXT:    v_mov_b32_e32 v0, s0
10983; GFX7-NEXT:    v_mov_b32_e32 v1, s1
10984; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
10985; GFX7-NEXT:    flat_store_dword v[0:1], v2
10986; GFX7-NEXT:    s_endpgm
10987;
10988; GFX10-WGP-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
10989; GFX10-WGP:       ; %bb.0: ; %entry
10990; GFX10-WGP-NEXT:    s_clause 0x1
10991; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
10992; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
10993; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
10994; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
10995; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
10996; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
10997; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
10998; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
10999; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11000; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11001; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11002; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11003; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11004; GFX10-WGP-NEXT:    buffer_gl0_inv
11005; GFX10-WGP-NEXT:    buffer_gl1_inv
11006; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11007; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11008; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11009; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
11010; GFX10-WGP-NEXT:    s_endpgm
11011;
11012; GFX10-CU-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
11013; GFX10-CU:       ; %bb.0: ; %entry
11014; GFX10-CU-NEXT:    s_clause 0x1
11015; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11016; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11017; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11018; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11019; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11020; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11021; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11022; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11023; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11024; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11025; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11026; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11027; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11028; GFX10-CU-NEXT:    buffer_gl0_inv
11029; GFX10-CU-NEXT:    buffer_gl1_inv
11030; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11031; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11032; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11033; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11034; GFX10-CU-NEXT:    s_endpgm
11035;
11036; SKIP-CACHE-INV-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
11037; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11038; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11039; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11040; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11041; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
11042; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
11043; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11044; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11045; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11046; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11047; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11048; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11049; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11050; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11051; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11052; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11053; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
11054; SKIP-CACHE-INV-NEXT:    s_endpgm
11055;
11056; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
11057; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11058; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11059; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11060; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11061; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11062; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11063; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
11064; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11065; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11066; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11067; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
11068; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
11069; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11070; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11071; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11072;
11073; GFX90A-TGSPLIT-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
11074; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11075; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11076; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11077; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11078; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11079; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11080; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
11081; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11082; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11083; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11084; GFX90A-TGSPLIT-NEXT:    buffer_invl2
11085; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11086; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11087; GFX90A-TGSPLIT-NEXT:    s_endpgm
11088;
11089; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
11090; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11091; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11092; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11093; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11094; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11095; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11096; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
11097; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11098; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
11099; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11100; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
11101; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11102; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11103; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11104;
11105; GFX940-TGSPLIT-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
11106; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11107; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11108; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11109; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11110; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11111; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11112; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
11113; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11114; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
11115; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11116; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
11117; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11118; GFX940-TGSPLIT-NEXT:    s_endpgm
11119    i32* %out, i32 %in, i32 %old) {
11120entry:
11121  %gep = getelementptr i32, i32* %out, i32 4
11122  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") release acquire
11123  %val0 = extractvalue { i32, i1 } %val, 0
11124  store i32 %val0, i32* %out, align 4
11125  ret void
11126}
11127
11128define amdgpu_kernel void @flat_system_one_as_acq_rel_acquire_ret_cmpxchg(
11129; GFX7-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
11130; GFX7:       ; %bb.0: ; %entry
11131; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11132; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11133; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11134; GFX7-NEXT:    s_add_u32 s4, s0, 16
11135; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11136; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11137; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11138; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11139; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11140; GFX7-NEXT:    s_waitcnt vmcnt(0)
11141; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11142; GFX7-NEXT:    s_waitcnt vmcnt(0)
11143; GFX7-NEXT:    buffer_wbinvl1_vol
11144; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11145; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11146; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11147; GFX7-NEXT:    flat_store_dword v[0:1], v2
11148; GFX7-NEXT:    s_endpgm
11149;
11150; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
11151; GFX10-WGP:       ; %bb.0: ; %entry
11152; GFX10-WGP-NEXT:    s_clause 0x1
11153; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11154; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11155; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11156; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
11157; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
11158; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
11159; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11160; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
11161; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11162; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11163; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11164; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11165; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11166; GFX10-WGP-NEXT:    buffer_gl0_inv
11167; GFX10-WGP-NEXT:    buffer_gl1_inv
11168; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11169; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11170; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11171; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
11172; GFX10-WGP-NEXT:    s_endpgm
11173;
11174; GFX10-CU-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
11175; GFX10-CU:       ; %bb.0: ; %entry
11176; GFX10-CU-NEXT:    s_clause 0x1
11177; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11178; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11179; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11180; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11181; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11182; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11183; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11184; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11185; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11186; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11187; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11188; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11189; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11190; GFX10-CU-NEXT:    buffer_gl0_inv
11191; GFX10-CU-NEXT:    buffer_gl1_inv
11192; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11193; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11194; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11195; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11196; GFX10-CU-NEXT:    s_endpgm
11197;
11198; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
11199; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11200; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11201; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11202; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11203; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
11204; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
11205; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11206; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11207; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11208; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11209; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11210; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11211; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11212; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11213; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11214; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11215; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
11216; SKIP-CACHE-INV-NEXT:    s_endpgm
11217;
11218; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
11219; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11220; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11221; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11222; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11223; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11224; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11225; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
11226; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11227; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11228; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11229; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
11230; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
11231; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11232; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11233; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11234;
11235; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
11236; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11237; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11238; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11239; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11240; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11241; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11242; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
11243; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11244; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11245; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11246; GFX90A-TGSPLIT-NEXT:    buffer_invl2
11247; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11248; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11249; GFX90A-TGSPLIT-NEXT:    s_endpgm
11250;
11251; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
11252; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11253; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11254; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11255; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11256; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11257; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11258; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
11259; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11260; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
11261; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11262; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
11263; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11264; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11265; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11266;
11267; GFX940-TGSPLIT-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
11268; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11269; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11270; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11271; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11272; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11273; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11274; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
11275; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11276; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
11277; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11278; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
11279; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11280; GFX940-TGSPLIT-NEXT:    s_endpgm
11281    i32* %out, i32 %in, i32 %old) {
11282entry:
11283  %gep = getelementptr i32, i32* %out, i32 4
11284  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel acquire
11285  %val0 = extractvalue { i32, i1 } %val, 0
11286  store i32 %val0, i32* %out, align 4
11287  ret void
11288}
11289
11290define amdgpu_kernel void @flat_system_one_as_seq_cst_acquire_ret_cmpxchg(
11291; GFX7-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
11292; GFX7:       ; %bb.0: ; %entry
11293; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11294; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11295; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11296; GFX7-NEXT:    s_add_u32 s4, s0, 16
11297; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11298; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11299; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11300; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11301; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11302; GFX7-NEXT:    s_waitcnt vmcnt(0)
11303; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11304; GFX7-NEXT:    s_waitcnt vmcnt(0)
11305; GFX7-NEXT:    buffer_wbinvl1_vol
11306; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11307; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11308; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11309; GFX7-NEXT:    flat_store_dword v[0:1], v2
11310; GFX7-NEXT:    s_endpgm
11311;
11312; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
11313; GFX10-WGP:       ; %bb.0: ; %entry
11314; GFX10-WGP-NEXT:    s_clause 0x1
11315; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11316; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11317; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11318; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
11319; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
11320; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
11321; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11322; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
11323; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11324; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11325; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11326; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11327; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11328; GFX10-WGP-NEXT:    buffer_gl0_inv
11329; GFX10-WGP-NEXT:    buffer_gl1_inv
11330; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11331; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11332; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11333; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
11334; GFX10-WGP-NEXT:    s_endpgm
11335;
11336; GFX10-CU-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
11337; GFX10-CU:       ; %bb.0: ; %entry
11338; GFX10-CU-NEXT:    s_clause 0x1
11339; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11340; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11341; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11342; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11343; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11344; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11345; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11346; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11347; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11348; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11349; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11350; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11351; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11352; GFX10-CU-NEXT:    buffer_gl0_inv
11353; GFX10-CU-NEXT:    buffer_gl1_inv
11354; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11355; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11356; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11357; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11358; GFX10-CU-NEXT:    s_endpgm
11359;
11360; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
11361; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11362; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11363; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11364; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11365; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
11366; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
11367; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11368; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11369; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11370; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11371; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11372; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11373; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11374; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11375; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11376; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11377; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
11378; SKIP-CACHE-INV-NEXT:    s_endpgm
11379;
11380; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
11381; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11382; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11383; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11384; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11385; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11386; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11387; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
11388; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11389; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11390; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11391; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
11392; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
11393; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11394; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11395; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11396;
11397; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
11398; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11399; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11400; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11401; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11402; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11403; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11404; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
11405; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11406; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11407; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11408; GFX90A-TGSPLIT-NEXT:    buffer_invl2
11409; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11410; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11411; GFX90A-TGSPLIT-NEXT:    s_endpgm
11412;
11413; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
11414; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11415; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11416; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11417; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11418; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11419; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11420; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
11421; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11422; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
11423; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11424; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
11425; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11426; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11427; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11428;
11429; GFX940-TGSPLIT-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
11430; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11431; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11432; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11433; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11434; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11435; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11436; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
11437; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11438; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
11439; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11440; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
11441; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11442; GFX940-TGSPLIT-NEXT:    s_endpgm
11443    i32* %out, i32 %in, i32 %old) {
11444entry:
11445  %gep = getelementptr i32, i32* %out, i32 4
11446  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst acquire
11447  %val0 = extractvalue { i32, i1 } %val, 0
11448  store i32 %val0, i32* %out, align 4
11449  ret void
11450}
11451
11452define amdgpu_kernel void @flat_system_one_as_monotonic_seq_cst_ret_cmpxchg(
11453; GFX7-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
11454; GFX7:       ; %bb.0: ; %entry
11455; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11456; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11457; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11458; GFX7-NEXT:    s_add_u32 s4, s0, 16
11459; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11460; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11461; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11462; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11463; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11464; GFX7-NEXT:    s_waitcnt vmcnt(0)
11465; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11466; GFX7-NEXT:    s_waitcnt vmcnt(0)
11467; GFX7-NEXT:    buffer_wbinvl1_vol
11468; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11469; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11470; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11471; GFX7-NEXT:    flat_store_dword v[0:1], v2
11472; GFX7-NEXT:    s_endpgm
11473;
11474; GFX10-WGP-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
11475; GFX10-WGP:       ; %bb.0: ; %entry
11476; GFX10-WGP-NEXT:    s_clause 0x1
11477; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11478; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11479; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11480; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
11481; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
11482; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
11483; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11484; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
11485; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11486; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11487; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11488; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11489; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11490; GFX10-WGP-NEXT:    buffer_gl0_inv
11491; GFX10-WGP-NEXT:    buffer_gl1_inv
11492; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11493; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11494; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11495; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
11496; GFX10-WGP-NEXT:    s_endpgm
11497;
11498; GFX10-CU-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
11499; GFX10-CU:       ; %bb.0: ; %entry
11500; GFX10-CU-NEXT:    s_clause 0x1
11501; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11502; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11503; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11504; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11505; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11506; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11507; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11508; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11509; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11510; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11511; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11512; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11513; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11514; GFX10-CU-NEXT:    buffer_gl0_inv
11515; GFX10-CU-NEXT:    buffer_gl1_inv
11516; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11517; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11518; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11519; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11520; GFX10-CU-NEXT:    s_endpgm
11521;
11522; SKIP-CACHE-INV-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
11523; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11524; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11525; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11526; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11527; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
11528; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
11529; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11530; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11531; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11532; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11533; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11534; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11535; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11536; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11537; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11538; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11539; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
11540; SKIP-CACHE-INV-NEXT:    s_endpgm
11541;
11542; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
11543; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11544; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11545; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11546; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11547; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11548; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11549; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
11550; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11551; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11552; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11553; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
11554; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
11555; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11556; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11557; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11558;
11559; GFX90A-TGSPLIT-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
11560; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11561; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11562; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11563; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11564; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11565; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11566; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
11567; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11568; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11569; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11570; GFX90A-TGSPLIT-NEXT:    buffer_invl2
11571; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11572; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11573; GFX90A-TGSPLIT-NEXT:    s_endpgm
11574;
11575; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
11576; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11577; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11578; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11579; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11580; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11581; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11582; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
11583; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11584; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
11585; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11586; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
11587; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11588; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11589; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11590;
11591; GFX940-TGSPLIT-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
11592; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11593; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11594; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11595; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11596; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11597; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11598; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
11599; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11600; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
11601; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11602; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
11603; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11604; GFX940-TGSPLIT-NEXT:    s_endpgm
11605    i32* %out, i32 %in, i32 %old) {
11606entry:
11607  %gep = getelementptr i32, i32* %out, i32 4
11608  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") monotonic seq_cst
11609  %val0 = extractvalue { i32, i1 } %val, 0
11610  store i32 %val0, i32* %out, align 4
11611  ret void
11612}
11613
11614define amdgpu_kernel void @flat_system_one_as_acquire_seq_cst_ret_cmpxchg(
11615; GFX7-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
11616; GFX7:       ; %bb.0: ; %entry
11617; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11618; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11619; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11620; GFX7-NEXT:    s_add_u32 s4, s0, 16
11621; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11622; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11623; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11624; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11625; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11626; GFX7-NEXT:    s_waitcnt vmcnt(0)
11627; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11628; GFX7-NEXT:    s_waitcnt vmcnt(0)
11629; GFX7-NEXT:    buffer_wbinvl1_vol
11630; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11631; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11632; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11633; GFX7-NEXT:    flat_store_dword v[0:1], v2
11634; GFX7-NEXT:    s_endpgm
11635;
11636; GFX10-WGP-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
11637; GFX10-WGP:       ; %bb.0: ; %entry
11638; GFX10-WGP-NEXT:    s_clause 0x1
11639; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11640; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11641; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11642; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
11643; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
11644; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
11645; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11646; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
11647; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11648; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11649; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11650; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11651; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11652; GFX10-WGP-NEXT:    buffer_gl0_inv
11653; GFX10-WGP-NEXT:    buffer_gl1_inv
11654; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11655; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11656; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11657; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
11658; GFX10-WGP-NEXT:    s_endpgm
11659;
11660; GFX10-CU-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
11661; GFX10-CU:       ; %bb.0: ; %entry
11662; GFX10-CU-NEXT:    s_clause 0x1
11663; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11664; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11665; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11666; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11667; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11668; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11669; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11670; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11671; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11672; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11673; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11674; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11675; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11676; GFX10-CU-NEXT:    buffer_gl0_inv
11677; GFX10-CU-NEXT:    buffer_gl1_inv
11678; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11679; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11680; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11681; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11682; GFX10-CU-NEXT:    s_endpgm
11683;
11684; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
11685; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11686; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11687; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11688; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11689; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
11690; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
11691; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11692; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11693; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11694; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11695; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11696; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11697; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11698; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11699; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11700; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11701; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
11702; SKIP-CACHE-INV-NEXT:    s_endpgm
11703;
11704; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
11705; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11706; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11707; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11708; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11709; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11710; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11711; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
11712; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11713; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11714; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11715; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
11716; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
11717; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11718; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11719; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11720;
11721; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
11722; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11723; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11724; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11725; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11726; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11727; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11728; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
11729; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11730; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11731; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11732; GFX90A-TGSPLIT-NEXT:    buffer_invl2
11733; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11734; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11735; GFX90A-TGSPLIT-NEXT:    s_endpgm
11736;
11737; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
11738; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11739; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11740; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11741; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11742; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11743; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11744; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
11745; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11746; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
11747; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11748; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
11749; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11750; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11751; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11752;
11753; GFX940-TGSPLIT-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
11754; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11755; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11756; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11757; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11758; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11759; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11760; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
11761; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11762; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
11763; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11764; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
11765; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11766; GFX940-TGSPLIT-NEXT:    s_endpgm
11767    i32* %out, i32 %in, i32 %old) {
11768entry:
11769  %gep = getelementptr i32, i32* %out, i32 4
11770  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acquire seq_cst
11771  %val0 = extractvalue { i32, i1 } %val, 0
11772  store i32 %val0, i32* %out, align 4
11773  ret void
11774}
11775
11776define amdgpu_kernel void @flat_system_one_as_release_seq_cst_ret_cmpxchg(
11777; GFX7-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
11778; GFX7:       ; %bb.0: ; %entry
11779; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11780; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11781; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11782; GFX7-NEXT:    s_add_u32 s4, s0, 16
11783; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11784; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11785; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11786; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11787; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11788; GFX7-NEXT:    s_waitcnt vmcnt(0)
11789; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11790; GFX7-NEXT:    s_waitcnt vmcnt(0)
11791; GFX7-NEXT:    buffer_wbinvl1_vol
11792; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11793; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11794; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11795; GFX7-NEXT:    flat_store_dword v[0:1], v2
11796; GFX7-NEXT:    s_endpgm
11797;
11798; GFX10-WGP-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
11799; GFX10-WGP:       ; %bb.0: ; %entry
11800; GFX10-WGP-NEXT:    s_clause 0x1
11801; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11802; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11803; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11804; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
11805; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
11806; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
11807; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11808; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
11809; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11810; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11811; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11812; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11813; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11814; GFX10-WGP-NEXT:    buffer_gl0_inv
11815; GFX10-WGP-NEXT:    buffer_gl1_inv
11816; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11817; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11818; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11819; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
11820; GFX10-WGP-NEXT:    s_endpgm
11821;
11822; GFX10-CU-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
11823; GFX10-CU:       ; %bb.0: ; %entry
11824; GFX10-CU-NEXT:    s_clause 0x1
11825; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11826; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11827; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11828; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11829; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11830; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11831; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11832; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11833; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11834; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11835; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11836; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11837; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11838; GFX10-CU-NEXT:    buffer_gl0_inv
11839; GFX10-CU-NEXT:    buffer_gl1_inv
11840; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
11841; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
11842; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11843; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
11844; GFX10-CU-NEXT:    s_endpgm
11845;
11846; SKIP-CACHE-INV-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
11847; SKIP-CACHE-INV:       ; %bb.0: ; %entry
11848; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11849; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
11850; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11851; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
11852; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
11853; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
11854; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
11855; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
11856; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
11857; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11858; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11859; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
11860; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
11861; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
11862; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
11863; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
11864; SKIP-CACHE-INV-NEXT:    s_endpgm
11865;
11866; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
11867; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
11868; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11869; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11870; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11871; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11872; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11873; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
11874; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11875; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11876; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11877; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
11878; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
11879; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11880; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11881; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
11882;
11883; GFX90A-TGSPLIT-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
11884; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
11885; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11886; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11887; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11888; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11889; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11890; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
11891; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11892; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11893; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11894; GFX90A-TGSPLIT-NEXT:    buffer_invl2
11895; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
11896; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11897; GFX90A-TGSPLIT-NEXT:    s_endpgm
11898;
11899; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
11900; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
11901; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11902; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11903; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11904; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11905; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11906; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
11907; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11908; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
11909; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11910; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
11911; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11912; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11913; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
11914;
11915; GFX940-TGSPLIT-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
11916; GFX940-TGSPLIT:       ; %bb.0: ; %entry
11917; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
11918; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
11919; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
11920; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
11921; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
11922; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
11923; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11924; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
11925; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
11926; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
11927; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
11928; GFX940-TGSPLIT-NEXT:    s_endpgm
11929    i32* %out, i32 %in, i32 %old) {
11930entry:
11931  %gep = getelementptr i32, i32* %out, i32 4
11932  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") release seq_cst
11933  %val0 = extractvalue { i32, i1 } %val, 0
11934  store i32 %val0, i32* %out, align 4
11935  ret void
11936}
11937
11938define amdgpu_kernel void @flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg(
11939; GFX7-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
11940; GFX7:       ; %bb.0: ; %entry
11941; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11942; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
11943; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11944; GFX7-NEXT:    s_add_u32 s4, s0, 16
11945; GFX7-NEXT:    s_addc_u32 s5, s1, 0
11946; GFX7-NEXT:    v_mov_b32_e32 v0, s4
11947; GFX7-NEXT:    v_mov_b32_e32 v2, s2
11948; GFX7-NEXT:    v_mov_b32_e32 v1, s5
11949; GFX7-NEXT:    v_mov_b32_e32 v3, s3
11950; GFX7-NEXT:    s_waitcnt vmcnt(0)
11951; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11952; GFX7-NEXT:    s_waitcnt vmcnt(0)
11953; GFX7-NEXT:    buffer_wbinvl1_vol
11954; GFX7-NEXT:    v_mov_b32_e32 v0, s0
11955; GFX7-NEXT:    v_mov_b32_e32 v1, s1
11956; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
11957; GFX7-NEXT:    flat_store_dword v[0:1], v2
11958; GFX7-NEXT:    s_endpgm
11959;
11960; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
11961; GFX10-WGP:       ; %bb.0: ; %entry
11962; GFX10-WGP-NEXT:    s_clause 0x1
11963; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11964; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11965; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11966; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
11967; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
11968; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
11969; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
11970; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
11971; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
11972; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11973; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
11974; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11975; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
11976; GFX10-WGP-NEXT:    buffer_gl0_inv
11977; GFX10-WGP-NEXT:    buffer_gl1_inv
11978; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
11979; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
11980; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
11981; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
11982; GFX10-WGP-NEXT:    s_endpgm
11983;
11984; GFX10-CU-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
11985; GFX10-CU:       ; %bb.0: ; %entry
11986; GFX10-CU-NEXT:    s_clause 0x1
11987; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
11988; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
11989; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
11990; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
11991; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
11992; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
11993; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
11994; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
11995; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
11996; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
11997; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
11998; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11999; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12000; GFX10-CU-NEXT:    buffer_gl0_inv
12001; GFX10-CU-NEXT:    buffer_gl1_inv
12002; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12003; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12004; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12005; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12006; GFX10-CU-NEXT:    s_endpgm
12007;
12008; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
12009; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12010; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12011; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12012; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12013; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12014; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12015; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12016; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12017; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12018; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12019; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12020; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12021; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12022; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12023; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12024; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12025; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12026; SKIP-CACHE-INV-NEXT:    s_endpgm
12027;
12028; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
12029; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12030; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12031; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12032; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12033; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12034; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12035; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
12036; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12037; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12038; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12039; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
12040; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
12041; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12042; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12043; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12044;
12045; GFX90A-TGSPLIT-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
12046; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12047; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12048; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12049; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12050; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12051; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12052; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
12053; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12054; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12055; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12056; GFX90A-TGSPLIT-NEXT:    buffer_invl2
12057; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
12058; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12059; GFX90A-TGSPLIT-NEXT:    s_endpgm
12060;
12061; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
12062; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12063; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12064; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12065; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12066; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12067; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12068; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
12069; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12070; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
12071; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12072; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
12073; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12074; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12075; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12076;
12077; GFX940-TGSPLIT-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
12078; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12079; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12080; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12081; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12082; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12083; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12084; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
12085; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12086; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
12087; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12088; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
12089; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12090; GFX940-TGSPLIT-NEXT:    s_endpgm
12091    i32* %out, i32 %in, i32 %old) {
12092entry:
12093  %gep = getelementptr i32, i32* %out, i32 4
12094  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel seq_cst
12095  %val0 = extractvalue { i32, i1 } %val, 0
12096  store i32 %val0, i32* %out, align 4
12097  ret void
12098}
12099
12100define amdgpu_kernel void @flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg(
12101; GFX7-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
12102; GFX7:       ; %bb.0: ; %entry
12103; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12104; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
12105; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12106; GFX7-NEXT:    s_add_u32 s4, s0, 16
12107; GFX7-NEXT:    s_addc_u32 s5, s1, 0
12108; GFX7-NEXT:    v_mov_b32_e32 v0, s4
12109; GFX7-NEXT:    v_mov_b32_e32 v2, s2
12110; GFX7-NEXT:    v_mov_b32_e32 v1, s5
12111; GFX7-NEXT:    v_mov_b32_e32 v3, s3
12112; GFX7-NEXT:    s_waitcnt vmcnt(0)
12113; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12114; GFX7-NEXT:    s_waitcnt vmcnt(0)
12115; GFX7-NEXT:    buffer_wbinvl1_vol
12116; GFX7-NEXT:    v_mov_b32_e32 v0, s0
12117; GFX7-NEXT:    v_mov_b32_e32 v1, s1
12118; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
12119; GFX7-NEXT:    flat_store_dword v[0:1], v2
12120; GFX7-NEXT:    s_endpgm
12121;
12122; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
12123; GFX10-WGP:       ; %bb.0: ; %entry
12124; GFX10-WGP-NEXT:    s_clause 0x1
12125; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12126; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12127; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12128; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
12129; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
12130; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
12131; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
12132; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
12133; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
12134; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12135; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
12136; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12137; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
12138; GFX10-WGP-NEXT:    buffer_gl0_inv
12139; GFX10-WGP-NEXT:    buffer_gl1_inv
12140; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
12141; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
12142; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
12143; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
12144; GFX10-WGP-NEXT:    s_endpgm
12145;
12146; GFX10-CU-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
12147; GFX10-CU:       ; %bb.0: ; %entry
12148; GFX10-CU-NEXT:    s_clause 0x1
12149; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12150; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12151; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12152; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
12153; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
12154; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
12155; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
12156; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
12157; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
12158; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12159; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
12160; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12161; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
12162; GFX10-CU-NEXT:    buffer_gl0_inv
12163; GFX10-CU-NEXT:    buffer_gl1_inv
12164; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
12165; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
12166; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
12167; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
12168; GFX10-CU-NEXT:    s_endpgm
12169;
12170; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
12171; SKIP-CACHE-INV:       ; %bb.0: ; %entry
12172; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12173; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2
12174; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12175; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
12176; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
12177; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
12178; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
12179; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
12180; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
12181; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12182; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
12183; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
12184; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
12185; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
12186; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
12187; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
12188; SKIP-CACHE-INV-NEXT:    s_endpgm
12189;
12190; GFX90A-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
12191; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
12192; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12193; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12194; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12195; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12196; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12197; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
12198; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12199; GFX90A-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12200; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12201; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
12202; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
12203; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12204; GFX90A-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12205; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
12206;
12207; GFX90A-TGSPLIT-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
12208; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
12209; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
12210; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
12211; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12212; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
12213; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
12214; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
12215; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12216; GFX90A-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
12217; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12218; GFX90A-TGSPLIT-NEXT:    buffer_invl2
12219; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
12220; GFX90A-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12221; GFX90A-TGSPLIT-NEXT:    s_endpgm
12222;
12223; GFX940-NOTTGSPLIT-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
12224; GFX940-NOTTGSPLIT:       ; %bb.0: ; %entry
12225; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12226; GFX940-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12227; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12228; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12229; GFX940-NOTTGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12230; GFX940-NOTTGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
12231; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12232; GFX940-NOTTGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
12233; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12234; GFX940-NOTTGSPLIT-NEXT:    buffer_inv sc0 sc1
12235; GFX940-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12236; GFX940-NOTTGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12237; GFX940-NOTTGSPLIT-NEXT:    s_endpgm
12238;
12239; GFX940-TGSPLIT-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
12240; GFX940-TGSPLIT:       ; %bb.0: ; %entry
12241; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
12242; GFX940-TGSPLIT-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
12243; GFX940-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
12244; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
12245; GFX940-TGSPLIT-NEXT:    v_mov_b64_e32 v[2:3], s[4:5]
12246; GFX940-TGSPLIT-NEXT:    buffer_wbl2 sc0 sc1
12247; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12248; GFX940-TGSPLIT-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0 sc1
12249; GFX940-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
12250; GFX940-TGSPLIT-NEXT:    buffer_inv sc0 sc1
12251; GFX940-TGSPLIT-NEXT:    flat_store_dword v[0:1], v2
12252; GFX940-TGSPLIT-NEXT:    s_endpgm
12253    i32* %out, i32 %in, i32 %old) {
12254entry:
12255  %gep = getelementptr i32, i32* %out, i32 4
12256  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst seq_cst
12257  %val0 = extractvalue { i32, i1 } %val, 0
12258  store i32 %val0, i32* %out, align 4
12259  ret void
12260}
12261
12262