1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
3; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
4; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
5; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
6; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11-WGP %s
7; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11-CU %s
8
9define amdgpu_kernel void @flat_nontemporal_load_0(
10; GFX7-LABEL: flat_nontemporal_load_0:
11; GFX7:       ; %bb.0: ; %entry
12; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
13; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
14; GFX7-NEXT:    v_mov_b32_e32 v0, s0
15; GFX7-NEXT:    v_mov_b32_e32 v1, s1
16; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
17; GFX7-NEXT:    s_waitcnt vmcnt(0)
18; GFX7-NEXT:    v_mov_b32_e32 v0, s2
19; GFX7-NEXT:    v_mov_b32_e32 v1, s3
20; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
21; GFX7-NEXT:    flat_store_dword v[0:1], v2
22; GFX7-NEXT:    s_endpgm
23;
24; GFX10-WGP-LABEL: flat_nontemporal_load_0:
25; GFX10-WGP:       ; %bb.0: ; %entry
26; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
27; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
28; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
29; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
30; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
31; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
32; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
33; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
34; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
35; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
36; GFX10-WGP-NEXT:    s_endpgm
37;
38; GFX10-CU-LABEL: flat_nontemporal_load_0:
39; GFX10-CU:       ; %bb.0: ; %entry
40; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
41; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
42; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
43; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
44; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
45; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
46; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
47; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
48; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
49; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
50; GFX10-CU-NEXT:    s_endpgm
51;
52; SKIP-CACHE-INV-LABEL: flat_nontemporal_load_0:
53; SKIP-CACHE-INV:       ; %bb.0: ; %entry
54; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
55; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
56; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
57; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
58; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
59; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
60; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
61; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
62; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
63; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
64; SKIP-CACHE-INV-NEXT:    s_endpgm
65;
66; GFX11-WGP-LABEL: flat_nontemporal_load_0:
67; GFX11-WGP:       ; %bb.0: ; %entry
68; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
69; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
70; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
71; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
72; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc dlc
73; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
74; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
75; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
76; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
77; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
78; GFX11-WGP-NEXT:    s_endpgm
79;
80; GFX11-CU-LABEL: flat_nontemporal_load_0:
81; GFX11-CU:       ; %bb.0: ; %entry
82; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
83; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
84; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
85; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
86; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1] glc dlc
87; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
88; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
89; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
90; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
91; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
92; GFX11-CU-NEXT:    s_endpgm
93    i32* %in, i32* %out) {
94entry:
95  %val = load volatile i32, i32* %in, align 4
96  store i32 %val, i32* %out
97  ret void
98}
99
100define amdgpu_kernel void @flat_nontemporal_load_1(
101; GFX7-LABEL: flat_nontemporal_load_1:
102; GFX7:       ; %bb.0: ; %entry
103; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
104; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
105; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
106; GFX7-NEXT:    v_mov_b32_e32 v1, s1
107; GFX7-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
108; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
109; GFX7-NEXT:    flat_load_dword v2, v[0:1] glc
110; GFX7-NEXT:    s_waitcnt vmcnt(0)
111; GFX7-NEXT:    v_mov_b32_e32 v0, s2
112; GFX7-NEXT:    v_mov_b32_e32 v1, s3
113; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
114; GFX7-NEXT:    flat_store_dword v[0:1], v2
115; GFX7-NEXT:    s_endpgm
116;
117; GFX10-WGP-LABEL: flat_nontemporal_load_1:
118; GFX10-WGP:       ; %bb.0: ; %entry
119; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
120; GFX10-WGP-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
121; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
122; GFX10-WGP-NEXT:    v_add_co_u32 v0, s0, s0, v0
123; GFX10-WGP-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
124; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
125; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
126; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
127; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
128; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
129; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
130; GFX10-WGP-NEXT:    s_endpgm
131;
132; GFX10-CU-LABEL: flat_nontemporal_load_1:
133; GFX10-CU:       ; %bb.0: ; %entry
134; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
135; GFX10-CU-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
136; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
137; GFX10-CU-NEXT:    v_add_co_u32 v0, s0, s0, v0
138; GFX10-CU-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
139; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
140; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
141; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
142; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
143; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
144; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
145; GFX10-CU-NEXT:    s_endpgm
146;
147; SKIP-CACHE-INV-LABEL: flat_nontemporal_load_1:
148; SKIP-CACHE-INV:       ; %bb.0: ; %entry
149; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
150; SKIP-CACHE-INV-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
151; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
152; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
153; SKIP-CACHE-INV-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
154; SKIP-CACHE-INV-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
155; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1] glc
156; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
157; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
158; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
159; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
160; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
161; SKIP-CACHE-INV-NEXT:    s_endpgm
162;
163; GFX11-WGP-LABEL: flat_nontemporal_load_1:
164; GFX11-WGP:       ; %bb.0: ; %entry
165; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
166; GFX11-WGP-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
167; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
168; GFX11-WGP-NEXT:    v_add_co_u32 v0, s0, s0, v0
169; GFX11-WGP-NEXT:    v_add_co_ci_u32_e64 v1, null, s1, 0, s0
170; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc dlc
171; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
172; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
173; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
174; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
175; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
176; GFX11-WGP-NEXT:    s_endpgm
177;
178; GFX11-CU-LABEL: flat_nontemporal_load_1:
179; GFX11-CU:       ; %bb.0: ; %entry
180; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
181; GFX11-CU-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
182; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
183; GFX11-CU-NEXT:    v_add_co_u32 v0, s0, s0, v0
184; GFX11-CU-NEXT:    v_add_co_ci_u32_e64 v1, null, s1, 0, s0
185; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1] glc dlc
186; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
187; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
188; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
189; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
190; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
191; GFX11-CU-NEXT:    s_endpgm
192    i32* %in, i32* %out) {
193entry:
194  %tid = call i32 @llvm.amdgcn.workitem.id.x()
195  %val.gep = getelementptr inbounds i32, i32* %in, i32 %tid
196  %val = load volatile i32, i32* %val.gep, align 4
197  store i32 %val, i32* %out
198  ret void
199}
200
201define amdgpu_kernel void @flat_nontemporal_store_0(
202; GFX7-LABEL: flat_nontemporal_store_0:
203; GFX7:       ; %bb.0: ; %entry
204; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
205; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
206; GFX7-NEXT:    v_mov_b32_e32 v0, s0
207; GFX7-NEXT:    v_mov_b32_e32 v1, s1
208; GFX7-NEXT:    flat_load_dword v2, v[0:1]
209; GFX7-NEXT:    v_mov_b32_e32 v0, s2
210; GFX7-NEXT:    v_mov_b32_e32 v1, s3
211; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
212; GFX7-NEXT:    flat_store_dword v[0:1], v2
213; GFX7-NEXT:    s_waitcnt vmcnt(0)
214; GFX7-NEXT:    s_endpgm
215;
216; GFX10-WGP-LABEL: flat_nontemporal_store_0:
217; GFX10-WGP:       ; %bb.0: ; %entry
218; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
219; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
220; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
221; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
222; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
223; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
224; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
225; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
226; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
227; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
228; GFX10-WGP-NEXT:    s_endpgm
229;
230; GFX10-CU-LABEL: flat_nontemporal_store_0:
231; GFX10-CU:       ; %bb.0: ; %entry
232; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
233; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
234; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
235; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
236; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
237; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
238; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
239; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
240; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
241; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
242; GFX10-CU-NEXT:    s_endpgm
243;
244; SKIP-CACHE-INV-LABEL: flat_nontemporal_store_0:
245; SKIP-CACHE-INV:       ; %bb.0: ; %entry
246; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
247; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
248; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
249; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
250; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
251; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
252; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
253; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
254; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
255; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
256; SKIP-CACHE-INV-NEXT:    s_endpgm
257;
258; GFX11-WGP-LABEL: flat_nontemporal_store_0:
259; GFX11-WGP:       ; %bb.0: ; %entry
260; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
261; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
262; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
263; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
264; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1]
265; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
266; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
267; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
268; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2 dlc
269; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
270; GFX11-WGP-NEXT:    s_endpgm
271;
272; GFX11-CU-LABEL: flat_nontemporal_store_0:
273; GFX11-CU:       ; %bb.0: ; %entry
274; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
275; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
276; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
277; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
278; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
279; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
280; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
281; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
282; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2 dlc
283; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
284; GFX11-CU-NEXT:    s_endpgm
285    i32* %in, i32* %out) {
286entry:
287  %val = load i32, i32* %in, align 4
288  store volatile i32 %val, i32* %out
289  ret void
290}
291
292define amdgpu_kernel void @flat_nontemporal_store_1(
293; GFX7-LABEL: flat_nontemporal_store_1:
294; GFX7:       ; %bb.0: ; %entry
295; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
296; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
297; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
298; GFX7-NEXT:    v_mov_b32_e32 v1, s0
299; GFX7-NEXT:    v_mov_b32_e32 v2, s1
300; GFX7-NEXT:    flat_load_dword v2, v[1:2]
301; GFX7-NEXT:    v_mov_b32_e32 v1, s3
302; GFX7-NEXT:    v_add_i32_e32 v0, vcc, s2, v0
303; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
304; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
305; GFX7-NEXT:    flat_store_dword v[0:1], v2
306; GFX7-NEXT:    s_waitcnt vmcnt(0)
307; GFX7-NEXT:    s_endpgm
308;
309; GFX10-WGP-LABEL: flat_nontemporal_store_1:
310; GFX10-WGP:       ; %bb.0: ; %entry
311; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
312; GFX10-WGP-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
313; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
314; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
315; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
316; GFX10-WGP-NEXT:    v_add_co_u32 v0, s0, s2, v0
317; GFX10-WGP-NEXT:    flat_load_dword v2, v[1:2]
318; GFX10-WGP-NEXT:    v_add_co_ci_u32_e64 v1, s0, s3, 0, s0
319; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
320; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
321; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
322; GFX10-WGP-NEXT:    s_endpgm
323;
324; GFX10-CU-LABEL: flat_nontemporal_store_1:
325; GFX10-CU:       ; %bb.0: ; %entry
326; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
327; GFX10-CU-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
328; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
329; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
330; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
331; GFX10-CU-NEXT:    v_add_co_u32 v0, s0, s2, v0
332; GFX10-CU-NEXT:    flat_load_dword v2, v[1:2]
333; GFX10-CU-NEXT:    v_add_co_ci_u32_e64 v1, s0, s3, 0, s0
334; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
335; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
336; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
337; GFX10-CU-NEXT:    s_endpgm
338;
339; SKIP-CACHE-INV-LABEL: flat_nontemporal_store_1:
340; SKIP-CACHE-INV:       ; %bb.0: ; %entry
341; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
342; SKIP-CACHE-INV-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
343; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
344; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
345; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
346; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[1:2]
347; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
348; SKIP-CACHE-INV-NEXT:    v_add_i32_e32 v0, vcc, s2, v0
349; SKIP-CACHE-INV-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
350; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
351; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
352; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
353; SKIP-CACHE-INV-NEXT:    s_endpgm
354;
355; GFX11-WGP-LABEL: flat_nontemporal_store_1:
356; GFX11-WGP:       ; %bb.0: ; %entry
357; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
358; GFX11-WGP-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
359; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
360; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s0
361; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s1
362; GFX11-WGP-NEXT:    v_add_co_u32 v0, s0, s2, v0
363; GFX11-WGP-NEXT:    flat_load_b32 v2, v[1:2]
364; GFX11-WGP-NEXT:    v_add_co_ci_u32_e64 v1, null, s3, 0, s0
365; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
366; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2 dlc
367; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
368; GFX11-WGP-NEXT:    s_endpgm
369;
370; GFX11-CU-LABEL: flat_nontemporal_store_1:
371; GFX11-CU:       ; %bb.0: ; %entry
372; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
373; GFX11-CU-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
374; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
375; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s0
376; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s1
377; GFX11-CU-NEXT:    v_add_co_u32 v0, s0, s2, v0
378; GFX11-CU-NEXT:    flat_load_b32 v2, v[1:2]
379; GFX11-CU-NEXT:    v_add_co_ci_u32_e64 v1, null, s3, 0, s0
380; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
381; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2 dlc
382; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
383; GFX11-CU-NEXT:    s_endpgm
384    i32* %in, i32* %out) {
385entry:
386  %tid = call i32 @llvm.amdgcn.workitem.id.x()
387  %val = load i32, i32* %in, align 4
388  %out.gep = getelementptr inbounds i32, i32* %out, i32 %tid
389  store volatile i32 %val, i32* %out.gep
390  ret void
391}
392
393define amdgpu_kernel void @flat_volatile_workgroup_acquire_load(
394; GFX7-LABEL: flat_volatile_workgroup_acquire_load:
395; GFX7:       ; %bb.0: ; %entry
396; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
397; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
398; GFX7-NEXT:    v_mov_b32_e32 v0, s0
399; GFX7-NEXT:    v_mov_b32_e32 v1, s1
400; GFX7-NEXT:    flat_load_dword v2, v[0:1]
401; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
402; GFX7-NEXT:    v_mov_b32_e32 v0, s2
403; GFX7-NEXT:    v_mov_b32_e32 v1, s3
404; GFX7-NEXT:    s_waitcnt vmcnt(0)
405; GFX7-NEXT:    flat_store_dword v[0:1], v2
406; GFX7-NEXT:    s_endpgm
407;
408; GFX10-WGP-LABEL: flat_volatile_workgroup_acquire_load:
409; GFX10-WGP:       ; %bb.0: ; %entry
410; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
411; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
412; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
413; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
414; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc
415; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
416; GFX10-WGP-NEXT:    buffer_gl0_inv
417; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
418; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
419; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
420; GFX10-WGP-NEXT:    s_endpgm
421;
422; GFX10-CU-LABEL: flat_volatile_workgroup_acquire_load:
423; GFX10-CU:       ; %bb.0: ; %entry
424; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
425; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
426; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
427; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
428; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
429; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
430; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
431; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
432; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
433; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
434; GFX10-CU-NEXT:    s_endpgm
435;
436; SKIP-CACHE-INV-LABEL: flat_volatile_workgroup_acquire_load:
437; SKIP-CACHE-INV:       ; %bb.0: ; %entry
438; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
439; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
440; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
441; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
442; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[0:1]
443; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
444; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
445; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
446; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
447; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
448; SKIP-CACHE-INV-NEXT:    s_endpgm
449;
450; GFX11-WGP-LABEL: flat_volatile_workgroup_acquire_load:
451; GFX11-WGP:       ; %bb.0: ; %entry
452; GFX11-WGP-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
453; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
454; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s0
455; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
456; GFX11-WGP-NEXT:    flat_load_b32 v2, v[0:1] glc
457; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
458; GFX11-WGP-NEXT:    buffer_gl0_inv
459; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
460; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
461; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
462; GFX11-WGP-NEXT:    s_endpgm
463;
464; GFX11-CU-LABEL: flat_volatile_workgroup_acquire_load:
465; GFX11-CU:       ; %bb.0: ; %entry
466; GFX11-CU-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
467; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
468; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s0
469; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
470; GFX11-CU-NEXT:    flat_load_b32 v2, v[0:1]
471; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
472; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
473; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
474; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
475; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
476; GFX11-CU-NEXT:    s_endpgm
477    i32* %in, i32* %out) {
478entry:
479  %val = load atomic volatile i32, i32* %in syncscope("workgroup") acquire, align 4
480  store i32 %val, i32* %out
481  ret void
482}
483
484define amdgpu_kernel void @flat_volatile_workgroup_release_store(
485; GFX7-LABEL: flat_volatile_workgroup_release_store:
486; GFX7:       ; %bb.0: ; %entry
487; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
488; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
489; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
490; GFX7-NEXT:    v_mov_b32_e32 v0, s0
491; GFX7-NEXT:    v_mov_b32_e32 v1, s1
492; GFX7-NEXT:    v_mov_b32_e32 v2, s2
493; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
494; GFX7-NEXT:    flat_store_dword v[0:1], v2
495; GFX7-NEXT:    s_endpgm
496;
497; GFX10-WGP-LABEL: flat_volatile_workgroup_release_store:
498; GFX10-WGP:       ; %bb.0: ; %entry
499; GFX10-WGP-NEXT:    s_clause 0x1
500; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
501; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
502; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
503; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
504; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
505; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
506; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
507; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
508; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
509; GFX10-WGP-NEXT:    s_endpgm
510;
511; GFX10-CU-LABEL: flat_volatile_workgroup_release_store:
512; GFX10-CU:       ; %bb.0: ; %entry
513; GFX10-CU-NEXT:    s_clause 0x1
514; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
515; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
516; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
517; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
518; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
519; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
520; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
521; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
522; GFX10-CU-NEXT:    s_endpgm
523;
524; SKIP-CACHE-INV-LABEL: flat_volatile_workgroup_release_store:
525; SKIP-CACHE-INV:       ; %bb.0: ; %entry
526; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2
527; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
528; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
529; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
530; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
531; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
532; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
533; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
534; SKIP-CACHE-INV-NEXT:    s_endpgm
535;
536; GFX11-WGP-LABEL: flat_volatile_workgroup_release_store:
537; GFX11-WGP:       ; %bb.0: ; %entry
538; GFX11-WGP-NEXT:    s_clause 0x1
539; GFX11-WGP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
540; GFX11-WGP-NEXT:    s_load_b32 s0, s[0:1], 0x0
541; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
542; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s2
543; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s3
544; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
545; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
546; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
547; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
548; GFX11-WGP-NEXT:    s_endpgm
549;
550; GFX11-CU-LABEL: flat_volatile_workgroup_release_store:
551; GFX11-CU:       ; %bb.0: ; %entry
552; GFX11-CU-NEXT:    s_clause 0x1
553; GFX11-CU-NEXT:    s_load_b64 s[2:3], s[0:1], 0x8
554; GFX11-CU-NEXT:    s_load_b32 s0, s[0:1], 0x0
555; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
556; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s2
557; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s3
558; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
559; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
560; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
561; GFX11-CU-NEXT:    s_endpgm
562   i32 %in, i32* %out) {
563entry:
564  store atomic volatile i32 %in, i32* %out syncscope("workgroup") release, align 4
565  ret void
566}
567
568declare i32 @llvm.amdgcn.workitem.id.x()
569