1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefix=GFX9 %s
3; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefix=VI %s
4; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefix=CI %s
5; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefix=GFX10 %s
6
7define amdgpu_kernel void @s_lshr_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> %lhs, <2 x i16> %rhs) #0 {
8; GFX9-LABEL: s_lshr_v2i16:
9; GFX9:       ; %bb.0:
10; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
11; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
12; GFX9-NEXT:    v_mov_b32_e32 v0, 0
13; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
14; GFX9-NEXT:    v_mov_b32_e32 v1, s2
15; GFX9-NEXT:    v_pk_lshrrev_b16 v1, s3, v1
16; GFX9-NEXT:    global_store_dword v0, v1, s[4:5]
17; GFX9-NEXT:    s_endpgm
18;
19; VI-LABEL: s_lshr_v2i16:
20; VI:       ; %bb.0:
21; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
22; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
23; VI-NEXT:    s_waitcnt lgkmcnt(0)
24; VI-NEXT:    s_and_b32 s4, s2, 0xffff
25; VI-NEXT:    s_lshr_b32 s2, s2, 16
26; VI-NEXT:    s_lshr_b32 s5, s3, 16
27; VI-NEXT:    s_lshr_b32 s2, s2, s5
28; VI-NEXT:    s_lshr_b32 s3, s4, s3
29; VI-NEXT:    s_lshl_b32 s2, s2, 16
30; VI-NEXT:    s_or_b32 s2, s3, s2
31; VI-NEXT:    v_mov_b32_e32 v0, s0
32; VI-NEXT:    v_mov_b32_e32 v1, s1
33; VI-NEXT:    v_mov_b32_e32 v2, s2
34; VI-NEXT:    flat_store_dword v[0:1], v2
35; VI-NEXT:    s_endpgm
36;
37; CI-LABEL: s_lshr_v2i16:
38; CI:       ; %bb.0:
39; CI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xb
40; CI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
41; CI-NEXT:    s_mov_b32 s3, 0xf000
42; CI-NEXT:    s_mov_b32 s2, -1
43; CI-NEXT:    s_waitcnt lgkmcnt(0)
44; CI-NEXT:    s_and_b32 s6, s4, 0xffff
45; CI-NEXT:    s_lshr_b32 s4, s4, 16
46; CI-NEXT:    s_lshr_b32 s7, s5, 16
47; CI-NEXT:    s_lshr_b32 s4, s4, s7
48; CI-NEXT:    s_lshl_b32 s4, s4, 16
49; CI-NEXT:    s_lshr_b32 s5, s6, s5
50; CI-NEXT:    s_or_b32 s4, s5, s4
51; CI-NEXT:    v_mov_b32_e32 v0, s4
52; CI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
53; CI-NEXT:    s_endpgm
54;
55; GFX10-LABEL: s_lshr_v2i16:
56; GFX10:       ; %bb.0:
57; GFX10-NEXT:    s_clause 0x1
58; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
59; GFX10-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
60; GFX10-NEXT:    v_mov_b32_e32 v0, 0
61; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
62; GFX10-NEXT:    v_pk_lshrrev_b16 v1, s3, s2
63; GFX10-NEXT:    global_store_dword v0, v1, s[4:5]
64; GFX10-NEXT:    s_endpgm
65  %result = lshr <2 x i16> %lhs, %rhs
66  store <2 x i16> %result, <2 x i16> addrspace(1)* %out
67  ret void
68}
69
70define amdgpu_kernel void @v_lshr_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) #0 {
71; GFX9-LABEL: v_lshr_v2i16:
72; GFX9:       ; %bb.0:
73; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
74; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
75; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
76; GFX9-NEXT:    global_load_dwordx2 v[0:1], v2, s[2:3]
77; GFX9-NEXT:    s_waitcnt vmcnt(0)
78; GFX9-NEXT:    v_pk_lshrrev_b16 v0, v1, v0
79; GFX9-NEXT:    global_store_dword v2, v0, s[0:1]
80; GFX9-NEXT:    s_endpgm
81;
82; VI-LABEL: v_lshr_v2i16:
83; VI:       ; %bb.0:
84; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
85; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
86; VI-NEXT:    s_waitcnt lgkmcnt(0)
87; VI-NEXT:    v_mov_b32_e32 v1, s3
88; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
89; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
90; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
91; VI-NEXT:    v_mov_b32_e32 v3, s1
92; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v2
93; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
94; VI-NEXT:    s_waitcnt vmcnt(0)
95; VI-NEXT:    v_lshrrev_b16_e32 v4, v1, v0
96; VI-NEXT:    v_lshrrev_b16_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
97; VI-NEXT:    v_or_b32_e32 v0, v4, v0
98; VI-NEXT:    flat_store_dword v[2:3], v0
99; VI-NEXT:    s_endpgm
100;
101; CI-LABEL: v_lshr_v2i16:
102; CI:       ; %bb.0:
103; CI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
104; CI-NEXT:    s_mov_b32 s7, 0xf000
105; CI-NEXT:    s_mov_b32 s6, 0
106; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
107; CI-NEXT:    v_mov_b32_e32 v1, 0
108; CI-NEXT:    s_waitcnt lgkmcnt(0)
109; CI-NEXT:    s_mov_b64 s[4:5], s[2:3]
110; CI-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64
111; CI-NEXT:    s_mov_b64 s[2:3], s[6:7]
112; CI-NEXT:    s_waitcnt vmcnt(0)
113; CI-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
114; CI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
115; CI-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
116; CI-NEXT:    v_lshrrev_b32_e32 v2, v3, v2
117; CI-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
118; CI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
119; CI-NEXT:    v_or_b32_e32 v2, v2, v3
120; CI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
121; CI-NEXT:    s_endpgm
122;
123; GFX10-LABEL: v_lshr_v2i16:
124; GFX10:       ; %bb.0:
125; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
126; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
127; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
128; GFX10-NEXT:    global_load_dwordx2 v[0:1], v2, s[2:3]
129; GFX10-NEXT:    s_waitcnt vmcnt(0)
130; GFX10-NEXT:    v_pk_lshrrev_b16 v0, v1, v0
131; GFX10-NEXT:    global_store_dword v2, v0, s[0:1]
132; GFX10-NEXT:    s_endpgm
133  %tid = call i32 @llvm.amdgcn.workitem.id.x()
134  %tid.ext = sext i32 %tid to i64
135  %in.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
136  %out.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %out, i64 %tid.ext
137  %b_ptr = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %in.gep, i32 1
138  %a = load <2 x i16>, <2 x i16> addrspace(1)* %in.gep
139  %b = load <2 x i16>, <2 x i16> addrspace(1)* %b_ptr
140  %result = lshr <2 x i16> %a, %b
141  store <2 x i16> %result, <2 x i16> addrspace(1)* %out.gep
142  ret void
143}
144
145define amdgpu_kernel void @lshr_v_s_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in, <2 x i16> %sgpr) #0 {
146; GFX9-LABEL: lshr_v_s_v2i16:
147; GFX9:       ; %bb.0:
148; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
149; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x34
150; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
151; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
152; GFX9-NEXT:    global_load_dword v1, v0, s[6:7]
153; GFX9-NEXT:    s_waitcnt vmcnt(0)
154; GFX9-NEXT:    v_pk_lshrrev_b16 v1, s2, v1
155; GFX9-NEXT:    global_store_dword v0, v1, s[4:5]
156; GFX9-NEXT:    s_endpgm
157;
158; VI-LABEL: lshr_v_s_v2i16:
159; VI:       ; %bb.0:
160; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
161; VI-NEXT:    s_load_dword s0, s[0:1], 0x34
162; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
163; VI-NEXT:    s_waitcnt lgkmcnt(0)
164; VI-NEXT:    v_mov_b32_e32 v1, s7
165; VI-NEXT:    v_add_u32_e32 v0, vcc, s6, v2
166; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
167; VI-NEXT:    flat_load_dword v3, v[0:1]
168; VI-NEXT:    s_lshr_b32 s1, s0, 16
169; VI-NEXT:    v_add_u32_e32 v0, vcc, s4, v2
170; VI-NEXT:    v_mov_b32_e32 v2, s1
171; VI-NEXT:    v_mov_b32_e32 v1, s5
172; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
173; VI-NEXT:    s_waitcnt vmcnt(0)
174; VI-NEXT:    v_lshrrev_b16_e32 v4, s0, v3
175; VI-NEXT:    v_lshrrev_b16_sdwa v2, v2, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
176; VI-NEXT:    v_or_b32_e32 v2, v4, v2
177; VI-NEXT:    flat_store_dword v[0:1], v2
178; VI-NEXT:    s_endpgm
179;
180; CI-LABEL: lshr_v_s_v2i16:
181; CI:       ; %bb.0:
182; CI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
183; CI-NEXT:    s_load_dword s8, s[0:1], 0xd
184; CI-NEXT:    s_mov_b32 s3, 0xf000
185; CI-NEXT:    s_mov_b32 s2, 0
186; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
187; CI-NEXT:    s_waitcnt lgkmcnt(0)
188; CI-NEXT:    s_mov_b64 s[0:1], s[6:7]
189; CI-NEXT:    v_mov_b32_e32 v1, 0
190; CI-NEXT:    buffer_load_dword v2, v[0:1], s[0:3], 0 addr64
191; CI-NEXT:    s_lshr_b32 s0, s8, 16
192; CI-NEXT:    s_mov_b64 s[6:7], s[2:3]
193; CI-NEXT:    s_waitcnt vmcnt(0)
194; CI-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
195; CI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
196; CI-NEXT:    v_lshrrev_b32_e32 v3, s0, v3
197; CI-NEXT:    v_lshrrev_b32_e32 v2, s8, v2
198; CI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
199; CI-NEXT:    v_or_b32_e32 v2, v2, v3
200; CI-NEXT:    buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
201; CI-NEXT:    s_endpgm
202;
203; GFX10-LABEL: lshr_v_s_v2i16:
204; GFX10:       ; %bb.0:
205; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
206; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
207; GFX10-NEXT:    s_load_dword s0, s[0:1], 0x34
208; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
209; GFX10-NEXT:    global_load_dword v1, v0, s[6:7]
210; GFX10-NEXT:    s_waitcnt vmcnt(0)
211; GFX10-NEXT:    v_pk_lshrrev_b16 v1, s0, v1
212; GFX10-NEXT:    global_store_dword v0, v1, s[4:5]
213; GFX10-NEXT:    s_endpgm
214  %tid = call i32 @llvm.amdgcn.workitem.id.x()
215  %tid.ext = sext i32 %tid to i64
216  %in.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
217  %out.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %out, i64 %tid.ext
218  %vgpr = load <2 x i16>, <2 x i16> addrspace(1)* %in.gep
219  %result = lshr <2 x i16> %vgpr, %sgpr
220  store <2 x i16> %result, <2 x i16> addrspace(1)* %out.gep
221  ret void
222}
223
224define amdgpu_kernel void @lshr_s_v_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in, <2 x i16> %sgpr) #0 {
225; GFX9-LABEL: lshr_s_v_v2i16:
226; GFX9:       ; %bb.0:
227; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
228; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x34
229; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
230; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
231; GFX9-NEXT:    global_load_dword v1, v0, s[6:7]
232; GFX9-NEXT:    s_waitcnt vmcnt(0)
233; GFX9-NEXT:    v_pk_lshrrev_b16 v1, v1, s2
234; GFX9-NEXT:    global_store_dword v0, v1, s[4:5]
235; GFX9-NEXT:    s_endpgm
236;
237; VI-LABEL: lshr_s_v_v2i16:
238; VI:       ; %bb.0:
239; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
240; VI-NEXT:    s_load_dword s0, s[0:1], 0x34
241; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
242; VI-NEXT:    s_waitcnt lgkmcnt(0)
243; VI-NEXT:    v_mov_b32_e32 v1, s7
244; VI-NEXT:    v_add_u32_e32 v0, vcc, s6, v2
245; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
246; VI-NEXT:    flat_load_dword v3, v[0:1]
247; VI-NEXT:    s_lshr_b32 s1, s0, 16
248; VI-NEXT:    v_add_u32_e32 v0, vcc, s4, v2
249; VI-NEXT:    v_mov_b32_e32 v2, s1
250; VI-NEXT:    v_mov_b32_e32 v1, s5
251; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
252; VI-NEXT:    s_waitcnt vmcnt(0)
253; VI-NEXT:    v_lshrrev_b16_e64 v4, v3, s0
254; VI-NEXT:    v_lshrrev_b16_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
255; VI-NEXT:    v_or_b32_e32 v2, v4, v2
256; VI-NEXT:    flat_store_dword v[0:1], v2
257; VI-NEXT:    s_endpgm
258;
259; CI-LABEL: lshr_s_v_v2i16:
260; CI:       ; %bb.0:
261; CI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
262; CI-NEXT:    s_load_dword s8, s[0:1], 0xd
263; CI-NEXT:    s_mov_b32 s3, 0xf000
264; CI-NEXT:    s_mov_b32 s2, 0
265; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
266; CI-NEXT:    s_waitcnt lgkmcnt(0)
267; CI-NEXT:    s_mov_b64 s[0:1], s[6:7]
268; CI-NEXT:    v_mov_b32_e32 v1, 0
269; CI-NEXT:    buffer_load_dword v2, v[0:1], s[0:3], 0 addr64
270; CI-NEXT:    s_lshr_b32 s0, s8, 16
271; CI-NEXT:    s_and_b32 s1, s8, 0xffff
272; CI-NEXT:    s_mov_b64 s[6:7], s[2:3]
273; CI-NEXT:    s_waitcnt vmcnt(0)
274; CI-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
275; CI-NEXT:    v_lshr_b32_e32 v3, s0, v3
276; CI-NEXT:    v_lshr_b32_e32 v2, s1, v2
277; CI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
278; CI-NEXT:    v_or_b32_e32 v2, v2, v3
279; CI-NEXT:    buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
280; CI-NEXT:    s_endpgm
281;
282; GFX10-LABEL: lshr_s_v_v2i16:
283; GFX10:       ; %bb.0:
284; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
285; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
286; GFX10-NEXT:    s_load_dword s0, s[0:1], 0x34
287; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
288; GFX10-NEXT:    global_load_dword v1, v0, s[6:7]
289; GFX10-NEXT:    s_waitcnt vmcnt(0)
290; GFX10-NEXT:    v_pk_lshrrev_b16 v1, v1, s0
291; GFX10-NEXT:    global_store_dword v0, v1, s[4:5]
292; GFX10-NEXT:    s_endpgm
293  %tid = call i32 @llvm.amdgcn.workitem.id.x()
294  %tid.ext = sext i32 %tid to i64
295  %in.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
296  %out.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %out, i64 %tid.ext
297  %vgpr = load <2 x i16>, <2 x i16> addrspace(1)* %in.gep
298  %result = lshr <2 x i16> %sgpr, %vgpr
299  store <2 x i16> %result, <2 x i16> addrspace(1)* %out.gep
300  ret void
301}
302
303define amdgpu_kernel void @lshr_imm_v_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) #0 {
304; GFX9-LABEL: lshr_imm_v_v2i16:
305; GFX9:       ; %bb.0:
306; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
307; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
308; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
309; GFX9-NEXT:    global_load_dword v1, v0, s[2:3]
310; GFX9-NEXT:    s_waitcnt vmcnt(0)
311; GFX9-NEXT:    v_pk_lshrrev_b16 v1, v1, 8 op_sel_hi:[1,0]
312; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
313; GFX9-NEXT:    s_endpgm
314;
315; VI-LABEL: lshr_imm_v_v2i16:
316; VI:       ; %bb.0:
317; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
318; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
319; VI-NEXT:    v_mov_b32_e32 v4, 8
320; VI-NEXT:    s_waitcnt lgkmcnt(0)
321; VI-NEXT:    v_mov_b32_e32 v1, s3
322; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
323; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
324; VI-NEXT:    flat_load_dword v3, v[0:1]
325; VI-NEXT:    v_mov_b32_e32 v1, s1
326; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
327; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
328; VI-NEXT:    s_waitcnt vmcnt(0)
329; VI-NEXT:    v_lshrrev_b16_e64 v2, v3, 8
330; VI-NEXT:    v_lshrrev_b16_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
331; VI-NEXT:    v_or_b32_e32 v2, v2, v3
332; VI-NEXT:    flat_store_dword v[0:1], v2
333; VI-NEXT:    s_endpgm
334;
335; CI-LABEL: lshr_imm_v_v2i16:
336; CI:       ; %bb.0:
337; CI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
338; CI-NEXT:    s_mov_b32 s7, 0xf000
339; CI-NEXT:    s_mov_b32 s6, 0
340; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
341; CI-NEXT:    v_mov_b32_e32 v1, 0
342; CI-NEXT:    s_waitcnt lgkmcnt(0)
343; CI-NEXT:    s_mov_b64 s[4:5], s[2:3]
344; CI-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
345; CI-NEXT:    s_mov_b64 s[2:3], s[6:7]
346; CI-NEXT:    s_waitcnt vmcnt(0)
347; CI-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
348; CI-NEXT:    v_lshr_b32_e32 v3, 8, v3
349; CI-NEXT:    v_lshr_b32_e32 v2, 8, v2
350; CI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
351; CI-NEXT:    v_or_b32_e32 v2, v2, v3
352; CI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
353; CI-NEXT:    s_endpgm
354;
355; GFX10-LABEL: lshr_imm_v_v2i16:
356; GFX10:       ; %bb.0:
357; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
358; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
359; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
360; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
361; GFX10-NEXT:    s_waitcnt vmcnt(0)
362; GFX10-NEXT:    v_pk_lshrrev_b16 v1, v1, 8 op_sel_hi:[1,0]
363; GFX10-NEXT:    global_store_dword v0, v1, s[0:1]
364; GFX10-NEXT:    s_endpgm
365  %tid = call i32 @llvm.amdgcn.workitem.id.x()
366  %tid.ext = sext i32 %tid to i64
367  %in.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
368  %out.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %out, i64 %tid.ext
369  %vgpr = load <2 x i16>, <2 x i16> addrspace(1)* %in.gep
370  %result = lshr <2 x i16> <i16 8, i16 8>, %vgpr
371  store <2 x i16> %result, <2 x i16> addrspace(1)* %out.gep
372  ret void
373}
374
375define amdgpu_kernel void @lshr_v_imm_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) #0 {
376; GFX9-LABEL: lshr_v_imm_v2i16:
377; GFX9:       ; %bb.0:
378; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
379; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
380; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
381; GFX9-NEXT:    global_load_dword v1, v0, s[2:3]
382; GFX9-NEXT:    s_waitcnt vmcnt(0)
383; GFX9-NEXT:    v_pk_lshrrev_b16 v1, 8, v1 op_sel_hi:[0,1]
384; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
385; GFX9-NEXT:    s_endpgm
386;
387; VI-LABEL: lshr_v_imm_v2i16:
388; VI:       ; %bb.0:
389; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
390; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
391; VI-NEXT:    s_waitcnt lgkmcnt(0)
392; VI-NEXT:    v_mov_b32_e32 v1, s3
393; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
394; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
395; VI-NEXT:    flat_load_dword v3, v[0:1]
396; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
397; VI-NEXT:    v_mov_b32_e32 v1, s1
398; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
399; VI-NEXT:    s_waitcnt vmcnt(0)
400; VI-NEXT:    v_lshrrev_b32_e32 v2, 24, v3
401; VI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
402; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
403; VI-NEXT:    flat_store_dword v[0:1], v2
404; VI-NEXT:    s_endpgm
405;
406; CI-LABEL: lshr_v_imm_v2i16:
407; CI:       ; %bb.0:
408; CI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
409; CI-NEXT:    s_mov_b32 s7, 0xf000
410; CI-NEXT:    s_mov_b32 s6, 0
411; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
412; CI-NEXT:    v_mov_b32_e32 v1, 0
413; CI-NEXT:    s_waitcnt lgkmcnt(0)
414; CI-NEXT:    s_mov_b64 s[4:5], s[2:3]
415; CI-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
416; CI-NEXT:    s_mov_b64 s[2:3], s[6:7]
417; CI-NEXT:    s_waitcnt vmcnt(0)
418; CI-NEXT:    v_lshrrev_b32_e32 v2, 8, v2
419; CI-NEXT:    v_and_b32_e32 v2, 0xff00ff, v2
420; CI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
421; CI-NEXT:    s_endpgm
422;
423; GFX10-LABEL: lshr_v_imm_v2i16:
424; GFX10:       ; %bb.0:
425; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
426; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
427; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
428; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
429; GFX10-NEXT:    s_waitcnt vmcnt(0)
430; GFX10-NEXT:    v_pk_lshrrev_b16 v1, 8, v1 op_sel_hi:[0,1]
431; GFX10-NEXT:    global_store_dword v0, v1, s[0:1]
432; GFX10-NEXT:    s_endpgm
433  %tid = call i32 @llvm.amdgcn.workitem.id.x()
434  %tid.ext = sext i32 %tid to i64
435  %in.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
436  %out.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %out, i64 %tid.ext
437  %vgpr = load <2 x i16>, <2 x i16> addrspace(1)* %in.gep
438  %result = lshr <2 x i16> %vgpr, <i16 8, i16 8>
439  store <2 x i16> %result, <2 x i16> addrspace(1)* %out.gep
440  ret void
441}
442
443define amdgpu_kernel void @v_lshr_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> addrspace(1)* %in) #0 {
444; GFX9-LABEL: v_lshr_v4i16:
445; GFX9:       ; %bb.0:
446; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
447; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
448; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
449; GFX9-NEXT:    global_load_dwordx4 v[0:3], v4, s[2:3]
450; GFX9-NEXT:    s_waitcnt vmcnt(0)
451; GFX9-NEXT:    v_pk_lshrrev_b16 v1, v3, v1
452; GFX9-NEXT:    v_pk_lshrrev_b16 v0, v2, v0
453; GFX9-NEXT:    global_store_dwordx2 v4, v[0:1], s[0:1]
454; GFX9-NEXT:    s_endpgm
455;
456; VI-LABEL: v_lshr_v4i16:
457; VI:       ; %bb.0:
458; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
459; VI-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
460; VI-NEXT:    s_waitcnt lgkmcnt(0)
461; VI-NEXT:    v_mov_b32_e32 v1, s3
462; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v4
463; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
464; VI-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
465; VI-NEXT:    v_mov_b32_e32 v5, s1
466; VI-NEXT:    v_add_u32_e32 v4, vcc, s0, v4
467; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
468; VI-NEXT:    s_waitcnt vmcnt(0)
469; VI-NEXT:    v_lshrrev_b16_e32 v6, v3, v1
470; VI-NEXT:    v_lshrrev_b16_sdwa v1, v3, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
471; VI-NEXT:    v_lshrrev_b16_e32 v3, v2, v0
472; VI-NEXT:    v_lshrrev_b16_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
473; VI-NEXT:    v_or_b32_e32 v1, v6, v1
474; VI-NEXT:    v_or_b32_e32 v0, v3, v0
475; VI-NEXT:    flat_store_dwordx2 v[4:5], v[0:1]
476; VI-NEXT:    s_endpgm
477;
478; CI-LABEL: v_lshr_v4i16:
479; CI:       ; %bb.0:
480; CI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
481; CI-NEXT:    s_mov_b32 s7, 0xf000
482; CI-NEXT:    s_mov_b32 s6, 0
483; CI-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
484; CI-NEXT:    v_mov_b32_e32 v5, 0
485; CI-NEXT:    s_waitcnt lgkmcnt(0)
486; CI-NEXT:    s_mov_b64 s[4:5], s[2:3]
487; CI-NEXT:    buffer_load_dwordx4 v[0:3], v[4:5], s[4:7], 0 addr64
488; CI-NEXT:    s_mov_b64 s[2:3], s[6:7]
489; CI-NEXT:    s_waitcnt vmcnt(0)
490; CI-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
491; CI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
492; CI-NEXT:    v_lshrrev_b32_e32 v7, 16, v1
493; CI-NEXT:    v_and_b32_e32 v1, 0xffff, v1
494; CI-NEXT:    v_lshrrev_b32_e32 v8, 16, v2
495; CI-NEXT:    v_lshrrev_b32_e32 v9, 16, v3
496; CI-NEXT:    v_lshrrev_b32_e32 v1, v3, v1
497; CI-NEXT:    v_lshrrev_b32_e32 v3, v9, v7
498; CI-NEXT:    v_lshrrev_b32_e32 v0, v2, v0
499; CI-NEXT:    v_lshrrev_b32_e32 v2, v8, v6
500; CI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
501; CI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
502; CI-NEXT:    v_or_b32_e32 v1, v1, v3
503; CI-NEXT:    v_or_b32_e32 v0, v0, v2
504; CI-NEXT:    buffer_store_dwordx2 v[0:1], v[4:5], s[0:3], 0 addr64
505; CI-NEXT:    s_endpgm
506;
507; GFX10-LABEL: v_lshr_v4i16:
508; GFX10:       ; %bb.0:
509; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
510; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
511; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
512; GFX10-NEXT:    global_load_dwordx4 v[0:3], v4, s[2:3]
513; GFX10-NEXT:    s_waitcnt vmcnt(0)
514; GFX10-NEXT:    v_pk_lshrrev_b16 v1, v3, v1
515; GFX10-NEXT:    v_pk_lshrrev_b16 v0, v2, v0
516; GFX10-NEXT:    global_store_dwordx2 v4, v[0:1], s[0:1]
517; GFX10-NEXT:    s_endpgm
518  %tid = call i32 @llvm.amdgcn.workitem.id.x()
519  %tid.ext = sext i32 %tid to i64
520  %in.gep = getelementptr inbounds <4 x i16>, <4 x i16> addrspace(1)* %in, i64 %tid.ext
521  %out.gep = getelementptr inbounds <4 x i16>, <4 x i16> addrspace(1)* %out, i64 %tid.ext
522  %b_ptr = getelementptr <4 x i16>, <4 x i16> addrspace(1)* %in.gep, i32 1
523  %a = load <4 x i16>, <4 x i16> addrspace(1)* %in.gep
524  %b = load <4 x i16>, <4 x i16> addrspace(1)* %b_ptr
525  %result = lshr <4 x i16> %a, %b
526  store <4 x i16> %result, <4 x i16> addrspace(1)* %out.gep
527  ret void
528}
529
530define amdgpu_kernel void @lshr_v_imm_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> addrspace(1)* %in) #0 {
531; GFX9-LABEL: lshr_v_imm_v4i16:
532; GFX9:       ; %bb.0:
533; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
534; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
535; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
536; GFX9-NEXT:    global_load_dwordx2 v[0:1], v2, s[2:3]
537; GFX9-NEXT:    s_waitcnt vmcnt(0)
538; GFX9-NEXT:    v_pk_lshrrev_b16 v1, 8, v1 op_sel_hi:[0,1]
539; GFX9-NEXT:    v_pk_lshrrev_b16 v0, 8, v0 op_sel_hi:[0,1]
540; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
541; GFX9-NEXT:    s_endpgm
542;
543; VI-LABEL: lshr_v_imm_v4i16:
544; VI:       ; %bb.0:
545; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
546; VI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
547; VI-NEXT:    s_waitcnt lgkmcnt(0)
548; VI-NEXT:    v_mov_b32_e32 v1, s3
549; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
550; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
551; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
552; VI-NEXT:    v_mov_b32_e32 v3, s1
553; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v2
554; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
555; VI-NEXT:    s_waitcnt vmcnt(0)
556; VI-NEXT:    v_lshrrev_b32_e32 v4, 24, v1
557; VI-NEXT:    v_lshrrev_b32_e32 v5, 24, v0
558; VI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
559; VI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
560; VI-NEXT:    v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
561; VI-NEXT:    v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
562; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
563; VI-NEXT:    s_endpgm
564;
565; CI-LABEL: lshr_v_imm_v4i16:
566; CI:       ; %bb.0:
567; CI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
568; CI-NEXT:    s_mov_b32 s7, 0xf000
569; CI-NEXT:    s_mov_b32 s6, 0
570; CI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
571; CI-NEXT:    v_mov_b32_e32 v1, 0
572; CI-NEXT:    s_waitcnt lgkmcnt(0)
573; CI-NEXT:    s_mov_b64 s[4:5], s[2:3]
574; CI-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64
575; CI-NEXT:    s_mov_b64 s[2:3], s[6:7]
576; CI-NEXT:    s_waitcnt vmcnt(0)
577; CI-NEXT:    v_lshrrev_b32_e32 v3, 8, v3
578; CI-NEXT:    v_lshrrev_b32_e32 v2, 8, v2
579; CI-NEXT:    v_and_b32_e32 v3, 0xff00ff, v3
580; CI-NEXT:    v_and_b32_e32 v2, 0xff00ff, v2
581; CI-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
582; CI-NEXT:    s_endpgm
583;
584; GFX10-LABEL: lshr_v_imm_v4i16:
585; GFX10:       ; %bb.0:
586; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
587; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
588; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
589; GFX10-NEXT:    global_load_dwordx2 v[0:1], v2, s[2:3]
590; GFX10-NEXT:    s_waitcnt vmcnt(0)
591; GFX10-NEXT:    v_pk_lshrrev_b16 v1, 8, v1 op_sel_hi:[0,1]
592; GFX10-NEXT:    v_pk_lshrrev_b16 v0, 8, v0 op_sel_hi:[0,1]
593; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
594; GFX10-NEXT:    s_endpgm
595  %tid = call i32 @llvm.amdgcn.workitem.id.x()
596  %tid.ext = sext i32 %tid to i64
597  %in.gep = getelementptr inbounds <4 x i16>, <4 x i16> addrspace(1)* %in, i64 %tid.ext
598  %out.gep = getelementptr inbounds <4 x i16>, <4 x i16> addrspace(1)* %out, i64 %tid.ext
599  %vgpr = load <4 x i16>, <4 x i16> addrspace(1)* %in.gep
600  %result = lshr <4 x i16> %vgpr, <i16 8, i16 8, i16 8, i16 8>
601  store <4 x i16> %result, <4 x i16> addrspace(1)* %out.gep
602  ret void
603}
604
605declare i32 @llvm.amdgcn.workitem.id.x() #1
606
607attributes #0 = { nounwind }
608attributes #1 = { nounwind readnone }
609