1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
3; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI,CIVI %s
4; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI,CIVI %s
5; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CIVI,VI %s
6; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CIVI,CI %s
7
8define amdgpu_kernel void @s_lshr_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> %lhs, <2 x i16> %rhs) #0 {
9; GFX9-LABEL: s_lshr_v2i16:
10; GFX9:       ; %bb.0:
11; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
12; GFX9-NEXT:    s_load_dword s4, s[0:1], 0x2c
13; GFX9-NEXT:    s_load_dword s0, s[0:1], 0x30
14; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
15; GFX9-NEXT:    v_mov_b32_e32 v0, s2
16; GFX9-NEXT:    v_mov_b32_e32 v2, s4
17; GFX9-NEXT:    v_mov_b32_e32 v1, s3
18; GFX9-NEXT:    v_pk_lshrrev_b16 v2, s0, v2
19; GFX9-NEXT:    global_store_dword v[0:1], v2, off
20; GFX9-NEXT:    s_endpgm
21;
22; VI-LABEL: s_lshr_v2i16:
23; VI:       ; %bb.0:
24; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
25; VI-NEXT:    s_load_dword s5, s[0:1], 0x2c
26; VI-NEXT:    s_load_dword s0, s[0:1], 0x30
27; VI-NEXT:    s_mov_b32 s4, 0xffff
28; VI-NEXT:    s_waitcnt lgkmcnt(0)
29; VI-NEXT:    v_mov_b32_e32 v0, s2
30; VI-NEXT:    s_lshr_b32 s1, s5, 16
31; VI-NEXT:    s_lshr_b32 s6, s0, 16
32; VI-NEXT:    s_lshr_b32 s1, s1, s6
33; VI-NEXT:    s_and_b32 s5, s5, s4
34; VI-NEXT:    s_and_b32 s0, s0, s4
35; VI-NEXT:    s_lshr_b32 s0, s5, s0
36; VI-NEXT:    s_lshl_b32 s1, s1, 16
37; VI-NEXT:    s_or_b32 s0, s0, s1
38; VI-NEXT:    v_mov_b32_e32 v1, s3
39; VI-NEXT:    v_mov_b32_e32 v2, s0
40; VI-NEXT:    flat_store_dword v[0:1], v2
41; VI-NEXT:    s_endpgm
42;
43; CI-LABEL: s_lshr_v2i16:
44; CI:       ; %bb.0:
45; CI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
46; CI-NEXT:    s_load_dword s2, s[0:1], 0xb
47; CI-NEXT:    s_load_dword s0, s[0:1], 0xc
48; CI-NEXT:    s_mov_b32 s3, 0xffff
49; CI-NEXT:    s_mov_b32 s7, 0xf000
50; CI-NEXT:    s_mov_b32 s6, -1
51; CI-NEXT:    s_waitcnt lgkmcnt(0)
52; CI-NEXT:    s_lshr_b32 s1, s2, 16
53; CI-NEXT:    s_lshr_b32 s8, s0, 16
54; CI-NEXT:    s_lshr_b32 s1, s1, s8
55; CI-NEXT:    s_and_b32 s2, s2, s3
56; CI-NEXT:    s_and_b32 s0, s0, s3
57; CI-NEXT:    s_lshr_b32 s0, s2, s0
58; CI-NEXT:    s_lshl_b32 s1, s1, 16
59; CI-NEXT:    s_or_b32 s0, s0, s1
60; CI-NEXT:    v_mov_b32_e32 v0, s0
61; CI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
62; CI-NEXT:    s_endpgm
63  %result = lshr <2 x i16> %lhs, %rhs
64  store <2 x i16> %result, <2 x i16> addrspace(1)* %out
65  ret void
66}
67
68define amdgpu_kernel void @v_lshr_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) #0 {
69; GFX9-LABEL: v_lshr_v2i16:
70; GFX9:       ; %bb.0:
71; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
72; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
73; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
74; GFX9-NEXT:    v_mov_b32_e32 v1, s3
75; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v2
76; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
77; GFX9-NEXT:    global_load_dword v3, v[0:1], off
78; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:4
79; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v2
80; GFX9-NEXT:    v_mov_b32_e32 v1, s1
81; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
82; GFX9-NEXT:    s_waitcnt vmcnt(0)
83; GFX9-NEXT:    v_pk_lshrrev_b16 v2, v4, v3
84; GFX9-NEXT:    global_store_dword v[0:1], v2, off
85; GFX9-NEXT:    s_endpgm
86;
87; VI-LABEL: v_lshr_v2i16:
88; VI:       ; %bb.0:
89; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
90; VI-NEXT:    v_lshlrev_b32_e32 v4, 2, v0
91; VI-NEXT:    s_waitcnt lgkmcnt(0)
92; VI-NEXT:    v_mov_b32_e32 v1, s3
93; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v4
94; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
95; VI-NEXT:    v_add_u32_e32 v2, vcc, 4, v0
96; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v1, vcc
97; VI-NEXT:    flat_load_dword v5, v[0:1]
98; VI-NEXT:    flat_load_dword v2, v[2:3]
99; VI-NEXT:    v_mov_b32_e32 v1, s1
100; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v4
101; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
102; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
103; VI-NEXT:    v_lshrrev_b16_e32 v3, v2, v5
104; VI-NEXT:    v_lshrrev_b16_sdwa v2, v2, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
105; VI-NEXT:    v_or_b32_e32 v2, v3, v2
106; VI-NEXT:    flat_store_dword v[0:1], v2
107; VI-NEXT:    s_endpgm
108;
109; CI-LABEL: v_lshr_v2i16:
110; CI:       ; %bb.0:
111; CI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
112; CI-NEXT:    s_mov_b32 s7, 0xf000
113; CI-NEXT:    s_mov_b32 s6, 0
114; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
115; CI-NEXT:    v_mov_b32_e32 v1, 0
116; CI-NEXT:    s_waitcnt lgkmcnt(0)
117; CI-NEXT:    s_mov_b64 s[4:5], s[2:3]
118; CI-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
119; CI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4
120; CI-NEXT:    s_mov_b32 s8, 0xffff
121; CI-NEXT:    s_mov_b64 s[2:3], s[6:7]
122; CI-NEXT:    s_waitcnt vmcnt(1)
123; CI-NEXT:    v_lshrrev_b32_e32 v4, 16, v2
124; CI-NEXT:    s_waitcnt vmcnt(0)
125; CI-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
126; CI-NEXT:    v_and_b32_e32 v2, s8, v2
127; CI-NEXT:    v_and_b32_e32 v3, s8, v3
128; CI-NEXT:    v_lshr_b32_e32 v2, v2, v3
129; CI-NEXT:    v_lshr_b32_e32 v3, v4, v5
130; CI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
131; CI-NEXT:    v_or_b32_e32 v2, v2, v3
132; CI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
133; CI-NEXT:    s_endpgm
134  %tid = call i32 @llvm.amdgcn.workitem.id.x()
135  %tid.ext = sext i32 %tid to i64
136  %in.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
137  %out.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %out, i64 %tid.ext
138  %b_ptr = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %in.gep, i32 1
139  %a = load <2 x i16>, <2 x i16> addrspace(1)* %in.gep
140  %b = load <2 x i16>, <2 x i16> addrspace(1)* %b_ptr
141  %result = lshr <2 x i16> %a, %b
142  store <2 x i16> %result, <2 x i16> addrspace(1)* %out.gep
143  ret void
144}
145
146define amdgpu_kernel void @lshr_v_s_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in, <2 x i16> %sgpr) #0 {
147; GFX9-LABEL: lshr_v_s_v2i16:
148; GFX9:       ; %bb.0:
149; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
150; GFX9-NEXT:    s_load_dword s0, s[0:1], 0x34
151; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
152; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
153; GFX9-NEXT:    v_mov_b32_e32 v1, s7
154; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s6, v2
155; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
156; GFX9-NEXT:    global_load_dword v3, v[0:1], off
157; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s4, v2
158; GFX9-NEXT:    v_mov_b32_e32 v1, s5
159; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
160; GFX9-NEXT:    s_waitcnt vmcnt(0)
161; GFX9-NEXT:    v_pk_lshrrev_b16 v2, s0, v3
162; GFX9-NEXT:    global_store_dword v[0:1], v2, off
163; GFX9-NEXT:    s_endpgm
164;
165; VI-LABEL: lshr_v_s_v2i16:
166; VI:       ; %bb.0:
167; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
168; VI-NEXT:    s_load_dword s0, s[0:1], 0x34
169; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
170; VI-NEXT:    s_waitcnt lgkmcnt(0)
171; VI-NEXT:    v_mov_b32_e32 v1, s7
172; VI-NEXT:    v_add_u32_e32 v0, vcc, s6, v2
173; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
174; VI-NEXT:    flat_load_dword v3, v[0:1]
175; VI-NEXT:    s_lshr_b32 s1, s0, 16
176; VI-NEXT:    v_mov_b32_e32 v4, s1
177; VI-NEXT:    v_add_u32_e32 v0, vcc, s4, v2
178; VI-NEXT:    v_mov_b32_e32 v1, s5
179; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
180; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
181; VI-NEXT:    v_lshrrev_b16_e32 v2, s0, v3
182; VI-NEXT:    v_lshrrev_b16_sdwa v3, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
183; VI-NEXT:    v_or_b32_e32 v2, v2, v3
184; VI-NEXT:    flat_store_dword v[0:1], v2
185; VI-NEXT:    s_endpgm
186;
187; CI-LABEL: lshr_v_s_v2i16:
188; CI:       ; %bb.0:
189; CI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
190; CI-NEXT:    s_load_dword s8, s[0:1], 0xd
191; CI-NEXT:    s_mov_b32 s3, 0xf000
192; CI-NEXT:    s_mov_b32 s2, 0
193; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
194; CI-NEXT:    s_waitcnt lgkmcnt(0)
195; CI-NEXT:    s_mov_b64 s[0:1], s[6:7]
196; CI-NEXT:    v_mov_b32_e32 v1, 0
197; CI-NEXT:    buffer_load_dword v2, v[0:1], s[0:3], 0 addr64
198; CI-NEXT:    s_lshr_b32 s9, s8, 16
199; CI-NEXT:    s_mov_b32 s10, 0xffff
200; CI-NEXT:    s_and_b32 s8, s8, s10
201; CI-NEXT:    s_mov_b64 s[6:7], s[2:3]
202; CI-NEXT:    s_waitcnt vmcnt(0)
203; CI-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
204; CI-NEXT:    v_and_b32_e32 v2, s10, v2
205; CI-NEXT:    v_lshrrev_b32_e32 v3, s9, v3
206; CI-NEXT:    v_lshrrev_b32_e32 v2, s8, v2
207; CI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
208; CI-NEXT:    v_or_b32_e32 v2, v2, v3
209; CI-NEXT:    buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
210; CI-NEXT:    s_endpgm
211  %tid = call i32 @llvm.amdgcn.workitem.id.x()
212  %tid.ext = sext i32 %tid to i64
213  %in.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
214  %out.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %out, i64 %tid.ext
215  %vgpr = load <2 x i16>, <2 x i16> addrspace(1)* %in.gep
216  %result = lshr <2 x i16> %vgpr, %sgpr
217  store <2 x i16> %result, <2 x i16> addrspace(1)* %out.gep
218  ret void
219}
220
221define amdgpu_kernel void @lshr_s_v_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in, <2 x i16> %sgpr) #0 {
222; GFX9-LABEL: lshr_s_v_v2i16:
223; GFX9:       ; %bb.0:
224; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
225; GFX9-NEXT:    s_load_dword s0, s[0:1], 0x34
226; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
227; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
228; GFX9-NEXT:    v_mov_b32_e32 v1, s7
229; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s6, v2
230; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
231; GFX9-NEXT:    global_load_dword v3, v[0:1], off
232; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s4, v2
233; GFX9-NEXT:    v_mov_b32_e32 v1, s5
234; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
235; GFX9-NEXT:    s_waitcnt vmcnt(0)
236; GFX9-NEXT:    v_pk_lshrrev_b16 v2, v3, s0
237; GFX9-NEXT:    global_store_dword v[0:1], v2, off
238; GFX9-NEXT:    s_endpgm
239;
240; VI-LABEL: lshr_s_v_v2i16:
241; VI:       ; %bb.0:
242; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
243; VI-NEXT:    s_load_dword s0, s[0:1], 0x34
244; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
245; VI-NEXT:    s_waitcnt lgkmcnt(0)
246; VI-NEXT:    v_mov_b32_e32 v1, s7
247; VI-NEXT:    v_add_u32_e32 v0, vcc, s6, v2
248; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
249; VI-NEXT:    flat_load_dword v3, v[0:1]
250; VI-NEXT:    s_lshr_b32 s1, s0, 16
251; VI-NEXT:    v_mov_b32_e32 v4, s1
252; VI-NEXT:    v_add_u32_e32 v0, vcc, s4, v2
253; VI-NEXT:    v_mov_b32_e32 v1, s5
254; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
255; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
256; VI-NEXT:    v_lshrrev_b16_e64 v2, v3, s0
257; VI-NEXT:    v_lshrrev_b16_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
258; VI-NEXT:    v_or_b32_e32 v2, v2, v3
259; VI-NEXT:    flat_store_dword v[0:1], v2
260; VI-NEXT:    s_endpgm
261;
262; CI-LABEL: lshr_s_v_v2i16:
263; CI:       ; %bb.0:
264; CI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
265; CI-NEXT:    s_load_dword s8, s[0:1], 0xd
266; CI-NEXT:    s_mov_b32 s3, 0xf000
267; CI-NEXT:    s_mov_b32 s2, 0
268; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
269; CI-NEXT:    s_waitcnt lgkmcnt(0)
270; CI-NEXT:    s_mov_b64 s[0:1], s[6:7]
271; CI-NEXT:    v_mov_b32_e32 v1, 0
272; CI-NEXT:    buffer_load_dword v2, v[0:1], s[0:3], 0 addr64
273; CI-NEXT:    s_lshr_b32 s9, s8, 16
274; CI-NEXT:    s_mov_b32 s10, 0xffff
275; CI-NEXT:    s_and_b32 s8, s8, s10
276; CI-NEXT:    s_mov_b64 s[6:7], s[2:3]
277; CI-NEXT:    s_waitcnt vmcnt(0)
278; CI-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
279; CI-NEXT:    v_and_b32_e32 v2, s10, v2
280; CI-NEXT:    v_lshr_b32_e32 v3, s9, v3
281; CI-NEXT:    v_lshr_b32_e32 v2, s8, v2
282; CI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
283; CI-NEXT:    v_or_b32_e32 v2, v2, v3
284; CI-NEXT:    buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
285; CI-NEXT:    s_endpgm
286  %tid = call i32 @llvm.amdgcn.workitem.id.x()
287  %tid.ext = sext i32 %tid to i64
288  %in.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
289  %out.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %out, i64 %tid.ext
290  %vgpr = load <2 x i16>, <2 x i16> addrspace(1)* %in.gep
291  %result = lshr <2 x i16> %sgpr, %vgpr
292  store <2 x i16> %result, <2 x i16> addrspace(1)* %out.gep
293  ret void
294}
295
296define amdgpu_kernel void @lshr_imm_v_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) #0 {
297; GFX9-LABEL: lshr_imm_v_v2i16:
298; GFX9:       ; %bb.0:
299; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
300; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
301; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
302; GFX9-NEXT:    v_mov_b32_e32 v1, s3
303; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v2
304; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
305; GFX9-NEXT:    global_load_dword v3, v[0:1], off
306; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v2
307; GFX9-NEXT:    v_mov_b32_e32 v1, s1
308; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
309; GFX9-NEXT:    s_waitcnt vmcnt(0)
310; GFX9-NEXT:    v_pk_lshrrev_b16 v2, v3, 8 op_sel_hi:[1,0]
311; GFX9-NEXT:    global_store_dword v[0:1], v2, off
312; GFX9-NEXT:    s_endpgm
313;
314; VI-LABEL: lshr_imm_v_v2i16:
315; VI:       ; %bb.0:
316; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
317; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
318; VI-NEXT:    v_mov_b32_e32 v3, 8
319; VI-NEXT:    s_waitcnt lgkmcnt(0)
320; VI-NEXT:    v_mov_b32_e32 v1, s3
321; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
322; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
323; VI-NEXT:    flat_load_dword v4, v[0:1]
324; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
325; VI-NEXT:    v_mov_b32_e32 v1, s1
326; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
327; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
328; VI-NEXT:    v_lshrrev_b16_e64 v2, v4, 8
329; VI-NEXT:    v_lshrrev_b16_sdwa v3, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
330; VI-NEXT:    v_or_b32_e32 v2, v2, v3
331; VI-NEXT:    flat_store_dword v[0:1], v2
332; VI-NEXT:    s_endpgm
333;
334; CI-LABEL: lshr_imm_v_v2i16:
335; CI:       ; %bb.0:
336; CI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
337; CI-NEXT:    s_mov_b32 s7, 0xf000
338; CI-NEXT:    s_mov_b32 s6, 0
339; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
340; CI-NEXT:    v_mov_b32_e32 v1, 0
341; CI-NEXT:    s_waitcnt lgkmcnt(0)
342; CI-NEXT:    s_mov_b64 s[4:5], s[2:3]
343; CI-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
344; CI-NEXT:    s_mov_b64 s[2:3], s[6:7]
345; CI-NEXT:    s_waitcnt vmcnt(0)
346; CI-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
347; CI-NEXT:    v_and_b32_e32 v2, 0xffff, v2
348; CI-NEXT:    v_lshr_b32_e32 v3, 8, v3
349; CI-NEXT:    v_lshr_b32_e32 v2, 8, v2
350; CI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
351; CI-NEXT:    v_or_b32_e32 v2, v2, v3
352; CI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
353; CI-NEXT:    s_endpgm
354  %tid = call i32 @llvm.amdgcn.workitem.id.x()
355  %tid.ext = sext i32 %tid to i64
356  %in.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
357  %out.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %out, i64 %tid.ext
358  %vgpr = load <2 x i16>, <2 x i16> addrspace(1)* %in.gep
359  %result = lshr <2 x i16> <i16 8, i16 8>, %vgpr
360  store <2 x i16> %result, <2 x i16> addrspace(1)* %out.gep
361  ret void
362}
363
364define amdgpu_kernel void @lshr_v_imm_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) #0 {
365; GFX9-LABEL: lshr_v_imm_v2i16:
366; GFX9:       ; %bb.0:
367; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
368; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
369; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
370; GFX9-NEXT:    v_mov_b32_e32 v1, s3
371; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v2
372; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
373; GFX9-NEXT:    global_load_dword v3, v[0:1], off
374; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v2
375; GFX9-NEXT:    v_mov_b32_e32 v1, s1
376; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
377; GFX9-NEXT:    s_waitcnt vmcnt(0)
378; GFX9-NEXT:    v_pk_lshrrev_b16 v2, 8, v3 op_sel_hi:[0,1]
379; GFX9-NEXT:    global_store_dword v[0:1], v2, off
380; GFX9-NEXT:    s_endpgm
381;
382; VI-LABEL: lshr_v_imm_v2i16:
383; VI:       ; %bb.0:
384; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
385; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
386; VI-NEXT:    s_waitcnt lgkmcnt(0)
387; VI-NEXT:    v_mov_b32_e32 v1, s3
388; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
389; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
390; VI-NEXT:    flat_load_dword v3, v[0:1]
391; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
392; VI-NEXT:    v_mov_b32_e32 v1, s1
393; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
394; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
395; VI-NEXT:    v_lshrrev_b32_e32 v2, 24, v3
396; VI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
397; VI-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
398; VI-NEXT:    flat_store_dword v[0:1], v2
399; VI-NEXT:    s_endpgm
400;
401; CI-LABEL: lshr_v_imm_v2i16:
402; CI:       ; %bb.0:
403; CI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
404; CI-NEXT:    s_mov_b32 s7, 0xf000
405; CI-NEXT:    s_mov_b32 s6, 0
406; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
407; CI-NEXT:    v_mov_b32_e32 v1, 0
408; CI-NEXT:    s_waitcnt lgkmcnt(0)
409; CI-NEXT:    s_mov_b64 s[4:5], s[2:3]
410; CI-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
411; CI-NEXT:    s_mov_b64 s[2:3], s[6:7]
412; CI-NEXT:    s_waitcnt vmcnt(0)
413; CI-NEXT:    v_lshrrev_b32_e32 v2, 8, v2
414; CI-NEXT:    v_and_b32_e32 v2, 0xff00ff, v2
415; CI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
416; CI-NEXT:    s_endpgm
417  %tid = call i32 @llvm.amdgcn.workitem.id.x()
418  %tid.ext = sext i32 %tid to i64
419  %in.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
420  %out.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %out, i64 %tid.ext
421  %vgpr = load <2 x i16>, <2 x i16> addrspace(1)* %in.gep
422  %result = lshr <2 x i16> %vgpr, <i16 8, i16 8>
423  store <2 x i16> %result, <2 x i16> addrspace(1)* %out.gep
424  ret void
425}
426
427define amdgpu_kernel void @v_lshr_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> addrspace(1)* %in) #0 {
428; GFX9-LABEL: v_lshr_v4i16:
429; GFX9:       ; %bb.0:
430; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
431; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
432; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
433; GFX9-NEXT:    v_mov_b32_e32 v1, s3
434; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v4
435; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
436; GFX9-NEXT:    global_load_dwordx2 v[2:3], v[0:1], off
437; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off offset:8
438; GFX9-NEXT:    v_mov_b32_e32 v5, s1
439; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, s0, v4
440; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
441; GFX9-NEXT:    s_waitcnt vmcnt(0)
442; GFX9-NEXT:    v_pk_lshrrev_b16 v1, v1, v3
443; GFX9-NEXT:    v_pk_lshrrev_b16 v0, v0, v2
444; GFX9-NEXT:    global_store_dwordx2 v[4:5], v[0:1], off
445; GFX9-NEXT:    s_endpgm
446;
447; VI-LABEL: v_lshr_v4i16:
448; VI:       ; %bb.0:
449; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
450; VI-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
451; VI-NEXT:    s_waitcnt lgkmcnt(0)
452; VI-NEXT:    v_mov_b32_e32 v1, s3
453; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v4
454; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
455; VI-NEXT:    v_add_u32_e32 v2, vcc, 8, v0
456; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v1, vcc
457; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
458; VI-NEXT:    flat_load_dwordx2 v[2:3], v[2:3]
459; VI-NEXT:    v_mov_b32_e32 v5, s1
460; VI-NEXT:    v_add_u32_e32 v4, vcc, s0, v4
461; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
462; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
463; VI-NEXT:    v_lshrrev_b16_e32 v6, v3, v1
464; VI-NEXT:    v_lshrrev_b16_sdwa v1, v3, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
465; VI-NEXT:    v_lshrrev_b16_e32 v3, v2, v0
466; VI-NEXT:    v_lshrrev_b16_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
467; VI-NEXT:    v_or_b32_e32 v1, v6, v1
468; VI-NEXT:    v_or_b32_e32 v0, v3, v0
469; VI-NEXT:    flat_store_dwordx2 v[4:5], v[0:1]
470; VI-NEXT:    s_endpgm
471;
472; CI-LABEL: v_lshr_v4i16:
473; CI:       ; %bb.0:
474; CI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
475; CI-NEXT:    s_mov_b32 s7, 0xf000
476; CI-NEXT:    s_mov_b32 s6, 0
477; CI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
478; CI-NEXT:    v_mov_b32_e32 v1, 0
479; CI-NEXT:    s_waitcnt lgkmcnt(0)
480; CI-NEXT:    s_mov_b64 s[4:5], s[2:3]
481; CI-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64
482; CI-NEXT:    buffer_load_dwordx2 v[4:5], v[0:1], s[4:7], 0 addr64 offset:8
483; CI-NEXT:    s_mov_b32 s8, 0xffff
484; CI-NEXT:    s_mov_b64 s[2:3], s[6:7]
485; CI-NEXT:    s_waitcnt vmcnt(1)
486; CI-NEXT:    v_lshrrev_b32_e32 v6, 16, v2
487; CI-NEXT:    v_lshrrev_b32_e32 v7, 16, v3
488; CI-NEXT:    s_waitcnt vmcnt(0)
489; CI-NEXT:    v_lshrrev_b32_e32 v8, 16, v4
490; CI-NEXT:    v_lshrrev_b32_e32 v9, 16, v5
491; CI-NEXT:    v_and_b32_e32 v2, s8, v2
492; CI-NEXT:    v_and_b32_e32 v4, s8, v4
493; CI-NEXT:    v_and_b32_e32 v3, s8, v3
494; CI-NEXT:    v_and_b32_e32 v5, s8, v5
495; CI-NEXT:    v_lshr_b32_e32 v3, v3, v5
496; CI-NEXT:    v_lshr_b32_e32 v5, v7, v9
497; CI-NEXT:    v_lshr_b32_e32 v2, v2, v4
498; CI-NEXT:    v_lshr_b32_e32 v4, v6, v8
499; CI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
500; CI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
501; CI-NEXT:    v_or_b32_e32 v3, v3, v5
502; CI-NEXT:    v_or_b32_e32 v2, v2, v4
503; CI-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
504; CI-NEXT:    s_endpgm
505  %tid = call i32 @llvm.amdgcn.workitem.id.x()
506  %tid.ext = sext i32 %tid to i64
507  %in.gep = getelementptr inbounds <4 x i16>, <4 x i16> addrspace(1)* %in, i64 %tid.ext
508  %out.gep = getelementptr inbounds <4 x i16>, <4 x i16> addrspace(1)* %out, i64 %tid.ext
509  %b_ptr = getelementptr <4 x i16>, <4 x i16> addrspace(1)* %in.gep, i32 1
510  %a = load <4 x i16>, <4 x i16> addrspace(1)* %in.gep
511  %b = load <4 x i16>, <4 x i16> addrspace(1)* %b_ptr
512  %result = lshr <4 x i16> %a, %b
513  store <4 x i16> %result, <4 x i16> addrspace(1)* %out.gep
514  ret void
515}
516
517define amdgpu_kernel void @lshr_v_imm_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> addrspace(1)* %in) #0 {
518; GFX9-LABEL: lshr_v_imm_v4i16:
519; GFX9:       ; %bb.0:
520; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
521; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
522; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
523; GFX9-NEXT:    v_mov_b32_e32 v1, s3
524; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s2, v2
525; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
526; GFX9-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
527; GFX9-NEXT:    v_mov_b32_e32 v3, s1
528; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s0, v2
529; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
530; GFX9-NEXT:    s_waitcnt vmcnt(0)
531; GFX9-NEXT:    v_pk_lshrrev_b16 v1, 8, v1 op_sel_hi:[0,1]
532; GFX9-NEXT:    v_pk_lshrrev_b16 v0, 8, v0 op_sel_hi:[0,1]
533; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
534; GFX9-NEXT:    s_endpgm
535;
536; VI-LABEL: lshr_v_imm_v4i16:
537; VI:       ; %bb.0:
538; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
539; VI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
540; VI-NEXT:    s_waitcnt lgkmcnt(0)
541; VI-NEXT:    v_mov_b32_e32 v1, s3
542; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
543; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
544; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
545; VI-NEXT:    v_mov_b32_e32 v3, s1
546; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v2
547; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
548; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
549; VI-NEXT:    v_lshrrev_b32_e32 v4, 24, v1
550; VI-NEXT:    v_lshrrev_b32_e32 v5, 24, v0
551; VI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
552; VI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
553; VI-NEXT:    v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
554; VI-NEXT:    v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
555; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
556; VI-NEXT:    s_endpgm
557;
558; CI-LABEL: lshr_v_imm_v4i16:
559; CI:       ; %bb.0:
560; CI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
561; CI-NEXT:    s_mov_b32 s7, 0xf000
562; CI-NEXT:    s_mov_b32 s6, 0
563; CI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
564; CI-NEXT:    v_mov_b32_e32 v1, 0
565; CI-NEXT:    s_waitcnt lgkmcnt(0)
566; CI-NEXT:    s_mov_b64 s[4:5], s[2:3]
567; CI-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64
568; CI-NEXT:    s_mov_b32 s8, 0xff00ff
569; CI-NEXT:    s_mov_b64 s[2:3], s[6:7]
570; CI-NEXT:    s_waitcnt vmcnt(0)
571; CI-NEXT:    v_lshrrev_b32_e32 v3, 8, v3
572; CI-NEXT:    v_lshrrev_b32_e32 v2, 8, v2
573; CI-NEXT:    v_and_b32_e32 v3, s8, v3
574; CI-NEXT:    v_and_b32_e32 v2, s8, v2
575; CI-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
576; CI-NEXT:    s_endpgm
577  %tid = call i32 @llvm.amdgcn.workitem.id.x()
578  %tid.ext = sext i32 %tid to i64
579  %in.gep = getelementptr inbounds <4 x i16>, <4 x i16> addrspace(1)* %in, i64 %tid.ext
580  %out.gep = getelementptr inbounds <4 x i16>, <4 x i16> addrspace(1)* %out, i64 %tid.ext
581  %vgpr = load <4 x i16>, <4 x i16> addrspace(1)* %in.gep
582  %result = lshr <4 x i16> %vgpr, <i16 8, i16 8, i16 8, i16 8>
583  store <4 x i16> %result, <4 x i16> addrspace(1)* %out.gep
584  ret void
585}
586
587declare i32 @llvm.amdgcn.workitem.id.x() #1
588
589attributes #0 = { nounwind }
590attributes #1 = { nounwind readnone }
591