1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=SI %s
3; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=VI %s
4; RUN: llc -march=r600 -mcpu=cypress -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=EG %s
5
6declare i16 @llvm.ctpop.i16(i16) nounwind readnone
7declare <2 x i16> @llvm.ctpop.v2i16(<2 x i16>) nounwind readnone
8declare <4 x i16> @llvm.ctpop.v4i16(<4 x i16>) nounwind readnone
9declare <8 x i16> @llvm.ctpop.v8i16(<8 x i16>) nounwind readnone
10declare <16 x i16> @llvm.ctpop.v16i16(<16 x i16>) nounwind readnone
11
12declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
13
14define amdgpu_kernel void @s_ctpop_i16(i16 addrspace(1)* noalias %out, i16 %val) nounwind {
15; SI-LABEL: s_ctpop_i16:
16; SI:       ; %bb.0:
17; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
18; SI-NEXT:    s_load_dword s0, s[0:1], 0xb
19; SI-NEXT:    s_mov_b32 s7, 0xf000
20; SI-NEXT:    s_mov_b32 s6, -1
21; SI-NEXT:    s_waitcnt lgkmcnt(0)
22; SI-NEXT:    s_and_b32 s0, s0, 0xffff
23; SI-NEXT:    s_bcnt1_i32_b32 s0, s0
24; SI-NEXT:    v_mov_b32_e32 v0, s0
25; SI-NEXT:    buffer_store_short v0, off, s[4:7], 0
26; SI-NEXT:    s_endpgm
27;
28; VI-LABEL: s_ctpop_i16:
29; VI:       ; %bb.0:
30; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
31; VI-NEXT:    s_load_dword s0, s[0:1], 0x2c
32; VI-NEXT:    s_mov_b32 s7, 0xf000
33; VI-NEXT:    s_mov_b32 s6, -1
34; VI-NEXT:    s_waitcnt lgkmcnt(0)
35; VI-NEXT:    s_and_b32 s0, s0, 0xffff
36; VI-NEXT:    s_bcnt1_i32_b32 s0, s0
37; VI-NEXT:    v_mov_b32_e32 v0, s0
38; VI-NEXT:    buffer_store_short v0, off, s[4:7], 0
39; VI-NEXT:    s_endpgm
40;
41; EG-LABEL: s_ctpop_i16:
42; EG:       ; %bb.0:
43; EG-NEXT:    ALU 0, @8, KC0[], KC1[]
44; EG-NEXT:    TEX 0 @6
45; EG-NEXT:    ALU 11, @9, KC0[CB0:0-32], KC1[]
46; EG-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
47; EG-NEXT:    CF_END
48; EG-NEXT:    PAD
49; EG-NEXT:    Fetch clause starting at 6:
50; EG-NEXT:     VTX_READ_16 T0.X, T0.X, 40, #3
51; EG-NEXT:    ALU clause starting at 8:
52; EG-NEXT:     MOV * T0.X, 0.0,
53; EG-NEXT:    ALU clause starting at 9:
54; EG-NEXT:     AND_INT * T0.W, KC0[2].Y, literal.x,
55; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
56; EG-NEXT:     BCNT_INT T1.W, T0.X,
57; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
58; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
59; EG-NEXT:     LSHL T0.X, PV.W, PS,
60; EG-NEXT:     LSHL * T0.W, literal.x, PS,
61; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
62; EG-NEXT:     MOV T0.Y, 0.0,
63; EG-NEXT:     MOV * T0.Z, 0.0,
64; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
65; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
66  %ctpop = call i16 @llvm.ctpop.i16(i16 %val) nounwind readnone
67  store i16 %ctpop, i16 addrspace(1)* %out, align 4
68  ret void
69}
70
71; XXX - Why 0 in register?
72define amdgpu_kernel void @v_ctpop_i16(i16 addrspace(1)* noalias %out, i16 addrspace(1)* noalias %in) nounwind {
73; SI-LABEL: v_ctpop_i16:
74; SI:       ; %bb.0:
75; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
76; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
77; SI-NEXT:    s_mov_b32 s7, 0xf000
78; SI-NEXT:    s_mov_b32 s2, 0
79; SI-NEXT:    s_mov_b32 s3, s7
80; SI-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
81; SI-NEXT:    v_mov_b32_e32 v1, 0
82; SI-NEXT:    s_waitcnt lgkmcnt(0)
83; SI-NEXT:    buffer_load_ushort v0, v[0:1], s[0:3], 0 addr64
84; SI-NEXT:    s_mov_b32 s6, -1
85; SI-NEXT:    s_waitcnt vmcnt(0)
86; SI-NEXT:    v_bcnt_u32_b32_e64 v0, v0, 0
87; SI-NEXT:    buffer_store_short v0, off, s[4:7], 0
88; SI-NEXT:    s_endpgm
89;
90; VI-LABEL: v_ctpop_i16:
91; VI:       ; %bb.0:
92; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
93; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
94; VI-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
95; VI-NEXT:    s_mov_b32 s7, 0xf000
96; VI-NEXT:    s_mov_b32 s6, -1
97; VI-NEXT:    s_waitcnt lgkmcnt(0)
98; VI-NEXT:    v_mov_b32_e32 v1, s1
99; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
100; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
101; VI-NEXT:    flat_load_ushort v0, v[0:1]
102; VI-NEXT:    s_waitcnt vmcnt(0)
103; VI-NEXT:    v_bcnt_u32_b32 v0, v0, 0
104; VI-NEXT:    buffer_store_short v0, off, s[4:7], 0
105; VI-NEXT:    s_endpgm
106;
107; EG-LABEL: v_ctpop_i16:
108; EG:       ; %bb.0:
109; EG-NEXT:    ALU 1, @8, KC0[CB0:0-32], KC1[]
110; EG-NEXT:    TEX 0 @6
111; EG-NEXT:    ALU 11, @10, KC0[CB0:0-32], KC1[]
112; EG-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
113; EG-NEXT:    CF_END
114; EG-NEXT:    PAD
115; EG-NEXT:    Fetch clause starting at 6:
116; EG-NEXT:     VTX_READ_16 T0.X, T0.X, 0, #1
117; EG-NEXT:    ALU clause starting at 8:
118; EG-NEXT:     LSHL * T0.W, T0.X, 1,
119; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, PV.W,
120; EG-NEXT:    ALU clause starting at 10:
121; EG-NEXT:     AND_INT * T0.W, KC0[2].Y, literal.x,
122; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
123; EG-NEXT:     BCNT_INT T1.W, T0.X,
124; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
125; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
126; EG-NEXT:     LSHL T0.X, PV.W, PS,
127; EG-NEXT:     LSHL * T0.W, literal.x, PS,
128; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
129; EG-NEXT:     MOV T0.Y, 0.0,
130; EG-NEXT:     MOV * T0.Z, 0.0,
131; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
132; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
133  %tid = call i32 @llvm.amdgcn.workitem.id.x()
134  %in.gep = getelementptr i16, i16 addrspace(1)* %in, i32 %tid
135  %val = load i16, i16 addrspace(1)* %in.gep, align 4
136  %ctpop = call i16 @llvm.ctpop.i16(i16 %val) nounwind readnone
137  store i16 %ctpop, i16 addrspace(1)* %out, align 4
138  ret void
139}
140
141define amdgpu_kernel void @v_ctpop_add_chain_i16(i16 addrspace(1)* noalias %out, i16 addrspace(1)* noalias %in0, i16 addrspace(1)* noalias %in1) nounwind {
142; SI-LABEL: v_ctpop_add_chain_i16:
143; SI:       ; %bb.0:
144; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
145; SI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xb
146; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
147; SI-NEXT:    s_mov_b32 s7, 0xf000
148; SI-NEXT:    s_mov_b32 s10, 0
149; SI-NEXT:    s_mov_b32 s11, s7
150; SI-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
151; SI-NEXT:    v_mov_b32_e32 v1, 0
152; SI-NEXT:    s_mov_b64 s[2:3], s[10:11]
153; SI-NEXT:    s_waitcnt lgkmcnt(0)
154; SI-NEXT:    buffer_load_ushort v2, v[0:1], s[8:11], 0 addr64 glc
155; SI-NEXT:    s_waitcnt vmcnt(0)
156; SI-NEXT:    buffer_load_ushort v0, v[0:1], s[0:3], 0 addr64 glc
157; SI-NEXT:    s_waitcnt vmcnt(0)
158; SI-NEXT:    s_mov_b32 s6, -1
159; SI-NEXT:    v_bcnt_u32_b32_e64 v0, v0, 0
160; SI-NEXT:    v_bcnt_u32_b32_e32 v0, v2, v0
161; SI-NEXT:    buffer_store_short v0, off, s[4:7], 0
162; SI-NEXT:    s_endpgm
163;
164; VI-LABEL: v_ctpop_add_chain_i16:
165; VI:       ; %bb.0:
166; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
167; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
168; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
169; VI-NEXT:    v_lshlrev_b32_e32 v2, 1, v0
170; VI-NEXT:    s_mov_b32 s7, 0xf000
171; VI-NEXT:    s_mov_b32 s6, -1
172; VI-NEXT:    s_waitcnt lgkmcnt(0)
173; VI-NEXT:    v_mov_b32_e32 v1, s3
174; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
175; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
176; VI-NEXT:    v_mov_b32_e32 v3, s1
177; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v2
178; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
179; VI-NEXT:    flat_load_ushort v0, v[0:1] glc
180; VI-NEXT:    s_waitcnt vmcnt(0)
181; VI-NEXT:    flat_load_ushort v1, v[2:3] glc
182; VI-NEXT:    s_waitcnt vmcnt(0)
183; VI-NEXT:    v_bcnt_u32_b32 v1, v1, 0
184; VI-NEXT:    v_bcnt_u32_b32 v0, v0, v1
185; VI-NEXT:    buffer_store_short v0, off, s[4:7], 0
186; VI-NEXT:    s_endpgm
187;
188; EG-LABEL: v_ctpop_add_chain_i16:
189; EG:       ; %bb.0:
190; EG-NEXT:    ALU 1, @12, KC0[CB0:0-32], KC1[]
191; EG-NEXT:    TEX 0 @8
192; EG-NEXT:    ALU 0, @14, KC0[CB0:0-32], KC1[]
193; EG-NEXT:    TEX 0 @10
194; EG-NEXT:    ALU 16, @15, KC0[CB0:0-32], KC1[]
195; EG-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
196; EG-NEXT:    CF_END
197; EG-NEXT:    PAD
198; EG-NEXT:    Fetch clause starting at 8:
199; EG-NEXT:     VTX_READ_16 T0.X, T0.X, 0, #1
200; EG-NEXT:    Fetch clause starting at 10:
201; EG-NEXT:     VTX_READ_16 T1.X, T1.X, 0, #1
202; EG-NEXT:    ALU clause starting at 12:
203; EG-NEXT:     LSHL * T0.W, T0.X, 1,
204; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, PV.W,
205; EG-NEXT:    ALU clause starting at 14:
206; EG-NEXT:     ADD_INT * T1.X, KC0[2].W, T0.W,
207; EG-NEXT:    ALU clause starting at 15:
208; EG-NEXT:     AND_INT T0.W, T0.X, literal.x,
209; EG-NEXT:     AND_INT * T1.W, T1.X, literal.x,
210; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
211; EG-NEXT:     BCNT_INT T0.Z, PS,
212; EG-NEXT:     BCNT_INT T0.W, PV.W,
213; EG-NEXT:     AND_INT * T1.W, KC0[2].Y, literal.x,
214; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
215; EG-NEXT:     ADD_INT T0.W, PV.W, PV.Z,
216; EG-NEXT:     LSHL * T1.W, PS, literal.x,
217; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
218; EG-NEXT:     LSHL T0.X, PV.W, PS,
219; EG-NEXT:     LSHL * T0.W, literal.x, PS,
220; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
221; EG-NEXT:     MOV T0.Y, 0.0,
222; EG-NEXT:     MOV * T0.Z, 0.0,
223; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
224; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
225  %tid = call i32 @llvm.amdgcn.workitem.id.x()
226  %in0.gep = getelementptr i16, i16 addrspace(1)* %in0, i32 %tid
227  %in1.gep = getelementptr i16, i16 addrspace(1)* %in1, i32 %tid
228  %val0 = load volatile i16, i16 addrspace(1)* %in0.gep, align 4
229  %val1 = load volatile i16, i16 addrspace(1)* %in1.gep, align 4
230  %ctpop0 = call i16 @llvm.ctpop.i16(i16 %val0) nounwind readnone
231  %ctpop1 = call i16 @llvm.ctpop.i16(i16 %val1) nounwind readnone
232  %add = add i16 %ctpop0, %ctpop1
233  store i16 %add, i16 addrspace(1)* %out, align 4
234  ret void
235}
236
237define amdgpu_kernel void @v_ctpop_add_sgpr_i16(i16 addrspace(1)* noalias %out, i16 addrspace(1)* noalias %in, i16 %sval) nounwind {
238; SI-LABEL: v_ctpop_add_sgpr_i16:
239; SI:       ; %bb.0:
240; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
241; SI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xb
242; SI-NEXT:    s_load_dword s0, s[0:1], 0xd
243; SI-NEXT:    s_mov_b32 s7, 0xf000
244; SI-NEXT:    s_mov_b32 s10, 0
245; SI-NEXT:    s_mov_b32 s11, s7
246; SI-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
247; SI-NEXT:    v_mov_b32_e32 v1, 0
248; SI-NEXT:    s_waitcnt lgkmcnt(0)
249; SI-NEXT:    buffer_load_ushort v0, v[0:1], s[8:11], 0 addr64
250; SI-NEXT:    s_mov_b32 s6, -1
251; SI-NEXT:    s_waitcnt vmcnt(0)
252; SI-NEXT:    v_bcnt_u32_b32_e64 v0, v0, s0
253; SI-NEXT:    buffer_store_short v0, off, s[4:7], 0
254; SI-NEXT:    s_endpgm
255;
256; VI-LABEL: v_ctpop_add_sgpr_i16:
257; VI:       ; %bb.0:
258; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
259; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
260; VI-NEXT:    s_load_dword s0, s[0:1], 0x34
261; VI-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
262; VI-NEXT:    s_mov_b32 s7, 0xf000
263; VI-NEXT:    s_mov_b32 s6, -1
264; VI-NEXT:    s_waitcnt lgkmcnt(0)
265; VI-NEXT:    v_mov_b32_e32 v1, s3
266; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v0
267; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
268; VI-NEXT:    flat_load_ushort v0, v[0:1]
269; VI-NEXT:    s_waitcnt vmcnt(0)
270; VI-NEXT:    v_bcnt_u32_b32 v0, v0, s0
271; VI-NEXT:    buffer_store_short v0, off, s[4:7], 0
272; VI-NEXT:    s_endpgm
273;
274; EG-LABEL: v_ctpop_add_sgpr_i16:
275; EG:       ; %bb.0:
276; EG-NEXT:    ALU 1, @12, KC0[CB0:0-32], KC1[]
277; EG-NEXT:    TEX 0 @8
278; EG-NEXT:    ALU 0, @14, KC0[], KC1[]
279; EG-NEXT:    TEX 0 @10
280; EG-NEXT:    ALU 13, @15, KC0[CB0:0-32], KC1[]
281; EG-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
282; EG-NEXT:    CF_END
283; EG-NEXT:    PAD
284; EG-NEXT:    Fetch clause starting at 8:
285; EG-NEXT:     VTX_READ_16 T0.X, T0.X, 0, #1
286; EG-NEXT:    Fetch clause starting at 10:
287; EG-NEXT:     VTX_READ_16 T1.X, T1.X, 44, #3
288; EG-NEXT:    ALU clause starting at 12:
289; EG-NEXT:     LSHL * T0.W, T0.X, 1,
290; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, PV.W,
291; EG-NEXT:    ALU clause starting at 14:
292; EG-NEXT:     MOV * T1.X, 0.0,
293; EG-NEXT:    ALU clause starting at 15:
294; EG-NEXT:     BCNT_INT T0.W, T0.X,
295; EG-NEXT:     AND_INT * T1.W, KC0[2].Y, literal.x,
296; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
297; EG-NEXT:     ADD_INT * T0.W, PV.W, T1.X,
298; EG-NEXT:     AND_INT T0.W, PV.W, literal.x,
299; EG-NEXT:     LSHL * T1.W, T1.W, literal.y,
300; EG-NEXT:    65535(9.183409e-41), 3(4.203895e-45)
301; EG-NEXT:     LSHL T0.X, PV.W, PS,
302; EG-NEXT:     LSHL * T0.W, literal.x, PS,
303; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
304; EG-NEXT:     MOV T0.Y, 0.0,
305; EG-NEXT:     MOV * T0.Z, 0.0,
306; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
307; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
308  %tid = call i32 @llvm.amdgcn.workitem.id.x()
309  %in.gep = getelementptr i16, i16 addrspace(1)* %in, i32 %tid
310  %val = load i16, i16 addrspace(1)* %in.gep, align 4
311  %ctpop = call i16 @llvm.ctpop.i16(i16 %val) nounwind readnone
312  %add = add i16 %ctpop, %sval
313  store i16 %add, i16 addrspace(1)* %out, align 4
314  ret void
315}
316
317define amdgpu_kernel void @v_ctpop_v2i16(<2 x i16> addrspace(1)* noalias %out, <2 x i16> addrspace(1)* noalias %in) nounwind {
318; SI-LABEL: v_ctpop_v2i16:
319; SI:       ; %bb.0:
320; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
321; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
322; SI-NEXT:    s_mov_b32 s7, 0xf000
323; SI-NEXT:    s_mov_b32 s2, 0
324; SI-NEXT:    s_mov_b32 s3, s7
325; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
326; SI-NEXT:    v_mov_b32_e32 v1, 0
327; SI-NEXT:    s_waitcnt lgkmcnt(0)
328; SI-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64
329; SI-NEXT:    s_mov_b32 s6, -1
330; SI-NEXT:    s_waitcnt vmcnt(0)
331; SI-NEXT:    v_and_b32_e32 v1, 0xffff, v0
332; SI-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
333; SI-NEXT:    v_bcnt_u32_b32_e64 v0, v0, 0
334; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
335; SI-NEXT:    v_bcnt_u32_b32_e64 v1, v1, 0
336; SI-NEXT:    v_or_b32_e32 v0, v1, v0
337; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
338; SI-NEXT:    s_endpgm
339;
340; VI-LABEL: v_ctpop_v2i16:
341; VI:       ; %bb.0:
342; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
343; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
344; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
345; VI-NEXT:    s_mov_b32 s7, 0xf000
346; VI-NEXT:    s_mov_b32 s6, -1
347; VI-NEXT:    s_waitcnt lgkmcnt(0)
348; VI-NEXT:    v_mov_b32_e32 v1, s1
349; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
350; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
351; VI-NEXT:    flat_load_dword v0, v[0:1]
352; VI-NEXT:    s_waitcnt vmcnt(0)
353; VI-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
354; VI-NEXT:    v_and_b32_e32 v0, 0xffff, v0
355; VI-NEXT:    v_bcnt_u32_b32 v1, v1, 0
356; VI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
357; VI-NEXT:    v_bcnt_u32_b32 v0, v0, 0
358; VI-NEXT:    v_or_b32_e32 v0, v0, v1
359; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
360; VI-NEXT:    s_endpgm
361;
362; EG-LABEL: v_ctpop_v2i16:
363; EG:       ; %bb.0:
364; EG-NEXT:    ALU 2, @8, KC0[CB0:0-32], KC1[]
365; EG-NEXT:    TEX 0 @6
366; EG-NEXT:    ALU 10, @11, KC0[CB0:0-32], KC1[]
367; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T6.X, 1
368; EG-NEXT:    CF_END
369; EG-NEXT:    PAD
370; EG-NEXT:    Fetch clause starting at 6:
371; EG-NEXT:     VTX_READ_32 T0.X, T0.X, 0, #1
372; EG-NEXT:    ALU clause starting at 8:
373; EG-NEXT:     LSHL * T0.W, T0.X, literal.x,
374; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
375; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, PV.W,
376; EG-NEXT:    ALU clause starting at 11:
377; EG-NEXT:     LSHR * T0.W, T0.X, literal.x,
378; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
379; EG-NEXT:     BCNT_INT T0.W, PV.W,
380; EG-NEXT:     AND_INT * T1.W, T0.X, literal.x,
381; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
382; EG-NEXT:     BCNT_INT T1.W, PS,
383; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
384; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
385; EG-NEXT:     OR_INT T0.X, PV.W, PS,
386; EG-NEXT:     LSHR * T6.X, KC0[2].Y, literal.x,
387; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
388  %tid = call i32 @llvm.amdgcn.workitem.id.x()
389  %in.gep = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %in, i32 %tid
390  %val = load <2 x i16>, <2 x i16> addrspace(1)* %in.gep, align 8
391  %ctpop = call <2 x i16> @llvm.ctpop.v2i16(<2 x i16> %val) nounwind readnone
392  store <2 x i16> %ctpop, <2 x i16> addrspace(1)* %out, align 8
393  ret void
394}
395
396define amdgpu_kernel void @v_ctpop_v4i16(<4 x i16> addrspace(1)* noalias %out, <4 x i16> addrspace(1)* noalias %in) nounwind {
397; SI-LABEL: v_ctpop_v4i16:
398; SI:       ; %bb.0:
399; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
400; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
401; SI-NEXT:    s_mov_b32 s7, 0xf000
402; SI-NEXT:    s_mov_b32 s2, 0
403; SI-NEXT:    s_mov_b32 s3, s7
404; SI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
405; SI-NEXT:    v_mov_b32_e32 v1, 0
406; SI-NEXT:    s_waitcnt lgkmcnt(0)
407; SI-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64
408; SI-NEXT:    s_mov_b32 s0, 0xffff
409; SI-NEXT:    s_mov_b32 s6, -1
410; SI-NEXT:    s_waitcnt vmcnt(0)
411; SI-NEXT:    v_and_b32_e32 v2, s0, v0
412; SI-NEXT:    v_and_b32_e32 v3, s0, v1
413; SI-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
414; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
415; SI-NEXT:    v_bcnt_u32_b32_e64 v1, v1, 0
416; SI-NEXT:    v_bcnt_u32_b32_e64 v0, v0, 0
417; SI-NEXT:    v_bcnt_u32_b32_e64 v3, v3, 0
418; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
419; SI-NEXT:    v_bcnt_u32_b32_e64 v2, v2, 0
420; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
421; SI-NEXT:    v_or_b32_e32 v1, v3, v1
422; SI-NEXT:    v_or_b32_e32 v0, v2, v0
423; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
424; SI-NEXT:    s_endpgm
425;
426; VI-LABEL: v_ctpop_v4i16:
427; VI:       ; %bb.0:
428; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
429; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
430; VI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
431; VI-NEXT:    s_mov_b32 s7, 0xf000
432; VI-NEXT:    s_mov_b32 s6, -1
433; VI-NEXT:    s_waitcnt lgkmcnt(0)
434; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
435; VI-NEXT:    v_mov_b32_e32 v1, s1
436; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
437; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
438; VI-NEXT:    s_mov_b32 s0, 0xffff
439; VI-NEXT:    s_waitcnt vmcnt(0)
440; VI-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
441; VI-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
442; VI-NEXT:    v_and_b32_e32 v1, s0, v1
443; VI-NEXT:    v_and_b32_e32 v0, s0, v0
444; VI-NEXT:    v_bcnt_u32_b32 v2, v2, 0
445; VI-NEXT:    v_bcnt_u32_b32 v3, v3, 0
446; VI-NEXT:    v_bcnt_u32_b32 v1, v1, 0
447; VI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
448; VI-NEXT:    v_bcnt_u32_b32 v0, v0, 0
449; VI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
450; VI-NEXT:    v_or_b32_e32 v1, v1, v2
451; VI-NEXT:    v_or_b32_e32 v0, v0, v3
452; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
453; VI-NEXT:    s_endpgm
454;
455; EG-LABEL: v_ctpop_v4i16:
456; EG:       ; %bb.0:
457; EG-NEXT:    ALU 2, @8, KC0[CB0:0-32], KC1[]
458; EG-NEXT:    TEX 0 @6
459; EG-NEXT:    ALU 42, @11, KC0[CB0:0-32], KC1[]
460; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T8.XY, T0.X, 1
461; EG-NEXT:    CF_END
462; EG-NEXT:    PAD
463; EG-NEXT:    Fetch clause starting at 6:
464; EG-NEXT:     VTX_READ_64 T0.XY, T0.X, 0, #1
465; EG-NEXT:    ALU clause starting at 8:
466; EG-NEXT:     LSHL * T0.W, T0.X, literal.x,
467; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
468; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, PV.W,
469; EG-NEXT:    ALU clause starting at 11:
470; EG-NEXT:     MOV T2.X, T0.X,
471; EG-NEXT:     MOV * T3.X, T0.Y,
472; EG-NEXT:     MOV T0.X, T4.X,
473; EG-NEXT:     MOV * T0.Y, PV.X,
474; EG-NEXT:     AND_INT * T0.W, PV.Y, literal.x,
475; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
476; EG-NEXT:     BCNT_INT T0.W, PV.W,
477; EG-NEXT:     AND_INT * T1.W, T0.X, literal.x,
478; EG-NEXT:    -65536(nan), 0(0.000000e+00)
479; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
480; EG-NEXT:     MOV T0.X, T3.X,
481; EG-NEXT:     MOV * T4.X, PV.W,
482; EG-NEXT:     MOV T0.Z, PS,
483; EG-NEXT:     LSHR * T0.W, T0.Y, literal.x,
484; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
485; EG-NEXT:     BCNT_INT T0.W, PV.W,
486; EG-NEXT:     AND_INT * T1.W, PV.Z, literal.x,
487; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
488; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
489; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
490; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
491; EG-NEXT:     MOV T4.X, PV.W,
492; EG-NEXT:     MOV T0.Y, T5.X,
493; EG-NEXT:     AND_INT * T0.W, T0.X, literal.x, BS:VEC_120/SCL_212
494; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
495; EG-NEXT:     BCNT_INT T0.W, PV.W,
496; EG-NEXT:     AND_INT * T1.W, PV.Y, literal.x,
497; EG-NEXT:    -65536(nan), 0(0.000000e+00)
498; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
499; EG-NEXT:     MOV * T5.X, PV.W,
500; EG-NEXT:     MOV T0.Y, PV.X,
501; EG-NEXT:     LSHR * T0.W, T0.X, literal.x,
502; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
503; EG-NEXT:     BCNT_INT T0.W, PV.W,
504; EG-NEXT:     AND_INT * T1.W, PV.Y, literal.x,
505; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
506; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
507; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
508; EG-NEXT:     LSHR T0.X, KC0[2].Y, literal.x,
509; EG-NEXT:     OR_INT * T8.Y, T1.W, PV.W,
510; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
511; EG-NEXT:     MOV T5.X, PV.Y,
512; EG-NEXT:     MOV * T8.X, T4.X,
513  %tid = call i32 @llvm.amdgcn.workitem.id.x()
514  %in.gep = getelementptr <4 x i16>, <4 x i16> addrspace(1)* %in, i32 %tid
515  %val = load <4 x i16>, <4 x i16> addrspace(1)* %in.gep, align 16
516  %ctpop = call <4 x i16> @llvm.ctpop.v4i16(<4 x i16> %val) nounwind readnone
517  store <4 x i16> %ctpop, <4 x i16> addrspace(1)* %out, align 16
518  ret void
519}
520
521define amdgpu_kernel void @v_ctpop_v8i16(<8 x i16> addrspace(1)* noalias %out, <8 x i16> addrspace(1)* noalias %in) nounwind {
522; SI-LABEL: v_ctpop_v8i16:
523; SI:       ; %bb.0:
524; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
525; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
526; SI-NEXT:    s_mov_b32 s7, 0xf000
527; SI-NEXT:    s_mov_b32 s2, 0
528; SI-NEXT:    s_mov_b32 s3, s7
529; SI-NEXT:    v_lshlrev_b32_e32 v0, 4, v0
530; SI-NEXT:    v_mov_b32_e32 v1, 0
531; SI-NEXT:    s_waitcnt lgkmcnt(0)
532; SI-NEXT:    buffer_load_dwordx4 v[0:3], v[0:1], s[0:3], 0 addr64
533; SI-NEXT:    s_mov_b32 s0, 0xffff
534; SI-NEXT:    s_mov_b32 s6, -1
535; SI-NEXT:    s_waitcnt vmcnt(0)
536; SI-NEXT:    v_and_b32_e32 v4, s0, v0
537; SI-NEXT:    v_and_b32_e32 v5, s0, v1
538; SI-NEXT:    v_and_b32_e32 v6, s0, v2
539; SI-NEXT:    v_and_b32_e32 v7, s0, v3
540; SI-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
541; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
542; SI-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
543; SI-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
544; SI-NEXT:    v_bcnt_u32_b32_e64 v3, v3, 0
545; SI-NEXT:    v_bcnt_u32_b32_e64 v2, v2, 0
546; SI-NEXT:    v_bcnt_u32_b32_e64 v1, v1, 0
547; SI-NEXT:    v_bcnt_u32_b32_e64 v0, v0, 0
548; SI-NEXT:    v_bcnt_u32_b32_e64 v7, v7, 0
549; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
550; SI-NEXT:    v_bcnt_u32_b32_e64 v6, v6, 0
551; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
552; SI-NEXT:    v_bcnt_u32_b32_e64 v5, v5, 0
553; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
554; SI-NEXT:    v_bcnt_u32_b32_e64 v4, v4, 0
555; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
556; SI-NEXT:    v_or_b32_e32 v3, v7, v3
557; SI-NEXT:    v_or_b32_e32 v2, v6, v2
558; SI-NEXT:    v_or_b32_e32 v1, v5, v1
559; SI-NEXT:    v_or_b32_e32 v0, v4, v0
560; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
561; SI-NEXT:    s_endpgm
562;
563; VI-LABEL: v_ctpop_v8i16:
564; VI:       ; %bb.0:
565; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
566; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
567; VI-NEXT:    v_lshlrev_b32_e32 v0, 4, v0
568; VI-NEXT:    s_mov_b32 s7, 0xf000
569; VI-NEXT:    s_mov_b32 s6, -1
570; VI-NEXT:    s_waitcnt lgkmcnt(0)
571; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
572; VI-NEXT:    v_mov_b32_e32 v1, s1
573; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
574; VI-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
575; VI-NEXT:    s_mov_b32 s0, 0xffff
576; VI-NEXT:    s_waitcnt vmcnt(0)
577; VI-NEXT:    v_lshrrev_b32_e32 v4, 16, v3
578; VI-NEXT:    v_lshrrev_b32_e32 v5, 16, v2
579; VI-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
580; VI-NEXT:    v_lshrrev_b32_e32 v7, 16, v0
581; VI-NEXT:    v_and_b32_e32 v3, s0, v3
582; VI-NEXT:    v_and_b32_e32 v2, s0, v2
583; VI-NEXT:    v_and_b32_e32 v1, s0, v1
584; VI-NEXT:    v_and_b32_e32 v0, s0, v0
585; VI-NEXT:    v_bcnt_u32_b32 v4, v4, 0
586; VI-NEXT:    v_bcnt_u32_b32 v5, v5, 0
587; VI-NEXT:    v_bcnt_u32_b32 v6, v6, 0
588; VI-NEXT:    v_bcnt_u32_b32 v7, v7, 0
589; VI-NEXT:    v_bcnt_u32_b32 v3, v3, 0
590; VI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
591; VI-NEXT:    v_bcnt_u32_b32 v2, v2, 0
592; VI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
593; VI-NEXT:    v_bcnt_u32_b32 v1, v1, 0
594; VI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
595; VI-NEXT:    v_bcnt_u32_b32 v0, v0, 0
596; VI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
597; VI-NEXT:    v_or_b32_e32 v3, v3, v4
598; VI-NEXT:    v_or_b32_e32 v2, v2, v5
599; VI-NEXT:    v_or_b32_e32 v1, v1, v6
600; VI-NEXT:    v_or_b32_e32 v0, v0, v7
601; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
602; VI-NEXT:    s_endpgm
603;
604; EG-LABEL: v_ctpop_v8i16:
605; EG:       ; %bb.0:
606; EG-NEXT:    ALU 3, @8, KC0[CB0:0-32], KC1[]
607; EG-NEXT:    TEX 0 @6
608; EG-NEXT:    ALU 73, @12, KC0[CB0:0-32], KC1[]
609; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XYZW, T12.X, 1
610; EG-NEXT:    CF_END
611; EG-NEXT:    PAD
612; EG-NEXT:    Fetch clause starting at 6:
613; EG-NEXT:     VTX_READ_128 T12.XYZW, T0.X, 0, #1
614; EG-NEXT:    ALU clause starting at 8:
615; EG-NEXT:     MOV T0.Y, T4.X,
616; EG-NEXT:     LSHL * T0.W, T0.X, literal.x, BS:VEC_120/SCL_212
617; EG-NEXT:    4(5.605194e-45), 0(0.000000e+00)
618; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, PV.W,
619; EG-NEXT:    ALU clause starting at 12:
620; EG-NEXT:     LSHR * T0.W, T12.X, literal.x,
621; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
622; EG-NEXT:     BCNT_INT * T0.W, PV.W,
623; EG-NEXT:     LSHL T0.W, PV.W, literal.x,
624; EG-NEXT:     AND_INT * T1.W, T0.Y, literal.y,
625; EG-NEXT:    16(2.242078e-44), 65535(9.183409e-41)
626; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
627; EG-NEXT:     MOV * T4.X, PV.W,
628; EG-NEXT:     MOV T0.X, PV.X,
629; EG-NEXT:     AND_INT * T0.W, T12.X, literal.x,
630; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
631; EG-NEXT:     BCNT_INT T0.W, PV.W,
632; EG-NEXT:     AND_INT * T1.W, PV.X, literal.x,
633; EG-NEXT:    -65536(nan), 0(0.000000e+00)
634; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
635; EG-NEXT:     MOV T4.X, PV.W,
636; EG-NEXT:     MOV * T0.X, T5.X,
637; EG-NEXT:     LSHR * T0.W, T12.Y, literal.x,
638; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
639; EG-NEXT:     BCNT_INT T0.W, PV.W,
640; EG-NEXT:     AND_INT * T1.W, T0.X, literal.x,
641; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
642; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
643; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
644; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
645; EG-NEXT:     MOV * T5.X, PV.W,
646; EG-NEXT:     MOV T0.X, PV.X,
647; EG-NEXT:     AND_INT * T0.W, T12.Y, literal.x,
648; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
649; EG-NEXT:     BCNT_INT T0.W, PV.W,
650; EG-NEXT:     AND_INT * T1.W, PV.X, literal.x,
651; EG-NEXT:    -65536(nan), 0(0.000000e+00)
652; EG-NEXT:     OR_INT * T0.Y, PS, PV.W,
653; EG-NEXT:     MOV T5.X, PV.Y,
654; EG-NEXT:     MOV * T0.X, T8.X,
655; EG-NEXT:     LSHR * T0.W, T12.Z, literal.x,
656; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
657; EG-NEXT:     BCNT_INT T0.W, PV.W,
658; EG-NEXT:     AND_INT * T1.W, T0.X, literal.x,
659; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
660; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
661; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
662; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
663; EG-NEXT:     MOV * T8.X, PV.W,
664; EG-NEXT:     MOV T0.X, PV.X,
665; EG-NEXT:     AND_INT * T0.W, T12.Z, literal.x,
666; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
667; EG-NEXT:     BCNT_INT T0.W, PV.W,
668; EG-NEXT:     AND_INT * T1.W, PV.X, literal.x,
669; EG-NEXT:    -65536(nan), 0(0.000000e+00)
670; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
671; EG-NEXT:     MOV T8.X, PV.W,
672; EG-NEXT:     MOV * T0.X, T9.X,
673; EG-NEXT:     LSHR * T0.W, T12.W, literal.x,
674; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
675; EG-NEXT:     BCNT_INT T0.W, PV.W,
676; EG-NEXT:     AND_INT * T1.W, T0.X, literal.x,
677; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
678; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
679; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
680; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
681; EG-NEXT:     MOV * T9.X, PV.W,
682; EG-NEXT:     MOV T0.X, PV.X,
683; EG-NEXT:     AND_INT * T0.W, T12.W, literal.x,
684; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
685; EG-NEXT:     BCNT_INT T0.W, PV.W,
686; EG-NEXT:     AND_INT * T1.W, PV.X, literal.x,
687; EG-NEXT:    -65536(nan), 0(0.000000e+00)
688; EG-NEXT:     LSHR T12.X, KC0[2].Y, literal.x,
689; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
690; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
691; EG-NEXT:     MOV T9.X, PV.W,
692; EG-NEXT:     MOV * T0.X, T4.X,
693; EG-NEXT:     MOV * T0.Z, T8.X,
694  %tid = call i32 @llvm.amdgcn.workitem.id.x()
695  %in.gep = getelementptr <8 x i16>, <8 x i16> addrspace(1)* %in, i32 %tid
696  %val = load <8 x i16>, <8 x i16> addrspace(1)* %in.gep, align 32
697  %ctpop = call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> %val) nounwind readnone
698  store <8 x i16> %ctpop, <8 x i16> addrspace(1)* %out, align 32
699  ret void
700}
701
702define amdgpu_kernel void @v_ctpop_v16i16(<16 x i16> addrspace(1)* noalias %out, <16 x i16> addrspace(1)* noalias %in) nounwind {
703; SI-LABEL: v_ctpop_v16i16:
704; SI:       ; %bb.0:
705; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
706; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
707; SI-NEXT:    s_mov_b32 s7, 0xf000
708; SI-NEXT:    s_mov_b32 s2, 0
709; SI-NEXT:    s_mov_b32 s3, s7
710; SI-NEXT:    v_lshlrev_b32_e32 v4, 5, v0
711; SI-NEXT:    v_mov_b32_e32 v5, 0
712; SI-NEXT:    s_waitcnt lgkmcnt(0)
713; SI-NEXT:    buffer_load_dwordx4 v[0:3], v[4:5], s[0:3], 0 addr64
714; SI-NEXT:    buffer_load_dwordx4 v[4:7], v[4:5], s[0:3], 0 addr64 offset:16
715; SI-NEXT:    s_mov_b32 s0, 0xffff
716; SI-NEXT:    s_mov_b32 s6, -1
717; SI-NEXT:    s_waitcnt vmcnt(1)
718; SI-NEXT:    v_and_b32_e32 v12, s0, v0
719; SI-NEXT:    s_waitcnt vmcnt(0)
720; SI-NEXT:    v_and_b32_e32 v8, s0, v4
721; SI-NEXT:    v_and_b32_e32 v9, s0, v5
722; SI-NEXT:    v_and_b32_e32 v10, s0, v6
723; SI-NEXT:    v_and_b32_e32 v11, s0, v7
724; SI-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
725; SI-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
726; SI-NEXT:    v_lshrrev_b32_e32 v6, 16, v6
727; SI-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
728; SI-NEXT:    v_and_b32_e32 v13, s0, v1
729; SI-NEXT:    v_and_b32_e32 v14, s0, v2
730; SI-NEXT:    v_and_b32_e32 v15, 0xffff, v3
731; SI-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
732; SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
733; SI-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
734; SI-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
735; SI-NEXT:    v_bcnt_u32_b32_e64 v7, v7, 0
736; SI-NEXT:    v_bcnt_u32_b32_e64 v6, v6, 0
737; SI-NEXT:    v_bcnt_u32_b32_e64 v5, v5, 0
738; SI-NEXT:    v_bcnt_u32_b32_e64 v4, v4, 0
739; SI-NEXT:    v_bcnt_u32_b32_e64 v3, v3, 0
740; SI-NEXT:    v_bcnt_u32_b32_e64 v2, v2, 0
741; SI-NEXT:    v_bcnt_u32_b32_e64 v1, v1, 0
742; SI-NEXT:    v_bcnt_u32_b32_e64 v0, v0, 0
743; SI-NEXT:    v_bcnt_u32_b32_e64 v11, v11, 0
744; SI-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
745; SI-NEXT:    v_bcnt_u32_b32_e64 v10, v10, 0
746; SI-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
747; SI-NEXT:    v_bcnt_u32_b32_e64 v9, v9, 0
748; SI-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
749; SI-NEXT:    v_bcnt_u32_b32_e64 v8, v8, 0
750; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
751; SI-NEXT:    v_bcnt_u32_b32_e64 v15, v15, 0
752; SI-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
753; SI-NEXT:    v_bcnt_u32_b32_e64 v14, v14, 0
754; SI-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
755; SI-NEXT:    v_bcnt_u32_b32_e64 v13, v13, 0
756; SI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
757; SI-NEXT:    v_bcnt_u32_b32_e64 v12, v12, 0
758; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
759; SI-NEXT:    v_or_b32_e32 v7, v11, v7
760; SI-NEXT:    v_or_b32_e32 v6, v10, v6
761; SI-NEXT:    v_or_b32_e32 v5, v9, v5
762; SI-NEXT:    v_or_b32_e32 v4, v8, v4
763; SI-NEXT:    v_or_b32_e32 v3, v15, v3
764; SI-NEXT:    v_or_b32_e32 v2, v14, v2
765; SI-NEXT:    v_or_b32_e32 v1, v13, v1
766; SI-NEXT:    v_or_b32_e32 v0, v12, v0
767; SI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:16
768; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
769; SI-NEXT:    s_endpgm
770;
771; VI-LABEL: v_ctpop_v16i16:
772; VI:       ; %bb.0:
773; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
774; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
775; VI-NEXT:    v_lshlrev_b32_e32 v0, 5, v0
776; VI-NEXT:    v_mov_b32_e32 v8, 0xffff
777; VI-NEXT:    s_mov_b32 s7, 0xf000
778; VI-NEXT:    s_mov_b32 s6, -1
779; VI-NEXT:    s_waitcnt lgkmcnt(0)
780; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
781; VI-NEXT:    v_mov_b32_e32 v1, s1
782; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
783; VI-NEXT:    v_add_u32_e32 v4, vcc, 16, v0
784; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
785; VI-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
786; VI-NEXT:    flat_load_dwordx4 v[4:7], v[4:5]
787; VI-NEXT:    s_mov_b32 s0, 0xffff
788; VI-NEXT:    s_waitcnt vmcnt(1)
789; VI-NEXT:    v_lshrrev_b32_e32 v9, 16, v3
790; VI-NEXT:    v_lshrrev_b32_e32 v10, 16, v2
791; VI-NEXT:    v_lshrrev_b32_e32 v11, 16, v1
792; VI-NEXT:    v_lshrrev_b32_e32 v12, 16, v0
793; VI-NEXT:    s_waitcnt vmcnt(0)
794; VI-NEXT:    v_lshrrev_b32_e32 v13, 16, v6
795; VI-NEXT:    v_lshrrev_b32_e32 v14, 16, v5
796; VI-NEXT:    v_lshrrev_b32_e32 v15, 16, v4
797; VI-NEXT:    v_and_b32_e32 v3, v8, v3
798; VI-NEXT:    v_and_b32_e32 v2, v8, v2
799; VI-NEXT:    v_and_b32_e32 v1, v8, v1
800; VI-NEXT:    v_and_b32_e32 v0, v8, v0
801; VI-NEXT:    v_lshrrev_b32_e32 v8, 16, v7
802; VI-NEXT:    v_bcnt_u32_b32 v9, v9, 0
803; VI-NEXT:    v_bcnt_u32_b32 v10, v10, 0
804; VI-NEXT:    v_bcnt_u32_b32 v11, v11, 0
805; VI-NEXT:    v_bcnt_u32_b32 v12, v12, 0
806; VI-NEXT:    v_and_b32_e32 v7, s0, v7
807; VI-NEXT:    v_and_b32_e32 v6, s0, v6
808; VI-NEXT:    v_and_b32_e32 v5, s0, v5
809; VI-NEXT:    v_and_b32_e32 v4, s0, v4
810; VI-NEXT:    v_bcnt_u32_b32 v8, v8, 0
811; VI-NEXT:    v_bcnt_u32_b32 v13, v13, 0
812; VI-NEXT:    v_bcnt_u32_b32 v14, v14, 0
813; VI-NEXT:    v_bcnt_u32_b32 v15, v15, 0
814; VI-NEXT:    v_bcnt_u32_b32 v3, v3, 0
815; VI-NEXT:    v_lshlrev_b32_e32 v9, 16, v9
816; VI-NEXT:    v_bcnt_u32_b32 v2, v2, 0
817; VI-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
818; VI-NEXT:    v_bcnt_u32_b32 v1, v1, 0
819; VI-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
820; VI-NEXT:    v_bcnt_u32_b32 v0, v0, 0
821; VI-NEXT:    v_lshlrev_b32_e32 v12, 16, v12
822; VI-NEXT:    v_bcnt_u32_b32 v7, v7, 0
823; VI-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
824; VI-NEXT:    v_bcnt_u32_b32 v6, v6, 0
825; VI-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
826; VI-NEXT:    v_bcnt_u32_b32 v5, v5, 0
827; VI-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
828; VI-NEXT:    v_bcnt_u32_b32 v4, v4, 0
829; VI-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
830; VI-NEXT:    v_or_b32_e32 v3, v3, v9
831; VI-NEXT:    v_or_b32_e32 v2, v2, v10
832; VI-NEXT:    v_or_b32_e32 v1, v1, v11
833; VI-NEXT:    v_or_b32_e32 v0, v0, v12
834; VI-NEXT:    v_or_b32_e32 v7, v7, v8
835; VI-NEXT:    v_or_b32_e32 v6, v6, v13
836; VI-NEXT:    v_or_b32_e32 v5, v5, v14
837; VI-NEXT:    v_or_b32_e32 v4, v4, v15
838; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
839; VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[4:7], 0 offset:16
840; VI-NEXT:    s_endpgm
841;
842; EG-LABEL: v_ctpop_v16i16:
843; EG:       ; %bb.0:
844; EG-NEXT:    ALU 3, @12, KC0[CB0:0-32], KC1[]
845; EG-NEXT:    TEX 1 @8
846; EG-NEXT:    ALU 114, @16, KC0[], KC1[]
847; EG-NEXT:    ALU 34, @131, KC0[CB0:0-32], KC1[]
848; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XYZW, T22.X, 0
849; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T20.XYZW, T21.X, 1
850; EG-NEXT:    CF_END
851; EG-NEXT:    PAD
852; EG-NEXT:    Fetch clause starting at 8:
853; EG-NEXT:     VTX_READ_128 T20.XYZW, T0.X, 16, #1
854; EG-NEXT:     VTX_READ_128 T21.XYZW, T0.X, 0, #1
855; EG-NEXT:    ALU clause starting at 12:
856; EG-NEXT:     MOV T0.Y, T4.X,
857; EG-NEXT:     LSHL * T0.W, T0.X, literal.x, BS:VEC_120/SCL_212
858; EG-NEXT:    5(7.006492e-45), 0(0.000000e+00)
859; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, PV.W,
860; EG-NEXT:    ALU clause starting at 16:
861; EG-NEXT:     LSHR * T0.W, T20.X, literal.x,
862; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
863; EG-NEXT:     BCNT_INT * T0.W, PV.W,
864; EG-NEXT:     LSHL T0.W, PV.W, literal.x,
865; EG-NEXT:     AND_INT * T1.W, T0.Y, literal.y,
866; EG-NEXT:    16(2.242078e-44), 65535(9.183409e-41)
867; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
868; EG-NEXT:     MOV * T4.X, PV.W,
869; EG-NEXT:     MOV T0.X, PV.X,
870; EG-NEXT:     AND_INT * T0.W, T20.X, literal.x,
871; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
872; EG-NEXT:     BCNT_INT T0.W, PV.W,
873; EG-NEXT:     AND_INT * T1.W, PV.X, literal.x,
874; EG-NEXT:    -65536(nan), 0(0.000000e+00)
875; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
876; EG-NEXT:     MOV T4.X, PV.W,
877; EG-NEXT:     MOV * T0.X, T5.X,
878; EG-NEXT:     LSHR * T0.W, T20.Y, literal.x,
879; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
880; EG-NEXT:     BCNT_INT T0.W, PV.W,
881; EG-NEXT:     AND_INT * T1.W, T0.X, literal.x,
882; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
883; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
884; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
885; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
886; EG-NEXT:     MOV * T5.X, PV.W,
887; EG-NEXT:     MOV T0.X, PV.X,
888; EG-NEXT:     AND_INT * T0.W, T20.Y, literal.x,
889; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
890; EG-NEXT:     BCNT_INT T0.W, PV.W,
891; EG-NEXT:     AND_INT * T1.W, PV.X, literal.x,
892; EG-NEXT:    -65536(nan), 0(0.000000e+00)
893; EG-NEXT:     OR_INT * T0.Y, PS, PV.W,
894; EG-NEXT:     MOV T5.X, PV.Y,
895; EG-NEXT:     MOV * T0.X, T8.X,
896; EG-NEXT:     LSHR * T0.W, T20.Z, literal.x,
897; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
898; EG-NEXT:     BCNT_INT T0.W, PV.W,
899; EG-NEXT:     AND_INT * T1.W, T0.X, literal.x,
900; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
901; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
902; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
903; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
904; EG-NEXT:     MOV * T8.X, PV.W,
905; EG-NEXT:     MOV T0.X, PV.X,
906; EG-NEXT:     AND_INT * T0.W, T20.Z, literal.x,
907; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
908; EG-NEXT:     BCNT_INT T0.W, PV.W,
909; EG-NEXT:     AND_INT * T1.W, PV.X, literal.x,
910; EG-NEXT:    -65536(nan), 0(0.000000e+00)
911; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
912; EG-NEXT:     MOV T8.X, PV.W,
913; EG-NEXT:     MOV * T0.X, T9.X,
914; EG-NEXT:     LSHR * T0.W, T20.W, literal.x,
915; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
916; EG-NEXT:     BCNT_INT T0.W, PV.W,
917; EG-NEXT:     AND_INT * T1.W, T0.X, literal.x,
918; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
919; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
920; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
921; EG-NEXT:     OR_INT * T0.W, T1.W, PV.W,
922; EG-NEXT:     MOV * T9.X, PV.W,
923; EG-NEXT:     MOV T0.X, PV.X,
924; EG-NEXT:     AND_INT * T0.W, T20.W, literal.x,
925; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
926; EG-NEXT:     BCNT_INT T0.W, PV.W,
927; EG-NEXT:     AND_INT * T1.W, PV.X, literal.x,
928; EG-NEXT:    -65536(nan), 0(0.000000e+00)
929; EG-NEXT:     OR_INT * T0.W, PS, PV.W,
930; EG-NEXT:     MOV T9.X, PV.W,
931; EG-NEXT:     MOV * T0.X, T12.X,
932; EG-NEXT:     LSHR * T1.W, T21.X, literal.x,
933; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
934; EG-NEXT:     BCNT_INT T1.W, PV.W,
935; EG-NEXT:     AND_INT * T2.W, T0.X, literal.x,
936; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
937; EG-NEXT:     LSHL * T1.W, PV.W, literal.x,
938; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
939; EG-NEXT:     OR_INT * T1.W, T2.W, PV.W,
940; EG-NEXT:     MOV * T12.X, PV.W,
941; EG-NEXT:     MOV T0.X, PV.X,
942; EG-NEXT:     AND_INT * T1.W, T21.X, literal.x,
943; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
944; EG-NEXT:     BCNT_INT T1.W, PV.W,
945; EG-NEXT:     AND_INT * T2.W, PV.X, literal.x,
946; EG-NEXT:    -65536(nan), 0(0.000000e+00)
947; EG-NEXT:     OR_INT * T1.W, PS, PV.W,
948; EG-NEXT:     MOV T12.X, PV.W,
949; EG-NEXT:     MOV * T0.X, T13.X,
950; EG-NEXT:     LSHR * T1.W, T21.Y, literal.x,
951; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
952; EG-NEXT:     BCNT_INT T1.W, PV.W,
953; EG-NEXT:     AND_INT * T2.W, T0.X, literal.x,
954; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
955; EG-NEXT:     LSHL * T1.W, PV.W, literal.x,
956; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
957; EG-NEXT:     OR_INT * T1.W, T2.W, PV.W,
958; EG-NEXT:     MOV * T13.X, PV.W,
959; EG-NEXT:     MOV T0.X, PV.X,
960; EG-NEXT:     AND_INT * T1.W, T21.Y, literal.x,
961; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
962; EG-NEXT:     BCNT_INT T1.W, PV.W,
963; EG-NEXT:     AND_INT * T2.W, PV.X, literal.x,
964; EG-NEXT:    -65536(nan), 0(0.000000e+00)
965; EG-NEXT:     OR_INT * T20.Y, PS, PV.W,
966; EG-NEXT:     MOV T13.X, PV.Y,
967; EG-NEXT:     MOV * T0.X, T16.X,
968; EG-NEXT:     LSHR * T1.W, T21.Z, literal.x,
969; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
970; EG-NEXT:     BCNT_INT T1.W, PV.W,
971; EG-NEXT:     AND_INT * T2.W, T0.X, literal.x,
972; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
973; EG-NEXT:     LSHL * T1.W, PV.W, literal.x,
974; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
975; EG-NEXT:     OR_INT * T1.W, T2.W, PV.W,
976; EG-NEXT:    ALU clause starting at 131:
977; EG-NEXT:     MOV * T16.X, T1.W,
978; EG-NEXT:     MOV T0.X, PV.X,
979; EG-NEXT:     AND_INT * T1.W, T21.Z, literal.x,
980; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
981; EG-NEXT:     BCNT_INT T1.W, PV.W,
982; EG-NEXT:     AND_INT * T2.W, PV.X, literal.x,
983; EG-NEXT:    -65536(nan), 0(0.000000e+00)
984; EG-NEXT:     OR_INT * T1.W, PS, PV.W,
985; EG-NEXT:     MOV T16.X, PV.W,
986; EG-NEXT:     MOV * T0.X, T17.X,
987; EG-NEXT:     LSHR * T1.W, T21.W, literal.x,
988; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
989; EG-NEXT:     BCNT_INT T1.W, PV.W,
990; EG-NEXT:     AND_INT * T2.W, T0.X, literal.x,
991; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
992; EG-NEXT:     LSHL * T1.W, PV.W, literal.x,
993; EG-NEXT:    16(2.242078e-44), 0(0.000000e+00)
994; EG-NEXT:     OR_INT * T1.W, T2.W, PV.W,
995; EG-NEXT:     MOV * T17.X, PV.W,
996; EG-NEXT:     MOV T0.X, PV.X,
997; EG-NEXT:     AND_INT T1.W, T21.W, literal.x,
998; EG-NEXT:     LSHR * T21.X, KC0[2].Y, literal.y,
999; EG-NEXT:    65535(9.183409e-41), 2(2.802597e-45)
1000; EG-NEXT:     AND_INT T0.Z, PV.X, literal.x,
1001; EG-NEXT:     BCNT_INT T1.W, PV.W,
1002; EG-NEXT:     ADD_INT * T2.W, KC0[2].Y, literal.y,
1003; EG-NEXT:    -65536(nan), 16(2.242078e-44)
1004; EG-NEXT:     LSHR T22.X, PS, literal.x,
1005; EG-NEXT:     OR_INT * T20.W, PV.Z, PV.W,
1006; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
1007; EG-NEXT:     MOV T17.X, PV.W,
1008; EG-NEXT:     MOV * T0.X, T4.X,
1009; EG-NEXT:     MOV * T0.Z, T8.X,
1010; EG-NEXT:     MOV T20.X, T12.X,
1011; EG-NEXT:     MOV * T20.Z, T16.X, BS:VEC_120/SCL_212
1012  %tid = call i32 @llvm.amdgcn.workitem.id.x()
1013  %in.gep = getelementptr <16 x i16>, <16 x i16> addrspace(1)* %in, i32 %tid
1014  %val = load <16 x i16>, <16 x i16> addrspace(1)* %in.gep, align 32
1015  %ctpop = call <16 x i16> @llvm.ctpop.v16i16(<16 x i16> %val) nounwind readnone
1016  store <16 x i16> %ctpop, <16 x i16> addrspace(1)* %out, align 32
1017  ret void
1018}
1019
1020define amdgpu_kernel void @v_ctpop_i16_add_inline_constant(i16 addrspace(1)* noalias %out, i16 addrspace(1)* noalias %in) nounwind {
1021; SI-LABEL: v_ctpop_i16_add_inline_constant:
1022; SI:       ; %bb.0:
1023; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
1024; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
1025; SI-NEXT:    s_mov_b32 s7, 0xf000
1026; SI-NEXT:    s_mov_b32 s2, 0
1027; SI-NEXT:    s_mov_b32 s3, s7
1028; SI-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
1029; SI-NEXT:    v_mov_b32_e32 v1, 0
1030; SI-NEXT:    s_waitcnt lgkmcnt(0)
1031; SI-NEXT:    buffer_load_ushort v0, v[0:1], s[0:3], 0 addr64
1032; SI-NEXT:    s_mov_b32 s6, -1
1033; SI-NEXT:    s_waitcnt vmcnt(0)
1034; SI-NEXT:    v_bcnt_u32_b32_e64 v0, v0, 4
1035; SI-NEXT:    buffer_store_short v0, off, s[4:7], 0
1036; SI-NEXT:    s_endpgm
1037;
1038; VI-LABEL: v_ctpop_i16_add_inline_constant:
1039; VI:       ; %bb.0:
1040; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
1041; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
1042; VI-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
1043; VI-NEXT:    s_mov_b32 s7, 0xf000
1044; VI-NEXT:    s_mov_b32 s6, -1
1045; VI-NEXT:    s_waitcnt lgkmcnt(0)
1046; VI-NEXT:    v_mov_b32_e32 v1, s1
1047; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
1048; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
1049; VI-NEXT:    flat_load_ushort v0, v[0:1]
1050; VI-NEXT:    s_waitcnt vmcnt(0)
1051; VI-NEXT:    v_bcnt_u32_b32 v0, v0, 4
1052; VI-NEXT:    buffer_store_short v0, off, s[4:7], 0
1053; VI-NEXT:    s_endpgm
1054;
1055; EG-LABEL: v_ctpop_i16_add_inline_constant:
1056; EG:       ; %bb.0:
1057; EG-NEXT:    ALU 1, @8, KC0[CB0:0-32], KC1[]
1058; EG-NEXT:    TEX 0 @6
1059; EG-NEXT:    ALU 12, @10, KC0[CB0:0-32], KC1[]
1060; EG-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
1061; EG-NEXT:    CF_END
1062; EG-NEXT:    PAD
1063; EG-NEXT:    Fetch clause starting at 6:
1064; EG-NEXT:     VTX_READ_16 T0.X, T0.X, 0, #1
1065; EG-NEXT:    ALU clause starting at 8:
1066; EG-NEXT:     LSHL * T0.W, T0.X, 1,
1067; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, PV.W,
1068; EG-NEXT:    ALU clause starting at 10:
1069; EG-NEXT:     BCNT_INT T0.W, T0.X,
1070; EG-NEXT:     AND_INT * T1.W, KC0[2].Y, literal.x,
1071; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
1072; EG-NEXT:     ADD_INT T0.W, PV.W, literal.x,
1073; EG-NEXT:     LSHL * T1.W, PS, literal.y,
1074; EG-NEXT:    4(5.605194e-45), 3(4.203895e-45)
1075; EG-NEXT:     LSHL T0.X, PV.W, PS,
1076; EG-NEXT:     LSHL * T0.W, literal.x, PS,
1077; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
1078; EG-NEXT:     MOV T0.Y, 0.0,
1079; EG-NEXT:     MOV * T0.Z, 0.0,
1080; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
1081; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
1082  %tid = call i32 @llvm.amdgcn.workitem.id.x()
1083  %in.gep = getelementptr i16, i16 addrspace(1)* %in, i32 %tid
1084  %val = load i16, i16 addrspace(1)* %in.gep, align 4
1085  %ctpop = call i16 @llvm.ctpop.i16(i16 %val) nounwind readnone
1086  %add = add i16 %ctpop, 4
1087  store i16 %add, i16 addrspace(1)* %out, align 4
1088  ret void
1089}
1090
1091define amdgpu_kernel void @v_ctpop_i16_add_inline_constant_inv(i16 addrspace(1)* noalias %out, i16 addrspace(1)* noalias %in) nounwind {
1092; SI-LABEL: v_ctpop_i16_add_inline_constant_inv:
1093; SI:       ; %bb.0:
1094; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
1095; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
1096; SI-NEXT:    s_mov_b32 s7, 0xf000
1097; SI-NEXT:    s_mov_b32 s2, 0
1098; SI-NEXT:    s_mov_b32 s3, s7
1099; SI-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
1100; SI-NEXT:    v_mov_b32_e32 v1, 0
1101; SI-NEXT:    s_waitcnt lgkmcnt(0)
1102; SI-NEXT:    buffer_load_ushort v0, v[0:1], s[0:3], 0 addr64
1103; SI-NEXT:    s_mov_b32 s6, -1
1104; SI-NEXT:    s_waitcnt vmcnt(0)
1105; SI-NEXT:    v_bcnt_u32_b32_e64 v0, v0, 4
1106; SI-NEXT:    buffer_store_short v0, off, s[4:7], 0
1107; SI-NEXT:    s_endpgm
1108;
1109; VI-LABEL: v_ctpop_i16_add_inline_constant_inv:
1110; VI:       ; %bb.0:
1111; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
1112; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
1113; VI-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
1114; VI-NEXT:    s_mov_b32 s7, 0xf000
1115; VI-NEXT:    s_mov_b32 s6, -1
1116; VI-NEXT:    s_waitcnt lgkmcnt(0)
1117; VI-NEXT:    v_mov_b32_e32 v1, s1
1118; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
1119; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
1120; VI-NEXT:    flat_load_ushort v0, v[0:1]
1121; VI-NEXT:    s_waitcnt vmcnt(0)
1122; VI-NEXT:    v_bcnt_u32_b32 v0, v0, 4
1123; VI-NEXT:    buffer_store_short v0, off, s[4:7], 0
1124; VI-NEXT:    s_endpgm
1125;
1126; EG-LABEL: v_ctpop_i16_add_inline_constant_inv:
1127; EG:       ; %bb.0:
1128; EG-NEXT:    ALU 1, @8, KC0[CB0:0-32], KC1[]
1129; EG-NEXT:    TEX 0 @6
1130; EG-NEXT:    ALU 12, @10, KC0[CB0:0-32], KC1[]
1131; EG-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
1132; EG-NEXT:    CF_END
1133; EG-NEXT:    PAD
1134; EG-NEXT:    Fetch clause starting at 6:
1135; EG-NEXT:     VTX_READ_16 T0.X, T0.X, 0, #1
1136; EG-NEXT:    ALU clause starting at 8:
1137; EG-NEXT:     LSHL * T0.W, T0.X, 1,
1138; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, PV.W,
1139; EG-NEXT:    ALU clause starting at 10:
1140; EG-NEXT:     BCNT_INT T0.W, T0.X,
1141; EG-NEXT:     AND_INT * T1.W, KC0[2].Y, literal.x,
1142; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
1143; EG-NEXT:     ADD_INT T0.W, PV.W, literal.x,
1144; EG-NEXT:     LSHL * T1.W, PS, literal.y,
1145; EG-NEXT:    4(5.605194e-45), 3(4.203895e-45)
1146; EG-NEXT:     LSHL T0.X, PV.W, PS,
1147; EG-NEXT:     LSHL * T0.W, literal.x, PS,
1148; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
1149; EG-NEXT:     MOV T0.Y, 0.0,
1150; EG-NEXT:     MOV * T0.Z, 0.0,
1151; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
1152; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
1153  %tid = call i32 @llvm.amdgcn.workitem.id.x()
1154  %in.gep = getelementptr i16, i16 addrspace(1)* %in, i32 %tid
1155  %val = load i16, i16 addrspace(1)* %in.gep, align 4
1156  %ctpop = call i16 @llvm.ctpop.i16(i16 %val) nounwind readnone
1157  %add = add i16 4, %ctpop
1158  store i16 %add, i16 addrspace(1)* %out, align 4
1159  ret void
1160}
1161
1162define amdgpu_kernel void @v_ctpop_i16_add_literal(i16 addrspace(1)* noalias %out, i16 addrspace(1)* noalias %in) nounwind {
1163; SI-LABEL: v_ctpop_i16_add_literal:
1164; SI:       ; %bb.0:
1165; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
1166; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
1167; SI-NEXT:    s_mov_b32 s7, 0xf000
1168; SI-NEXT:    s_mov_b32 s2, 0
1169; SI-NEXT:    s_mov_b32 s3, s7
1170; SI-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
1171; SI-NEXT:    v_mov_b32_e32 v1, 0
1172; SI-NEXT:    s_waitcnt lgkmcnt(0)
1173; SI-NEXT:    buffer_load_ushort v0, v[0:1], s[0:3], 0 addr64
1174; SI-NEXT:    s_movk_i32 s0, 0x3e7
1175; SI-NEXT:    s_mov_b32 s6, -1
1176; SI-NEXT:    s_waitcnt vmcnt(0)
1177; SI-NEXT:    v_bcnt_u32_b32_e64 v0, v0, s0
1178; SI-NEXT:    buffer_store_short v0, off, s[4:7], 0
1179; SI-NEXT:    s_endpgm
1180;
1181; VI-LABEL: v_ctpop_i16_add_literal:
1182; VI:       ; %bb.0:
1183; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
1184; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
1185; VI-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
1186; VI-NEXT:    s_mov_b32 s7, 0xf000
1187; VI-NEXT:    s_mov_b32 s6, -1
1188; VI-NEXT:    s_waitcnt lgkmcnt(0)
1189; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
1190; VI-NEXT:    v_mov_b32_e32 v1, s1
1191; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
1192; VI-NEXT:    flat_load_ushort v0, v[0:1]
1193; VI-NEXT:    s_movk_i32 s0, 0x3e7
1194; VI-NEXT:    s_waitcnt vmcnt(0)
1195; VI-NEXT:    v_bcnt_u32_b32 v0, v0, s0
1196; VI-NEXT:    buffer_store_short v0, off, s[4:7], 0
1197; VI-NEXT:    s_endpgm
1198;
1199; EG-LABEL: v_ctpop_i16_add_literal:
1200; EG:       ; %bb.0:
1201; EG-NEXT:    ALU 1, @8, KC0[CB0:0-32], KC1[]
1202; EG-NEXT:    TEX 0 @6
1203; EG-NEXT:    ALU 12, @10, KC0[CB0:0-32], KC1[]
1204; EG-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
1205; EG-NEXT:    CF_END
1206; EG-NEXT:    PAD
1207; EG-NEXT:    Fetch clause starting at 6:
1208; EG-NEXT:     VTX_READ_16 T0.X, T0.X, 0, #1
1209; EG-NEXT:    ALU clause starting at 8:
1210; EG-NEXT:     LSHL * T0.W, T0.X, 1,
1211; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, PV.W,
1212; EG-NEXT:    ALU clause starting at 10:
1213; EG-NEXT:     BCNT_INT T0.W, T0.X,
1214; EG-NEXT:     AND_INT * T1.W, KC0[2].Y, literal.x,
1215; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
1216; EG-NEXT:     ADD_INT T0.W, PV.W, literal.x,
1217; EG-NEXT:     LSHL * T1.W, PS, literal.y,
1218; EG-NEXT:    999(1.399897e-42), 3(4.203895e-45)
1219; EG-NEXT:     LSHL T0.X, PV.W, PS,
1220; EG-NEXT:     LSHL * T0.W, literal.x, PS,
1221; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
1222; EG-NEXT:     MOV T0.Y, 0.0,
1223; EG-NEXT:     MOV * T0.Z, 0.0,
1224; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
1225; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
1226  %tid = call i32 @llvm.amdgcn.workitem.id.x()
1227  %in.gep = getelementptr i16, i16 addrspace(1)* %in, i32 %tid
1228  %val = load i16, i16 addrspace(1)* %in.gep, align 4
1229  %ctpop = call i16 @llvm.ctpop.i16(i16 %val) nounwind readnone
1230  %add = add i16 %ctpop, 999
1231  store i16 %add, i16 addrspace(1)* %out, align 4
1232  ret void
1233}
1234
1235define amdgpu_kernel void @v_ctpop_i16_add_var(i16 addrspace(1)* noalias %out, i16 addrspace(1)* noalias %in, i16 %const) nounwind {
1236; SI-LABEL: v_ctpop_i16_add_var:
1237; SI:       ; %bb.0:
1238; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
1239; SI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xb
1240; SI-NEXT:    s_load_dword s0, s[0:1], 0xd
1241; SI-NEXT:    s_mov_b32 s7, 0xf000
1242; SI-NEXT:    s_mov_b32 s10, 0
1243; SI-NEXT:    s_mov_b32 s11, s7
1244; SI-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
1245; SI-NEXT:    v_mov_b32_e32 v1, 0
1246; SI-NEXT:    s_waitcnt lgkmcnt(0)
1247; SI-NEXT:    buffer_load_ushort v0, v[0:1], s[8:11], 0 addr64
1248; SI-NEXT:    s_mov_b32 s6, -1
1249; SI-NEXT:    s_waitcnt vmcnt(0)
1250; SI-NEXT:    v_bcnt_u32_b32_e64 v0, v0, s0
1251; SI-NEXT:    buffer_store_short v0, off, s[4:7], 0
1252; SI-NEXT:    s_endpgm
1253;
1254; VI-LABEL: v_ctpop_i16_add_var:
1255; VI:       ; %bb.0:
1256; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
1257; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
1258; VI-NEXT:    s_load_dword s0, s[0:1], 0x34
1259; VI-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
1260; VI-NEXT:    s_mov_b32 s7, 0xf000
1261; VI-NEXT:    s_mov_b32 s6, -1
1262; VI-NEXT:    s_waitcnt lgkmcnt(0)
1263; VI-NEXT:    v_mov_b32_e32 v1, s3
1264; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v0
1265; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
1266; VI-NEXT:    flat_load_ushort v0, v[0:1]
1267; VI-NEXT:    s_waitcnt vmcnt(0)
1268; VI-NEXT:    v_bcnt_u32_b32 v0, v0, s0
1269; VI-NEXT:    buffer_store_short v0, off, s[4:7], 0
1270; VI-NEXT:    s_endpgm
1271;
1272; EG-LABEL: v_ctpop_i16_add_var:
1273; EG:       ; %bb.0:
1274; EG-NEXT:    ALU 1, @12, KC0[CB0:0-32], KC1[]
1275; EG-NEXT:    TEX 0 @8
1276; EG-NEXT:    ALU 0, @14, KC0[], KC1[]
1277; EG-NEXT:    TEX 0 @10
1278; EG-NEXT:    ALU 13, @15, KC0[CB0:0-32], KC1[]
1279; EG-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
1280; EG-NEXT:    CF_END
1281; EG-NEXT:    PAD
1282; EG-NEXT:    Fetch clause starting at 8:
1283; EG-NEXT:     VTX_READ_16 T0.X, T0.X, 0, #1
1284; EG-NEXT:    Fetch clause starting at 10:
1285; EG-NEXT:     VTX_READ_16 T1.X, T1.X, 44, #3
1286; EG-NEXT:    ALU clause starting at 12:
1287; EG-NEXT:     LSHL * T0.W, T0.X, 1,
1288; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, PV.W,
1289; EG-NEXT:    ALU clause starting at 14:
1290; EG-NEXT:     MOV * T1.X, 0.0,
1291; EG-NEXT:    ALU clause starting at 15:
1292; EG-NEXT:     BCNT_INT T0.W, T0.X,
1293; EG-NEXT:     AND_INT * T1.W, KC0[2].Y, literal.x,
1294; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
1295; EG-NEXT:     ADD_INT * T0.W, PV.W, T1.X,
1296; EG-NEXT:     AND_INT T0.W, PV.W, literal.x,
1297; EG-NEXT:     LSHL * T1.W, T1.W, literal.y,
1298; EG-NEXT:    65535(9.183409e-41), 3(4.203895e-45)
1299; EG-NEXT:     LSHL T0.X, PV.W, PS,
1300; EG-NEXT:     LSHL * T0.W, literal.x, PS,
1301; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
1302; EG-NEXT:     MOV T0.Y, 0.0,
1303; EG-NEXT:     MOV * T0.Z, 0.0,
1304; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
1305; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
1306  %tid = call i32 @llvm.amdgcn.workitem.id.x()
1307  %in.gep = getelementptr i16, i16 addrspace(1)* %in, i32 %tid
1308  %val = load i16, i16 addrspace(1)* %in.gep, align 4
1309  %ctpop = call i16 @llvm.ctpop.i16(i16 %val) nounwind readnone
1310  %add = add i16 %ctpop, %const
1311  store i16 %add, i16 addrspace(1)* %out, align 4
1312  ret void
1313}
1314
1315define amdgpu_kernel void @v_ctpop_i16_add_var_inv(i16 addrspace(1)* noalias %out, i16 addrspace(1)* noalias %in, i16 %const) nounwind {
1316; SI-LABEL: v_ctpop_i16_add_var_inv:
1317; SI:       ; %bb.0:
1318; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
1319; SI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xb
1320; SI-NEXT:    s_load_dword s0, s[0:1], 0xd
1321; SI-NEXT:    s_mov_b32 s7, 0xf000
1322; SI-NEXT:    s_mov_b32 s10, 0
1323; SI-NEXT:    s_mov_b32 s11, s7
1324; SI-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
1325; SI-NEXT:    v_mov_b32_e32 v1, 0
1326; SI-NEXT:    s_waitcnt lgkmcnt(0)
1327; SI-NEXT:    buffer_load_ushort v0, v[0:1], s[8:11], 0 addr64
1328; SI-NEXT:    s_mov_b32 s6, -1
1329; SI-NEXT:    s_waitcnt vmcnt(0)
1330; SI-NEXT:    v_bcnt_u32_b32_e64 v0, v0, s0
1331; SI-NEXT:    buffer_store_short v0, off, s[4:7], 0
1332; SI-NEXT:    s_endpgm
1333;
1334; VI-LABEL: v_ctpop_i16_add_var_inv:
1335; VI:       ; %bb.0:
1336; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
1337; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
1338; VI-NEXT:    s_load_dword s0, s[0:1], 0x34
1339; VI-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
1340; VI-NEXT:    s_mov_b32 s7, 0xf000
1341; VI-NEXT:    s_mov_b32 s6, -1
1342; VI-NEXT:    s_waitcnt lgkmcnt(0)
1343; VI-NEXT:    v_mov_b32_e32 v1, s3
1344; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v0
1345; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
1346; VI-NEXT:    flat_load_ushort v0, v[0:1]
1347; VI-NEXT:    s_waitcnt vmcnt(0)
1348; VI-NEXT:    v_bcnt_u32_b32 v0, v0, s0
1349; VI-NEXT:    buffer_store_short v0, off, s[4:7], 0
1350; VI-NEXT:    s_endpgm
1351;
1352; EG-LABEL: v_ctpop_i16_add_var_inv:
1353; EG:       ; %bb.0:
1354; EG-NEXT:    ALU 1, @12, KC0[CB0:0-32], KC1[]
1355; EG-NEXT:    TEX 0 @8
1356; EG-NEXT:    ALU 0, @14, KC0[], KC1[]
1357; EG-NEXT:    TEX 0 @10
1358; EG-NEXT:    ALU 13, @15, KC0[CB0:0-32], KC1[]
1359; EG-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
1360; EG-NEXT:    CF_END
1361; EG-NEXT:    PAD
1362; EG-NEXT:    Fetch clause starting at 8:
1363; EG-NEXT:     VTX_READ_16 T0.X, T0.X, 0, #1
1364; EG-NEXT:    Fetch clause starting at 10:
1365; EG-NEXT:     VTX_READ_16 T1.X, T1.X, 44, #3
1366; EG-NEXT:    ALU clause starting at 12:
1367; EG-NEXT:     LSHL * T0.W, T0.X, 1,
1368; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, PV.W,
1369; EG-NEXT:    ALU clause starting at 14:
1370; EG-NEXT:     MOV * T1.X, 0.0,
1371; EG-NEXT:    ALU clause starting at 15:
1372; EG-NEXT:     BCNT_INT T0.W, T0.X,
1373; EG-NEXT:     AND_INT * T1.W, KC0[2].Y, literal.x,
1374; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
1375; EG-NEXT:     ADD_INT * T0.W, T1.X, PV.W,
1376; EG-NEXT:     AND_INT T0.W, PV.W, literal.x,
1377; EG-NEXT:     LSHL * T1.W, T1.W, literal.y,
1378; EG-NEXT:    65535(9.183409e-41), 3(4.203895e-45)
1379; EG-NEXT:     LSHL T0.X, PV.W, PS,
1380; EG-NEXT:     LSHL * T0.W, literal.x, PS,
1381; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
1382; EG-NEXT:     MOV T0.Y, 0.0,
1383; EG-NEXT:     MOV * T0.Z, 0.0,
1384; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
1385; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
1386  %tid = call i32 @llvm.amdgcn.workitem.id.x()
1387  %in.gep = getelementptr i16, i16 addrspace(1)* %in, i32 %tid
1388  %val = load i16, i16 addrspace(1)* %in.gep, align 4
1389  %ctpop = call i16 @llvm.ctpop.i16(i16 %val) nounwind readnone
1390  %add = add i16 %const, %ctpop
1391  store i16 %add, i16 addrspace(1)* %out, align 4
1392  ret void
1393}
1394
1395define amdgpu_kernel void @v_ctpop_i16_add_vvar_inv(i16 addrspace(1)* noalias %out, i16 addrspace(1)* noalias %in, i16 addrspace(1)* noalias %constptr) nounwind {
1396; SI-LABEL: v_ctpop_i16_add_vvar_inv:
1397; SI:       ; %bb.0:
1398; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
1399; SI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xb
1400; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
1401; SI-NEXT:    s_mov_b32 s7, 0xf000
1402; SI-NEXT:    s_mov_b32 s10, 0
1403; SI-NEXT:    s_mov_b32 s11, s7
1404; SI-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
1405; SI-NEXT:    v_mov_b32_e32 v1, 0
1406; SI-NEXT:    s_mov_b64 s[2:3], s[10:11]
1407; SI-NEXT:    s_waitcnt lgkmcnt(0)
1408; SI-NEXT:    buffer_load_ushort v2, v[0:1], s[8:11], 0 addr64
1409; SI-NEXT:    buffer_load_ushort v0, v[0:1], s[0:3], 0 addr64
1410; SI-NEXT:    s_mov_b32 s6, -1
1411; SI-NEXT:    s_waitcnt vmcnt(0)
1412; SI-NEXT:    v_bcnt_u32_b32_e32 v0, v2, v0
1413; SI-NEXT:    buffer_store_short v0, off, s[4:7], 0
1414; SI-NEXT:    s_endpgm
1415;
1416; VI-LABEL: v_ctpop_i16_add_vvar_inv:
1417; VI:       ; %bb.0:
1418; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
1419; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
1420; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
1421; VI-NEXT:    v_lshlrev_b32_e32 v2, 1, v0
1422; VI-NEXT:    s_mov_b32 s7, 0xf000
1423; VI-NEXT:    s_mov_b32 s6, -1
1424; VI-NEXT:    s_waitcnt lgkmcnt(0)
1425; VI-NEXT:    v_mov_b32_e32 v1, s3
1426; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
1427; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
1428; VI-NEXT:    flat_load_ushort v3, v[0:1]
1429; VI-NEXT:    v_mov_b32_e32 v1, s1
1430; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
1431; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
1432; VI-NEXT:    flat_load_ushort v0, v[0:1]
1433; VI-NEXT:    s_waitcnt vmcnt(0)
1434; VI-NEXT:    v_bcnt_u32_b32 v0, v3, v0
1435; VI-NEXT:    buffer_store_short v0, off, s[4:7], 0
1436; VI-NEXT:    s_endpgm
1437;
1438; EG-LABEL: v_ctpop_i16_add_vvar_inv:
1439; EG:       ; %bb.0:
1440; EG-NEXT:    ALU 1, @12, KC0[CB0:0-32], KC1[]
1441; EG-NEXT:    TEX 0 @8
1442; EG-NEXT:    ALU 0, @14, KC0[CB0:0-32], KC1[]
1443; EG-NEXT:    TEX 0 @10
1444; EG-NEXT:    ALU 13, @15, KC0[CB0:0-32], KC1[]
1445; EG-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
1446; EG-NEXT:    CF_END
1447; EG-NEXT:    PAD
1448; EG-NEXT:    Fetch clause starting at 8:
1449; EG-NEXT:     VTX_READ_16 T0.X, T0.X, 0, #1
1450; EG-NEXT:    Fetch clause starting at 10:
1451; EG-NEXT:     VTX_READ_16 T1.X, T1.X, 0, #1
1452; EG-NEXT:    ALU clause starting at 12:
1453; EG-NEXT:     LSHL * T0.W, T0.X, 1,
1454; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, PV.W,
1455; EG-NEXT:    ALU clause starting at 14:
1456; EG-NEXT:     ADD_INT * T1.X, KC0[2].W, T0.W,
1457; EG-NEXT:    ALU clause starting at 15:
1458; EG-NEXT:     BCNT_INT T0.W, T0.X,
1459; EG-NEXT:     AND_INT * T1.W, KC0[2].Y, literal.x,
1460; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
1461; EG-NEXT:     ADD_INT * T0.W, T1.X, PV.W,
1462; EG-NEXT:     AND_INT T0.W, PV.W, literal.x,
1463; EG-NEXT:     LSHL * T1.W, T1.W, literal.y,
1464; EG-NEXT:    65535(9.183409e-41), 3(4.203895e-45)
1465; EG-NEXT:     LSHL T0.X, PV.W, PS,
1466; EG-NEXT:     LSHL * T0.W, literal.x, PS,
1467; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
1468; EG-NEXT:     MOV T0.Y, 0.0,
1469; EG-NEXT:     MOV * T0.Z, 0.0,
1470; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
1471; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
1472  %tid = call i32 @llvm.amdgcn.workitem.id.x()
1473  %in.gep = getelementptr i16, i16 addrspace(1)* %in, i32 %tid
1474  %val = load i16, i16 addrspace(1)* %in.gep, align 4
1475  %ctpop = call i16 @llvm.ctpop.i16(i16 %val) nounwind readnone
1476  %gep = getelementptr i16, i16 addrspace(1)* %constptr, i32 %tid
1477  %const = load i16, i16 addrspace(1)* %gep, align 4
1478  %add = add i16 %const, %ctpop
1479  store i16 %add, i16 addrspace(1)* %out, align 4
1480  ret void
1481}
1482
1483; FIXME: We currently disallow SALU instructions in all branches,
1484; but there are some cases when the should be allowed.
1485define amdgpu_kernel void @ctpop_i16_in_br(i16 addrspace(1)* %out, i16 addrspace(1)* %in, i16 %ctpop_arg, i16 %cond) {
1486; SI-LABEL: ctpop_i16_in_br:
1487; SI:       ; %bb.0: ; %entry
1488; SI-NEXT:    s_load_dword s4, s[0:1], 0xd
1489; SI-NEXT:    s_waitcnt lgkmcnt(0)
1490; SI-NEXT:    s_lshr_b32 s2, s4, 16
1491; SI-NEXT:    s_cmp_lg_u32 s2, 0
1492; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
1493; SI-NEXT:    s_cbranch_scc0 BB14_2
1494; SI-NEXT:  ; %bb.1: ; %else
1495; SI-NEXT:    s_mov_b32 s11, 0xf000
1496; SI-NEXT:    s_mov_b32 s10, -1
1497; SI-NEXT:    s_waitcnt lgkmcnt(0)
1498; SI-NEXT:    s_mov_b32 s8, s2
1499; SI-NEXT:    s_mov_b32 s9, s3
1500; SI-NEXT:    buffer_load_ushort v0, off, s[8:11], 0 offset:2
1501; SI-NEXT:    s_mov_b64 s[2:3], 0
1502; SI-NEXT:    s_cbranch_execz BB14_3
1503; SI-NEXT:    s_branch BB14_4
1504; SI-NEXT:  BB14_2:
1505; SI-NEXT:    s_waitcnt lgkmcnt(0)
1506; SI-NEXT:    s_mov_b64 s[2:3], -1
1507; SI-NEXT:    v_mov_b32_e32 v0, 0
1508; SI-NEXT:  BB14_3: ; %if
1509; SI-NEXT:    s_and_b32 s2, s4, 0xffff
1510; SI-NEXT:    s_bcnt1_i32_b32 s2, s2
1511; SI-NEXT:    s_waitcnt vmcnt(0)
1512; SI-NEXT:    v_mov_b32_e32 v0, s2
1513; SI-NEXT:  BB14_4: ; %endif
1514; SI-NEXT:    s_mov_b32 s3, 0xf000
1515; SI-NEXT:    s_mov_b32 s2, -1
1516; SI-NEXT:    s_waitcnt vmcnt(0)
1517; SI-NEXT:    buffer_store_short v0, off, s[0:3], 0
1518; SI-NEXT:    s_endpgm
1519;
1520; VI-LABEL: ctpop_i16_in_br:
1521; VI:       ; %bb.0: ; %entry
1522; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
1523; VI-NEXT:    s_load_dword s0, s[0:1], 0x34
1524; VI-NEXT:    s_waitcnt lgkmcnt(0)
1525; VI-NEXT:    s_lshr_b32 s1, s0, 16
1526; VI-NEXT:    v_cmp_ne_u16_e64 s[2:3], s1, 0
1527; VI-NEXT:    s_and_b64 vcc, exec, s[2:3]
1528; VI-NEXT:    s_cbranch_vccz BB14_2
1529; VI-NEXT:  ; %bb.1: ; %else
1530; VI-NEXT:    s_mov_b32 s11, 0xf000
1531; VI-NEXT:    s_mov_b32 s10, -1
1532; VI-NEXT:    s_mov_b32 s8, s6
1533; VI-NEXT:    s_mov_b32 s9, s7
1534; VI-NEXT:    buffer_load_ushort v0, off, s[8:11], 0 offset:2
1535; VI-NEXT:    s_cbranch_execz BB14_3
1536; VI-NEXT:    s_branch BB14_4
1537; VI-NEXT:  BB14_2:
1538; VI-NEXT:    ; implicit-def: $vgpr0
1539; VI-NEXT:  BB14_3: ; %if
1540; VI-NEXT:    s_and_b32 s0, s0, 0xffff
1541; VI-NEXT:    s_bcnt1_i32_b32 s0, s0
1542; VI-NEXT:    s_waitcnt vmcnt(0)
1543; VI-NEXT:    v_mov_b32_e32 v0, s0
1544; VI-NEXT:  BB14_4: ; %endif
1545; VI-NEXT:    s_mov_b32 s7, 0xf000
1546; VI-NEXT:    s_mov_b32 s6, -1
1547; VI-NEXT:    s_waitcnt vmcnt(0)
1548; VI-NEXT:    buffer_store_short v0, off, s[4:7], 0
1549; VI-NEXT:    s_endpgm
1550;
1551; EG-LABEL: ctpop_i16_in_br:
1552; EG:       ; %bb.0: ; %entry
1553; EG-NEXT:    ALU 0, @20, KC0[], KC1[]
1554; EG-NEXT:    TEX 0 @14
1555; EG-NEXT:    ALU_PUSH_BEFORE 6, @21, KC0[], KC1[]
1556; EG-NEXT:    JUMP @7 POP:1
1557; EG-NEXT:    ALU 0, @28, KC0[CB0:0-32], KC1[]
1558; EG-NEXT:    TEX 0 @16
1559; EG-NEXT:    ALU_POP_AFTER 1, @29, KC0[], KC1[]
1560; EG-NEXT:    ALU_PUSH_BEFORE 2, @31, KC0[CB0:0-32], KC1[]
1561; EG-NEXT:    JUMP @11 POP:1
1562; EG-NEXT:    TEX 0 @18
1563; EG-NEXT:    ALU_POP_AFTER 0, @34, KC0[], KC1[]
1564; EG-NEXT:    ALU 11, @35, KC0[], KC1[]
1565; EG-NEXT:    MEM_RAT MSKOR T1.XW, T0.X
1566; EG-NEXT:    CF_END
1567; EG-NEXT:    Fetch clause starting at 14:
1568; EG-NEXT:     VTX_READ_16 T1.X, T0.X, 46, #3
1569; EG-NEXT:    Fetch clause starting at 16:
1570; EG-NEXT:     VTX_READ_16 T1.X, T1.X, 2, #1
1571; EG-NEXT:    Fetch clause starting at 18:
1572; EG-NEXT:     VTX_READ_16 T0.X, T0.X, 44, #3
1573; EG-NEXT:    ALU clause starting at 20:
1574; EG-NEXT:     MOV * T0.X, 0.0,
1575; EG-NEXT:    ALU clause starting at 21:
1576; EG-NEXT:     AND_INT * T0.W, T1.X, literal.x,
1577; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
1578; EG-NEXT:     MOV T1.X, literal.x,
1579; EG-NEXT:     MOV T1.W, literal.y,
1580; EG-NEXT:     SETNE_INT * T0.W, PV.W, 0.0,
1581; EG-NEXT:    0(0.000000e+00), 1(1.401298e-45)
1582; EG-NEXT:     PRED_SETNE_INT * ExecMask,PredicateBit (MASKED), PS, 0.0,
1583; EG-NEXT:    ALU clause starting at 28:
1584; EG-NEXT:     MOV * T1.X, KC0[2].Z,
1585; EG-NEXT:    ALU clause starting at 29:
1586; EG-NEXT:     MOV * T1.W, literal.x,
1587; EG-NEXT:    0(0.000000e+00), 0(0.000000e+00)
1588; EG-NEXT:    ALU clause starting at 31:
1589; EG-NEXT:     MOV T0.W, KC0[2].Y,
1590; EG-NEXT:     SETE_INT * T1.W, T1.W, 0.0,
1591; EG-NEXT:     PRED_SETE_INT * ExecMask,PredicateBit (MASKED), PS, 0.0,
1592; EG-NEXT:    ALU clause starting at 34:
1593; EG-NEXT:     BCNT_INT * T1.X, T0.X,
1594; EG-NEXT:    ALU clause starting at 35:
1595; EG-NEXT:     LSHL * T1.W, T0.W, literal.x,
1596; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
1597; EG-NEXT:     AND_INT T1.W, PV.W, literal.x,
1598; EG-NEXT:     AND_INT * T2.W, T1.X, literal.y,
1599; EG-NEXT:    24(3.363116e-44), 65535(9.183409e-41)
1600; EG-NEXT:     LSHL T1.X, PS, PV.W,
1601; EG-NEXT:     LSHL * T1.W, literal.x, PV.W,
1602; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
1603; EG-NEXT:     MOV T1.Y, 0.0,
1604; EG-NEXT:     MOV * T1.Z, 0.0,
1605; EG-NEXT:     LSHR * T0.X, T0.W, literal.x,
1606; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
1607entry:
1608  %tmp0 = icmp eq i16 %cond, 0
1609  br i1 %tmp0, label %if, label %else
1610
1611if:
1612  %tmp2 = call i16 @llvm.ctpop.i16(i16 %ctpop_arg)
1613  br label %endif
1614
1615else:
1616  %tmp3 = getelementptr i16, i16 addrspace(1)* %in, i16 1
1617  %tmp4 = load i16, i16 addrspace(1)* %tmp3
1618  br label %endif
1619
1620endif:
1621  %tmp5 = phi i16 [%tmp2, %if], [%tmp4, %else]
1622  store i16 %tmp5, i16 addrspace(1)* %out
1623  ret void
1624}
1625