1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc < %s -march=amdgcn -verify-machineinstrs | FileCheck %s -enable-var-scope --check-prefix=SI
3; RUN: llc < %s -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs | FileCheck %s -enable-var-scope --check-prefix=VI
4; RUN: llc < %s -march=r600 -mcpu=cypress -verify-machineinstrs | FileCheck %s -enable-var-scope --check-prefix=EG
5; RUN: llc < %s -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs | FileCheck %s -enable-var-scope --check-prefix=GFX10
6; RUN: llc < %s -global-isel -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs | FileCheck %s -enable-var-scope --check-prefix=GFX10-GISEL
7
8declare i7 @llvm.cttz.i7(i7, i1) nounwind readnone
9declare i8 @llvm.cttz.i8(i8, i1) nounwind readnone
10declare i16 @llvm.cttz.i16(i16, i1) nounwind readnone
11
12declare i32 @llvm.cttz.i32(i32, i1) nounwind readnone
13declare <2 x i32> @llvm.cttz.v2i32(<2 x i32>, i1) nounwind readnone
14declare <4 x i32> @llvm.cttz.v4i32(<4 x i32>, i1) nounwind readnone
15
16declare i64 @llvm.cttz.i64(i64, i1) nounwind readnone
17declare <2 x i64> @llvm.cttz.v2i64(<2 x i64>, i1) nounwind readnone
18declare <4 x i64> @llvm.cttz.v4i64(<4 x i64>, i1) nounwind readnone
19
20declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
21
22define amdgpu_kernel void @s_cttz_i32(i32 addrspace(1)* noalias %out, i32 %val) nounwind {
23; SI-LABEL: s_cttz_i32:
24; SI:       ; %bb.0:
25; SI-NEXT:    s_load_dword s4, s[0:1], 0xb
26; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
27; SI-NEXT:    s_mov_b32 s3, 0xf000
28; SI-NEXT:    s_waitcnt lgkmcnt(0)
29; SI-NEXT:    s_ff1_i32_b32 s5, s4
30; SI-NEXT:    s_mov_b32 s2, -1
31; SI-NEXT:    v_mov_b32_e32 v0, s5
32; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 0
33; SI-NEXT:    v_cndmask_b32_e32 v0, 32, v0, vcc
34; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
35; SI-NEXT:    s_endpgm
36;
37; VI-LABEL: s_cttz_i32:
38; VI:       ; %bb.0:
39; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
40; VI-NEXT:    s_load_dword s0, s[0:1], 0x2c
41; VI-NEXT:    s_mov_b32 s7, 0xf000
42; VI-NEXT:    s_mov_b32 s6, -1
43; VI-NEXT:    s_waitcnt lgkmcnt(0)
44; VI-NEXT:    s_ff1_i32_b32 s1, s0
45; VI-NEXT:    s_cmp_lg_u32 s0, 0
46; VI-NEXT:    s_cselect_b32 s0, s1, 32
47; VI-NEXT:    v_mov_b32_e32 v0, s0
48; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
49; VI-NEXT:    s_endpgm
50;
51; EG-LABEL: s_cttz_i32:
52; EG:       ; %bb.0:
53; EG-NEXT:    ALU 3, @4, KC0[CB0:0-32], KC1[]
54; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
55; EG-NEXT:    CF_END
56; EG-NEXT:    PAD
57; EG-NEXT:    ALU clause starting at 4:
58; EG-NEXT:     FFBL_INT * T0.W, KC0[2].Z,
59; EG-NEXT:     CNDE_INT T0.X, KC0[2].Z, literal.x, PV.W,
60; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.y,
61; EG-NEXT:    32(4.484155e-44), 2(2.802597e-45)
62;
63; GFX10-LABEL: s_cttz_i32:
64; GFX10:       ; %bb.0:
65; GFX10-NEXT:    s_clause 0x1
66; GFX10-NEXT:    s_load_dword s4, s[0:1], 0x2c
67; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
68; GFX10-NEXT:    v_mov_b32_e32 v0, 0
69; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
70; GFX10-NEXT:    s_ff1_i32_b32 s0, s4
71; GFX10-NEXT:    s_cmp_lg_u32 s4, 0
72; GFX10-NEXT:    s_cselect_b32 s0, s0, 32
73; GFX10-NEXT:    v_mov_b32_e32 v1, s0
74; GFX10-NEXT:    global_store_dword v0, v1, s[2:3]
75; GFX10-NEXT:    s_endpgm
76;
77; GFX10-GISEL-LABEL: s_cttz_i32:
78; GFX10-GISEL:       ; %bb.0:
79; GFX10-GISEL-NEXT:    s_clause 0x1
80; GFX10-GISEL-NEXT:    s_load_dword s4, s[0:1], 0x2c
81; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
82; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, 0
83; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
84; GFX10-GISEL-NEXT:    s_ff1_i32_b32 s0, s4
85; GFX10-GISEL-NEXT:    s_cmp_eq_u32 s4, 0
86; GFX10-GISEL-NEXT:    s_cselect_b32 s0, 32, s0
87; GFX10-GISEL-NEXT:    v_mov_b32_e32 v0, s0
88; GFX10-GISEL-NEXT:    global_store_dword v1, v0, s[2:3]
89; GFX10-GISEL-NEXT:    s_endpgm
90  %cttz = call i32 @llvm.cttz.i32(i32 %val, i1 false) nounwind readnone
91  store i32 %cttz, i32 addrspace(1)* %out, align 4
92  ret void
93}
94
95define amdgpu_kernel void @v_cttz_i32(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalias %valptr) nounwind {
96; SI-LABEL: v_cttz_i32:
97; SI:       ; %bb.0:
98; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xb
99; SI-NEXT:    s_mov_b32 s3, 0xf000
100; SI-NEXT:    s_mov_b32 s6, 0
101; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
102; SI-NEXT:    v_mov_b32_e32 v1, 0
103; SI-NEXT:    s_mov_b32 s7, s3
104; SI-NEXT:    s_waitcnt lgkmcnt(0)
105; SI-NEXT:    buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
106; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
107; SI-NEXT:    s_mov_b32 s2, -1
108; SI-NEXT:    s_waitcnt vmcnt(0)
109; SI-NEXT:    v_ffbl_b32_e32 v1, v0
110; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
111; SI-NEXT:    v_cndmask_b32_e32 v0, 32, v1, vcc
112; SI-NEXT:    s_waitcnt lgkmcnt(0)
113; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
114; SI-NEXT:    s_endpgm
115;
116; VI-LABEL: v_cttz_i32:
117; VI:       ; %bb.0:
118; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
119; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
120; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
121; VI-NEXT:    s_mov_b32 s7, 0xf000
122; VI-NEXT:    s_mov_b32 s6, -1
123; VI-NEXT:    s_waitcnt lgkmcnt(0)
124; VI-NEXT:    v_mov_b32_e32 v1, s1
125; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
126; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
127; VI-NEXT:    flat_load_dword v0, v[0:1]
128; VI-NEXT:    s_waitcnt vmcnt(0)
129; VI-NEXT:    v_ffbl_b32_e32 v1, v0
130; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
131; VI-NEXT:    v_cndmask_b32_e32 v0, 32, v1, vcc
132; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
133; VI-NEXT:    s_endpgm
134;
135; EG-LABEL: v_cttz_i32:
136; EG:       ; %bb.0:
137; EG-NEXT:    ALU 2, @8, KC0[CB0:0-32], KC1[]
138; EG-NEXT:    TEX 0 @6
139; EG-NEXT:    ALU 3, @11, KC0[CB0:0-32], KC1[]
140; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
141; EG-NEXT:    CF_END
142; EG-NEXT:    PAD
143; EG-NEXT:    Fetch clause starting at 6:
144; EG-NEXT:     VTX_READ_32 T0.X, T0.X, 0, #1
145; EG-NEXT:    ALU clause starting at 8:
146; EG-NEXT:     LSHL * T0.W, T0.X, literal.x,
147; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
148; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, PV.W,
149; EG-NEXT:    ALU clause starting at 11:
150; EG-NEXT:     FFBL_INT * T0.W, T0.X,
151; EG-NEXT:     CNDE_INT T0.X, T0.X, literal.x, PV.W,
152; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.y,
153; EG-NEXT:    32(4.484155e-44), 2(2.802597e-45)
154;
155; GFX10-LABEL: v_cttz_i32:
156; GFX10:       ; %bb.0:
157; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
158; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
159; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
160; GFX10-NEXT:    v_mov_b32_e32 v2, 0
161; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
162; GFX10-NEXT:    global_load_dword v0, v0, s[2:3]
163; GFX10-NEXT:    s_waitcnt vmcnt(0)
164; GFX10-NEXT:    v_ffbl_b32_e32 v1, v0
165; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
166; GFX10-NEXT:    v_cndmask_b32_e32 v0, 32, v1, vcc_lo
167; GFX10-NEXT:    global_store_dword v2, v0, s[0:1]
168; GFX10-NEXT:    s_endpgm
169;
170; GFX10-GISEL-LABEL: v_cttz_i32:
171; GFX10-GISEL:       ; %bb.0:
172; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
173; GFX10-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
174; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
175; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
176; GFX10-GISEL-NEXT:    global_load_dword v0, v0, s[2:3]
177; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0)
178; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v1, v0
179; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
180; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v0, v1, 32, vcc_lo
181; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, 0
182; GFX10-GISEL-NEXT:    global_store_dword v1, v0, s[0:1]
183; GFX10-GISEL-NEXT:    s_endpgm
184  %tid = call i32 @llvm.amdgcn.workitem.id.x()
185  %in.gep = getelementptr i32, i32 addrspace(1)* %valptr, i32 %tid
186  %val = load i32, i32 addrspace(1)* %in.gep, align 4
187  %cttz = call i32 @llvm.cttz.i32(i32 %val, i1 false) nounwind readnone
188  store i32 %cttz, i32 addrspace(1)* %out, align 4
189  ret void
190}
191
192define amdgpu_kernel void @v_cttz_v2i32(<2 x i32> addrspace(1)* noalias %out, <2 x i32> addrspace(1)* noalias %valptr) nounwind {
193; SI-LABEL: v_cttz_v2i32:
194; SI:       ; %bb.0:
195; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xb
196; SI-NEXT:    s_mov_b32 s3, 0xf000
197; SI-NEXT:    s_mov_b32 s6, 0
198; SI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
199; SI-NEXT:    v_mov_b32_e32 v1, 0
200; SI-NEXT:    s_mov_b32 s7, s3
201; SI-NEXT:    s_waitcnt lgkmcnt(0)
202; SI-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
203; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
204; SI-NEXT:    s_mov_b32 s2, -1
205; SI-NEXT:    s_waitcnt vmcnt(0)
206; SI-NEXT:    v_ffbl_b32_e32 v2, v1
207; SI-NEXT:    v_ffbl_b32_e32 v3, v0
208; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
209; SI-NEXT:    v_cndmask_b32_e32 v1, 32, v2, vcc
210; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
211; SI-NEXT:    v_cndmask_b32_e32 v0, 32, v3, vcc
212; SI-NEXT:    s_waitcnt lgkmcnt(0)
213; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
214; SI-NEXT:    s_endpgm
215;
216; VI-LABEL: v_cttz_v2i32:
217; VI:       ; %bb.0:
218; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
219; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
220; VI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
221; VI-NEXT:    s_mov_b32 s7, 0xf000
222; VI-NEXT:    s_mov_b32 s6, -1
223; VI-NEXT:    s_waitcnt lgkmcnt(0)
224; VI-NEXT:    v_mov_b32_e32 v1, s1
225; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
226; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
227; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
228; VI-NEXT:    s_waitcnt vmcnt(0)
229; VI-NEXT:    v_ffbl_b32_e32 v2, v1
230; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
231; VI-NEXT:    v_cndmask_b32_e32 v1, 32, v2, vcc
232; VI-NEXT:    v_ffbl_b32_e32 v3, v0
233; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
234; VI-NEXT:    v_cndmask_b32_e32 v0, 32, v3, vcc
235; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
236; VI-NEXT:    s_endpgm
237;
238; EG-LABEL: v_cttz_v2i32:
239; EG:       ; %bb.0:
240; EG-NEXT:    ALU 2, @8, KC0[CB0:0-32], KC1[]
241; EG-NEXT:    TEX 0 @6
242; EG-NEXT:    ALU 6, @11, KC0[CB0:0-32], KC1[]
243; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
244; EG-NEXT:    CF_END
245; EG-NEXT:    PAD
246; EG-NEXT:    Fetch clause starting at 6:
247; EG-NEXT:     VTX_READ_64 T0.XY, T0.X, 0, #1
248; EG-NEXT:    ALU clause starting at 8:
249; EG-NEXT:     LSHL * T0.W, T0.X, literal.x,
250; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
251; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, PV.W,
252; EG-NEXT:    ALU clause starting at 11:
253; EG-NEXT:     FFBL_INT * T0.W, T0.Y,
254; EG-NEXT:     CNDE_INT T0.Y, T0.Y, literal.x, PV.W,
255; EG-NEXT:     FFBL_INT * T0.W, T0.X,
256; EG-NEXT:    32(4.484155e-44), 0(0.000000e+00)
257; EG-NEXT:     CNDE_INT T0.X, T0.X, literal.x, PV.W,
258; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.y,
259; EG-NEXT:    32(4.484155e-44), 2(2.802597e-45)
260;
261; GFX10-LABEL: v_cttz_v2i32:
262; GFX10:       ; %bb.0:
263; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
264; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
265; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
266; GFX10-NEXT:    v_mov_b32_e32 v4, 0
267; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
268; GFX10-NEXT:    global_load_dwordx2 v[0:1], v0, s[2:3]
269; GFX10-NEXT:    s_waitcnt vmcnt(0)
270; GFX10-NEXT:    v_ffbl_b32_e32 v2, v1
271; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v1
272; GFX10-NEXT:    v_ffbl_b32_e32 v3, v0
273; GFX10-NEXT:    v_cndmask_b32_e32 v1, 32, v2, vcc_lo
274; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
275; GFX10-NEXT:    v_cndmask_b32_e32 v0, 32, v3, vcc_lo
276; GFX10-NEXT:    global_store_dwordx2 v4, v[0:1], s[0:1]
277; GFX10-NEXT:    s_endpgm
278;
279; GFX10-GISEL-LABEL: v_cttz_v2i32:
280; GFX10-GISEL:       ; %bb.0:
281; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
282; GFX10-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
283; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
284; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
285; GFX10-GISEL-NEXT:    global_load_dwordx2 v[0:1], v0, s[2:3]
286; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0)
287; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v2, v0
288; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
289; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v3, v1
290; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v0, v2, 32, vcc_lo
291; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
292; GFX10-GISEL-NEXT:    v_mov_b32_e32 v2, 0
293; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v1, v3, 32, vcc_lo
294; GFX10-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
295; GFX10-GISEL-NEXT:    s_endpgm
296  %tid = call i32 @llvm.amdgcn.workitem.id.x()
297  %in.gep = getelementptr <2 x i32>, <2 x i32> addrspace(1)* %valptr, i32 %tid
298  %val = load <2 x i32>, <2 x i32> addrspace(1)* %in.gep, align 8
299  %cttz = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %val, i1 false) nounwind readnone
300  store <2 x i32> %cttz, <2 x i32> addrspace(1)* %out, align 8
301  ret void
302}
303
304define amdgpu_kernel void @v_cttz_v4i32(<4 x i32> addrspace(1)* noalias %out, <4 x i32> addrspace(1)* noalias %valptr) nounwind {
305; SI-LABEL: v_cttz_v4i32:
306; SI:       ; %bb.0:
307; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xb
308; SI-NEXT:    s_mov_b32 s3, 0xf000
309; SI-NEXT:    s_mov_b32 s6, 0
310; SI-NEXT:    v_lshlrev_b32_e32 v0, 4, v0
311; SI-NEXT:    v_mov_b32_e32 v1, 0
312; SI-NEXT:    s_mov_b32 s7, s3
313; SI-NEXT:    s_waitcnt lgkmcnt(0)
314; SI-NEXT:    buffer_load_dwordx4 v[0:3], v[0:1], s[4:7], 0 addr64
315; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
316; SI-NEXT:    s_mov_b32 s2, -1
317; SI-NEXT:    s_waitcnt vmcnt(0)
318; SI-NEXT:    v_ffbl_b32_e32 v4, v3
319; SI-NEXT:    v_ffbl_b32_e32 v5, v2
320; SI-NEXT:    v_ffbl_b32_e32 v6, v1
321; SI-NEXT:    v_ffbl_b32_e32 v7, v0
322; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v3
323; SI-NEXT:    v_cndmask_b32_e32 v3, 32, v4, vcc
324; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
325; SI-NEXT:    v_cndmask_b32_e32 v2, 32, v5, vcc
326; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
327; SI-NEXT:    v_cndmask_b32_e32 v1, 32, v6, vcc
328; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
329; SI-NEXT:    v_cndmask_b32_e32 v0, 32, v7, vcc
330; SI-NEXT:    s_waitcnt lgkmcnt(0)
331; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
332; SI-NEXT:    s_endpgm
333;
334; VI-LABEL: v_cttz_v4i32:
335; VI:       ; %bb.0:
336; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
337; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
338; VI-NEXT:    v_lshlrev_b32_e32 v0, 4, v0
339; VI-NEXT:    s_mov_b32 s7, 0xf000
340; VI-NEXT:    s_mov_b32 s6, -1
341; VI-NEXT:    s_waitcnt lgkmcnt(0)
342; VI-NEXT:    v_mov_b32_e32 v1, s1
343; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
344; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
345; VI-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
346; VI-NEXT:    s_waitcnt vmcnt(0)
347; VI-NEXT:    v_ffbl_b32_e32 v4, v3
348; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v3
349; VI-NEXT:    v_cndmask_b32_e32 v3, 32, v4, vcc
350; VI-NEXT:    v_ffbl_b32_e32 v5, v2
351; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
352; VI-NEXT:    v_cndmask_b32_e32 v2, 32, v5, vcc
353; VI-NEXT:    v_ffbl_b32_e32 v6, v1
354; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
355; VI-NEXT:    v_cndmask_b32_e32 v1, 32, v6, vcc
356; VI-NEXT:    v_ffbl_b32_e32 v7, v0
357; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
358; VI-NEXT:    v_cndmask_b32_e32 v0, 32, v7, vcc
359; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
360; VI-NEXT:    s_endpgm
361;
362; EG-LABEL: v_cttz_v4i32:
363; EG:       ; %bb.0:
364; EG-NEXT:    ALU 2, @8, KC0[CB0:0-32], KC1[]
365; EG-NEXT:    TEX 0 @6
366; EG-NEXT:    ALU 12, @11, KC0[CB0:0-32], KC1[]
367; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XYZW, T1.X, 1
368; EG-NEXT:    CF_END
369; EG-NEXT:    PAD
370; EG-NEXT:    Fetch clause starting at 6:
371; EG-NEXT:     VTX_READ_128 T0.XYZW, T0.X, 0, #1
372; EG-NEXT:    ALU clause starting at 8:
373; EG-NEXT:     LSHL * T0.W, T0.X, literal.x,
374; EG-NEXT:    4(5.605194e-45), 0(0.000000e+00)
375; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, PV.W,
376; EG-NEXT:    ALU clause starting at 11:
377; EG-NEXT:     FFBL_INT * T1.W, T0.W,
378; EG-NEXT:     FFBL_INT T2.W, T0.Z,
379; EG-NEXT:     CNDE_INT * T0.W, T0.W, literal.x, PV.W, BS:VEC_021/SCL_122
380; EG-NEXT:    32(4.484155e-44), 0(0.000000e+00)
381; EG-NEXT:     CNDE_INT T0.Z, T0.Z, literal.x, PV.W,
382; EG-NEXT:     FFBL_INT * T1.W, T0.Y,
383; EG-NEXT:    32(4.484155e-44), 0(0.000000e+00)
384; EG-NEXT:     CNDE_INT T0.Y, T0.Y, literal.x, PV.W,
385; EG-NEXT:     FFBL_INT * T1.W, T0.X,
386; EG-NEXT:    32(4.484155e-44), 0(0.000000e+00)
387; EG-NEXT:     CNDE_INT T0.X, T0.X, literal.x, PV.W,
388; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.y,
389; EG-NEXT:    32(4.484155e-44), 2(2.802597e-45)
390;
391; GFX10-LABEL: v_cttz_v4i32:
392; GFX10:       ; %bb.0:
393; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
394; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 4, v0
395; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
396; GFX10-NEXT:    v_mov_b32_e32 v4, 0
397; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
398; GFX10-NEXT:    global_load_dwordx4 v[0:3], v0, s[2:3]
399; GFX10-NEXT:    s_waitcnt vmcnt(0)
400; GFX10-NEXT:    v_ffbl_b32_e32 v5, v3
401; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v3
402; GFX10-NEXT:    v_ffbl_b32_e32 v6, v2
403; GFX10-NEXT:    v_ffbl_b32_e32 v7, v1
404; GFX10-NEXT:    v_ffbl_b32_e32 v8, v0
405; GFX10-NEXT:    v_cndmask_b32_e32 v3, 32, v5, vcc_lo
406; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v2
407; GFX10-NEXT:    v_cndmask_b32_e32 v2, 32, v6, vcc_lo
408; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v1
409; GFX10-NEXT:    v_cndmask_b32_e32 v1, 32, v7, vcc_lo
410; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
411; GFX10-NEXT:    v_cndmask_b32_e32 v0, 32, v8, vcc_lo
412; GFX10-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
413; GFX10-NEXT:    s_endpgm
414;
415; GFX10-GISEL-LABEL: v_cttz_v4i32:
416; GFX10-GISEL:       ; %bb.0:
417; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
418; GFX10-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 4, v0
419; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
420; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
421; GFX10-GISEL-NEXT:    global_load_dwordx4 v[0:3], v0, s[2:3]
422; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0)
423; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v4, v0
424; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
425; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v5, v1
426; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v6, v2
427; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v7, v3
428; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v0, v4, 32, vcc_lo
429; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
430; GFX10-GISEL-NEXT:    v_mov_b32_e32 v4, 0
431; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v1, v5, 32, vcc_lo
432; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v2
433; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v2, v6, 32, vcc_lo
434; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
435; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v3, v7, 32, vcc_lo
436; GFX10-GISEL-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
437; GFX10-GISEL-NEXT:    s_endpgm
438  %tid = call i32 @llvm.amdgcn.workitem.id.x()
439  %in.gep = getelementptr <4 x i32>, <4 x i32> addrspace(1)* %valptr, i32 %tid
440  %val = load <4 x i32>, <4 x i32> addrspace(1)* %in.gep, align 16
441  %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %val, i1 false) nounwind readnone
442  store <4 x i32> %cttz, <4 x i32> addrspace(1)* %out, align 16
443  ret void
444}
445
446define amdgpu_kernel void @v_cttz_i8(i8 addrspace(1)* noalias %out, i8 addrspace(1)* noalias %valptr) nounwind {
447; SI-LABEL: v_cttz_i8:
448; SI:       ; %bb.0:
449; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xb
450; SI-NEXT:    s_mov_b32 s3, 0xf000
451; SI-NEXT:    s_mov_b32 s2, -1
452; SI-NEXT:    s_mov_b32 s6, s2
453; SI-NEXT:    s_mov_b32 s7, s3
454; SI-NEXT:    s_waitcnt lgkmcnt(0)
455; SI-NEXT:    buffer_load_ubyte v0, off, s[4:7], 0
456; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
457; SI-NEXT:    s_waitcnt vmcnt(0)
458; SI-NEXT:    v_or_b32_e32 v0, 0x100, v0
459; SI-NEXT:    v_ffbl_b32_e32 v0, v0
460; SI-NEXT:    s_waitcnt lgkmcnt(0)
461; SI-NEXT:    buffer_store_byte v0, off, s[0:3], 0
462; SI-NEXT:    s_endpgm
463;
464; VI-LABEL: v_cttz_i8:
465; VI:       ; %bb.0:
466; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
467; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
468; VI-NEXT:    s_mov_b32 s7, 0xf000
469; VI-NEXT:    s_mov_b32 s6, -1
470; VI-NEXT:    s_mov_b32 s2, s6
471; VI-NEXT:    s_mov_b32 s3, s7
472; VI-NEXT:    s_waitcnt lgkmcnt(0)
473; VI-NEXT:    buffer_load_ubyte v0, off, s[0:3], 0
474; VI-NEXT:    s_waitcnt vmcnt(0)
475; VI-NEXT:    v_or_b32_e32 v0, 0x100, v0
476; VI-NEXT:    v_ffbl_b32_e32 v0, v0
477; VI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
478; VI-NEXT:    s_endpgm
479;
480; EG-LABEL: v_cttz_i8:
481; EG:       ; %bb.0:
482; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
483; EG-NEXT:    TEX 0 @6
484; EG-NEXT:    ALU 14, @9, KC0[CB0:0-32], KC1[]
485; EG-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
486; EG-NEXT:    CF_END
487; EG-NEXT:    PAD
488; EG-NEXT:    Fetch clause starting at 6:
489; EG-NEXT:     VTX_READ_8 T0.X, T0.X, 0, #1
490; EG-NEXT:    ALU clause starting at 8:
491; EG-NEXT:     MOV * T0.X, KC0[2].Z,
492; EG-NEXT:    ALU clause starting at 9:
493; EG-NEXT:     OR_INT * T0.W, T0.X, literal.x,
494; EG-NEXT:    256(3.587324e-43), 0(0.000000e+00)
495; EG-NEXT:     FFBL_INT T0.W, PV.W,
496; EG-NEXT:     AND_INT * T1.W, KC0[2].Y, literal.x,
497; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
498; EG-NEXT:     AND_INT T0.W, PV.W, literal.x,
499; EG-NEXT:     LSHL * T1.W, PS, literal.y,
500; EG-NEXT:    255(3.573311e-43), 3(4.203895e-45)
501; EG-NEXT:     LSHL T0.X, PV.W, PS,
502; EG-NEXT:     LSHL * T0.W, literal.x, PS,
503; EG-NEXT:    255(3.573311e-43), 0(0.000000e+00)
504; EG-NEXT:     MOV T0.Y, 0.0,
505; EG-NEXT:     MOV * T0.Z, 0.0,
506; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
507; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
508;
509; GFX10-LABEL: v_cttz_i8:
510; GFX10:       ; %bb.0:
511; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
512; GFX10-NEXT:    v_mov_b32_e32 v0, 0
513; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
514; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
515; GFX10-NEXT:    global_load_ubyte v1, v0, s[2:3]
516; GFX10-NEXT:    s_waitcnt vmcnt(0)
517; GFX10-NEXT:    v_or_b32_e32 v1, 0x100, v1
518; GFX10-NEXT:    v_ffbl_b32_e32 v1, v1
519; GFX10-NEXT:    global_store_byte v0, v1, s[0:1]
520; GFX10-NEXT:    s_endpgm
521;
522; GFX10-GISEL-LABEL: v_cttz_i8:
523; GFX10-GISEL:       ; %bb.0:
524; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
525; GFX10-GISEL-NEXT:    v_mov_b32_e32 v0, 0
526; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
527; GFX10-GISEL-NEXT:    v_mov_b32_e32 v2, 0x100
528; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
529; GFX10-GISEL-NEXT:    global_load_ubyte v1, v0, s[2:3]
530; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0)
531; GFX10-GISEL-NEXT:    v_and_or_b32 v1, 0xff, v1, v2
532; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v1, v1
533; GFX10-GISEL-NEXT:    global_store_byte v0, v1, s[0:1]
534; GFX10-GISEL-NEXT:    s_endpgm
535  %val = load i8, i8 addrspace(1)* %valptr
536  %cttz = call i8 @llvm.cttz.i8(i8 %val, i1 false) nounwind readnone
537  store i8 %cttz, i8 addrspace(1)* %out
538  ret void
539}
540
541define amdgpu_kernel void @s_cttz_i64(i64 addrspace(1)* noalias %out, [8 x i32], i64 %val) nounwind {
542; SI-LABEL: s_cttz_i64:
543; SI:       ; %bb.0:
544; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x13
545; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
546; SI-NEXT:    s_mov_b32 s3, 0xf000
547; SI-NEXT:    s_mov_b32 s2, -1
548; SI-NEXT:    s_waitcnt lgkmcnt(0)
549; SI-NEXT:    s_ff1_i32_b32 s5, s5
550; SI-NEXT:    s_min_u32 s5, s5, 0xffffffdf
551; SI-NEXT:    s_add_i32 s5, s5, 32
552; SI-NEXT:    s_ff1_i32_b32 s4, s4
553; SI-NEXT:    v_mov_b32_e32 v0, s5
554; SI-NEXT:    v_min3_u32 v0, s4, v0, 64
555; SI-NEXT:    v_mov_b32_e32 v1, 0
556; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
557; SI-NEXT:    s_endpgm
558;
559; VI-LABEL: s_cttz_i64:
560; VI:       ; %bb.0:
561; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
562; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x4c
563; VI-NEXT:    s_mov_b32 s7, 0xf000
564; VI-NEXT:    s_mov_b32 s6, -1
565; VI-NEXT:    v_mov_b32_e32 v1, 0
566; VI-NEXT:    s_waitcnt lgkmcnt(0)
567; VI-NEXT:    s_ff1_i32_b32 s1, s1
568; VI-NEXT:    v_add_u32_e64 v0, s[2:3], s1, 32 clamp
569; VI-NEXT:    s_ff1_i32_b32 s0, s0
570; VI-NEXT:    v_min3_u32 v0, s0, v0, 64
571; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
572; VI-NEXT:    s_endpgm
573;
574; EG-LABEL: s_cttz_i64:
575; EG:       ; %bb.0:
576; EG-NEXT:    ALU 9, @4, KC0[CB0:0-32], KC1[]
577; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
578; EG-NEXT:    CF_END
579; EG-NEXT:    PAD
580; EG-NEXT:    ALU clause starting at 4:
581; EG-NEXT:     FFBL_INT * T0.W, KC0[5].X,
582; EG-NEXT:     CNDE_INT * T0.W, KC0[5].X, literal.x, PV.W,
583; EG-NEXT:    32(4.484155e-44), 0(0.000000e+00)
584; EG-NEXT:     FFBL_INT T1.W, KC0[4].W,
585; EG-NEXT:     ADD_INT * T0.W, PV.W, literal.x,
586; EG-NEXT:    32(4.484155e-44), 0(0.000000e+00)
587; EG-NEXT:     CNDE_INT T0.X, KC0[4].W, PS, PV.W,
588; EG-NEXT:     MOV T0.Y, 0.0,
589; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
590; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
591;
592; GFX10-LABEL: s_cttz_i64:
593; GFX10:       ; %bb.0:
594; GFX10-NEXT:    s_clause 0x1
595; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x4c
596; GFX10-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
597; GFX10-NEXT:    v_mov_b32_e32 v1, 0
598; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
599; GFX10-NEXT:    s_ff1_i32_b32 s0, s3
600; GFX10-NEXT:    v_add_nc_u32_e64 v0, s0, 32 clamp
601; GFX10-NEXT:    s_ff1_i32_b32 s0, s2
602; GFX10-NEXT:    v_min3_u32 v0, s0, v0, 64
603; GFX10-NEXT:    global_store_dwordx2 v1, v[0:1], s[4:5]
604; GFX10-NEXT:    s_endpgm
605;
606; GFX10-GISEL-LABEL: s_cttz_i64:
607; GFX10-GISEL:       ; %bb.0:
608; GFX10-GISEL-NEXT:    s_clause 0x1
609; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x4c
610; GFX10-GISEL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
611; GFX10-GISEL-NEXT:    v_mov_b32_e32 v2, 0
612; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
613; GFX10-GISEL-NEXT:    s_ff1_i32_b64 s0, s[2:3]
614; GFX10-GISEL-NEXT:    s_cmp_eq_u64 s[2:3], 0
615; GFX10-GISEL-NEXT:    s_cselect_b32 s0, 64, s0
616; GFX10-GISEL-NEXT:    s_bfe_u64 s[0:1], s[0:1], 0x200000
617; GFX10-GISEL-NEXT:    v_mov_b32_e32 v0, s0
618; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, s1
619; GFX10-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[4:5]
620; GFX10-GISEL-NEXT:    s_endpgm
621  %cttz = call i64 @llvm.cttz.i64(i64 %val, i1 false)
622  store i64 %cttz, i64 addrspace(1)* %out
623  ret void
624}
625
626define amdgpu_kernel void @s_cttz_i64_trunc(i32 addrspace(1)* noalias %out, i64 %val) nounwind {
627; SI-LABEL: s_cttz_i64_trunc:
628; SI:       ; %bb.0:
629; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xb
630; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
631; SI-NEXT:    s_mov_b32 s3, 0xf000
632; SI-NEXT:    s_mov_b32 s2, -1
633; SI-NEXT:    s_waitcnt lgkmcnt(0)
634; SI-NEXT:    s_ff1_i32_b32 s5, s5
635; SI-NEXT:    s_min_u32 s5, s5, 0xffffffdf
636; SI-NEXT:    s_add_i32 s5, s5, 32
637; SI-NEXT:    s_ff1_i32_b32 s4, s4
638; SI-NEXT:    v_mov_b32_e32 v0, s5
639; SI-NEXT:    v_min3_u32 v0, s4, v0, 64
640; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
641; SI-NEXT:    s_endpgm
642;
643; VI-LABEL: s_cttz_i64_trunc:
644; VI:       ; %bb.0:
645; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
646; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
647; VI-NEXT:    s_mov_b32 s7, 0xf000
648; VI-NEXT:    s_mov_b32 s6, -1
649; VI-NEXT:    s_waitcnt lgkmcnt(0)
650; VI-NEXT:    s_ff1_i32_b32 s1, s1
651; VI-NEXT:    v_add_u32_e64 v0, s[2:3], s1, 32 clamp
652; VI-NEXT:    s_ff1_i32_b32 s0, s0
653; VI-NEXT:    v_min3_u32 v0, s0, v0, 64
654; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
655; VI-NEXT:    s_endpgm
656;
657; EG-LABEL: s_cttz_i64_trunc:
658; EG:       ; %bb.0:
659; EG-NEXT:    ALU 8, @4, KC0[CB0:0-32], KC1[]
660; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
661; EG-NEXT:    CF_END
662; EG-NEXT:    PAD
663; EG-NEXT:    ALU clause starting at 4:
664; EG-NEXT:     FFBL_INT * T0.W, KC0[3].X,
665; EG-NEXT:     CNDE_INT * T0.W, KC0[3].X, literal.x, PV.W,
666; EG-NEXT:    32(4.484155e-44), 0(0.000000e+00)
667; EG-NEXT:     FFBL_INT T1.W, KC0[2].W,
668; EG-NEXT:     ADD_INT * T0.W, PV.W, literal.x,
669; EG-NEXT:    32(4.484155e-44), 0(0.000000e+00)
670; EG-NEXT:     CNDE_INT T0.X, KC0[2].W, PS, PV.W,
671; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
672; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
673;
674; GFX10-LABEL: s_cttz_i64_trunc:
675; GFX10:       ; %bb.0:
676; GFX10-NEXT:    s_clause 0x1
677; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
678; GFX10-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
679; GFX10-NEXT:    v_mov_b32_e32 v1, 0
680; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
681; GFX10-NEXT:    s_ff1_i32_b32 s0, s3
682; GFX10-NEXT:    v_add_nc_u32_e64 v0, s0, 32 clamp
683; GFX10-NEXT:    s_ff1_i32_b32 s0, s2
684; GFX10-NEXT:    v_min3_u32 v0, s0, v0, 64
685; GFX10-NEXT:    global_store_dword v1, v0, s[4:5]
686; GFX10-NEXT:    s_endpgm
687;
688; GFX10-GISEL-LABEL: s_cttz_i64_trunc:
689; GFX10-GISEL:       ; %bb.0:
690; GFX10-GISEL-NEXT:    s_clause 0x1
691; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
692; GFX10-GISEL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
693; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, 0
694; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
695; GFX10-GISEL-NEXT:    s_ff1_i32_b64 s0, s[2:3]
696; GFX10-GISEL-NEXT:    s_cmp_eq_u64 s[2:3], 0
697; GFX10-GISEL-NEXT:    s_cselect_b32 s0, 64, s0
698; GFX10-GISEL-NEXT:    v_mov_b32_e32 v0, s0
699; GFX10-GISEL-NEXT:    global_store_dword v1, v0, s[4:5]
700; GFX10-GISEL-NEXT:    s_endpgm
701  %cttz = call i64 @llvm.cttz.i64(i64 %val, i1 false)
702  %trunc = trunc i64 %cttz to i32
703  store i32 %trunc, i32 addrspace(1)* %out
704  ret void
705}
706
707define amdgpu_kernel void @v_cttz_i64(i64 addrspace(1)* noalias %out, i64 addrspace(1)* noalias %in) nounwind {
708; SI-LABEL: v_cttz_i64:
709; SI:       ; %bb.0:
710; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xb
711; SI-NEXT:    s_mov_b32 s7, 0xf000
712; SI-NEXT:    s_mov_b32 s6, 0
713; SI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
714; SI-NEXT:    v_mov_b32_e32 v1, 0
715; SI-NEXT:    s_waitcnt lgkmcnt(0)
716; SI-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64
717; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
718; SI-NEXT:    s_waitcnt vmcnt(0)
719; SI-NEXT:    v_ffbl_b32_e32 v3, v3
720; SI-NEXT:    v_min_u32_e32 v3, 0xffffffdf, v3
721; SI-NEXT:    v_add_i32_e32 v3, vcc, 32, v3
722; SI-NEXT:    v_ffbl_b32_e32 v2, v2
723; SI-NEXT:    v_min3_u32 v2, v2, v3, 64
724; SI-NEXT:    v_mov_b32_e32 v3, v1
725; SI-NEXT:    s_waitcnt lgkmcnt(0)
726; SI-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64
727; SI-NEXT:    s_endpgm
728;
729; VI-LABEL: v_cttz_i64:
730; VI:       ; %bb.0:
731; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
732; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
733; VI-NEXT:    v_lshlrev_b32_e32 v3, 3, v0
734; VI-NEXT:    v_mov_b32_e32 v2, 0
735; VI-NEXT:    s_waitcnt lgkmcnt(0)
736; VI-NEXT:    v_mov_b32_e32 v4, s3
737; VI-NEXT:    v_mov_b32_e32 v1, s1
738; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v3
739; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
740; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
741; VI-NEXT:    v_add_u32_e32 v3, vcc, s2, v3
742; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
743; VI-NEXT:    s_waitcnt vmcnt(0)
744; VI-NEXT:    v_ffbl_b32_e32 v1, v1
745; VI-NEXT:    v_add_u32_e64 v1, s[0:1], v1, 32 clamp
746; VI-NEXT:    v_ffbl_b32_e32 v0, v0
747; VI-NEXT:    v_min3_u32 v1, v0, v1, 64
748; VI-NEXT:    flat_store_dwordx2 v[3:4], v[1:2]
749; VI-NEXT:    s_endpgm
750;
751; EG-LABEL: v_cttz_i64:
752; EG:       ; %bb.0:
753; EG-NEXT:    ALU 2, @8, KC0[CB0:0-32], KC1[]
754; EG-NEXT:    TEX 0 @6
755; EG-NEXT:    ALU 10, @11, KC0[CB0:0-32], KC1[]
756; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
757; EG-NEXT:    CF_END
758; EG-NEXT:    PAD
759; EG-NEXT:    Fetch clause starting at 6:
760; EG-NEXT:     VTX_READ_64 T0.XY, T0.X, 0, #1
761; EG-NEXT:    ALU clause starting at 8:
762; EG-NEXT:     LSHL * T0.W, T0.X, literal.x,
763; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
764; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, PV.W,
765; EG-NEXT:    ALU clause starting at 11:
766; EG-NEXT:     FFBL_INT * T1.W, T0.Y,
767; EG-NEXT:     CNDE_INT * T1.W, T0.Y, literal.x, PV.W,
768; EG-NEXT:    32(4.484155e-44), 0(0.000000e+00)
769; EG-NEXT:     FFBL_INT T2.W, T0.X,
770; EG-NEXT:     ADD_INT * T1.W, PV.W, literal.x,
771; EG-NEXT:    32(4.484155e-44), 0(0.000000e+00)
772; EG-NEXT:     CNDE_INT T0.X, T0.X, PS, PV.W,
773; EG-NEXT:     MOV T0.Y, 0.0,
774; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, T0.W,
775; EG-NEXT:     LSHR * T1.X, PV.W, literal.x,
776; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
777;
778; GFX10-LABEL: v_cttz_i64:
779; GFX10:       ; %bb.0:
780; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
781; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
782; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
783; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
784; GFX10-NEXT:    global_load_dwordx2 v[0:1], v2, s[2:3]
785; GFX10-NEXT:    s_waitcnt vmcnt(0)
786; GFX10-NEXT:    v_ffbl_b32_e32 v1, v1
787; GFX10-NEXT:    v_ffbl_b32_e32 v0, v0
788; GFX10-NEXT:    v_add_nc_u32_e64 v1, v1, 32 clamp
789; GFX10-NEXT:    v_min3_u32 v0, v0, v1, 64
790; GFX10-NEXT:    v_mov_b32_e32 v1, 0
791; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
792; GFX10-NEXT:    s_endpgm
793;
794; GFX10-GISEL-LABEL: v_cttz_i64:
795; GFX10-GISEL:       ; %bb.0:
796; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
797; GFX10-GISEL-NEXT:    v_lshlrev_b32_e32 v3, 3, v0
798; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
799; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
800; GFX10-GISEL-NEXT:    global_load_dwordx2 v[0:1], v3, s[2:3]
801; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0)
802; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v2, v1
803; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v4, v0
804; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
805; GFX10-GISEL-NEXT:    v_add_nc_u32_e32 v2, 32, v2
806; GFX10-GISEL-NEXT:    v_cndmask_b32_e32 v4, v4, v2, vcc_lo
807; GFX10-GISEL-NEXT:    v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]
808; GFX10-GISEL-NEXT:    v_mov_b32_e32 v2, 0
809; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v1, v4, 64, vcc_lo
810; GFX10-GISEL-NEXT:    global_store_dwordx2 v3, v[1:2], s[0:1]
811; GFX10-GISEL-NEXT:    s_endpgm
812  %tid = call i32 @llvm.amdgcn.workitem.id.x()
813  %in.gep = getelementptr i64, i64 addrspace(1)* %in, i32 %tid
814  %out.gep = getelementptr i64, i64 addrspace(1)* %out, i32 %tid
815  %val = load i64, i64 addrspace(1)* %in.gep
816  %cttz = call i64 @llvm.cttz.i64(i64 %val, i1 false)
817  store i64 %cttz, i64 addrspace(1)* %out.gep
818  ret void
819}
820
821define amdgpu_kernel void @v_cttz_i64_trunc(i32 addrspace(1)* noalias %out, i64 addrspace(1)* noalias %in) nounwind {
822; SI-LABEL: v_cttz_i64_trunc:
823; SI:       ; %bb.0:
824; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xb
825; SI-NEXT:    s_mov_b32 s7, 0xf000
826; SI-NEXT:    s_mov_b32 s6, 0
827; SI-NEXT:    v_lshlrev_b32_e32 v1, 3, v0
828; SI-NEXT:    v_mov_b32_e32 v2, 0
829; SI-NEXT:    s_waitcnt lgkmcnt(0)
830; SI-NEXT:    buffer_load_dwordx2 v[3:4], v[1:2], s[4:7], 0 addr64
831; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
832; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
833; SI-NEXT:    s_waitcnt vmcnt(0)
834; SI-NEXT:    v_ffbl_b32_e32 v0, v4
835; SI-NEXT:    v_min_u32_e32 v0, 0xffffffdf, v0
836; SI-NEXT:    v_add_i32_e32 v0, vcc, 32, v0
837; SI-NEXT:    v_ffbl_b32_e32 v3, v3
838; SI-NEXT:    v_min3_u32 v0, v3, v0, 64
839; SI-NEXT:    s_waitcnt lgkmcnt(0)
840; SI-NEXT:    buffer_store_dword v0, v[1:2], s[4:7], 0 addr64
841; SI-NEXT:    s_endpgm
842;
843; VI-LABEL: v_cttz_i64_trunc:
844; VI:       ; %bb.0:
845; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
846; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
847; VI-NEXT:    v_lshlrev_b32_e32 v1, 3, v0
848; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
849; VI-NEXT:    s_waitcnt lgkmcnt(0)
850; VI-NEXT:    v_mov_b32_e32 v4, s3
851; VI-NEXT:    v_mov_b32_e32 v2, s1
852; VI-NEXT:    v_add_u32_e32 v1, vcc, s0, v1
853; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
854; VI-NEXT:    flat_load_dwordx2 v[1:2], v[1:2]
855; VI-NEXT:    v_add_u32_e32 v3, vcc, s2, v0
856; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
857; VI-NEXT:    s_waitcnt vmcnt(0)
858; VI-NEXT:    v_ffbl_b32_e32 v0, v2
859; VI-NEXT:    v_add_u32_e64 v0, s[0:1], v0, 32 clamp
860; VI-NEXT:    v_ffbl_b32_e32 v1, v1
861; VI-NEXT:    v_min3_u32 v0, v1, v0, 64
862; VI-NEXT:    flat_store_dword v[3:4], v0
863; VI-NEXT:    s_endpgm
864;
865; EG-LABEL: v_cttz_i64_trunc:
866; EG:       ; %bb.0:
867; EG-NEXT:    ALU 2, @8, KC0[CB0:0-32], KC1[]
868; EG-NEXT:    TEX 0 @6
869; EG-NEXT:    ALU 10, @11, KC0[CB0:0-32], KC1[]
870; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
871; EG-NEXT:    CF_END
872; EG-NEXT:    PAD
873; EG-NEXT:    Fetch clause starting at 6:
874; EG-NEXT:     VTX_READ_64 T1.XY, T1.X, 0, #1
875; EG-NEXT:    ALU clause starting at 8:
876; EG-NEXT:     LSHL * T0.W, T0.X, literal.x,
877; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
878; EG-NEXT:     ADD_INT * T1.X, KC0[2].Z, PV.W,
879; EG-NEXT:    ALU clause starting at 11:
880; EG-NEXT:     FFBL_INT * T0.W, T1.Y,
881; EG-NEXT:     CNDE_INT * T0.W, T1.Y, literal.x, PV.W,
882; EG-NEXT:    32(4.484155e-44), 0(0.000000e+00)
883; EG-NEXT:     LSHL T0.Z, T0.X, literal.x,
884; EG-NEXT:     FFBL_INT T1.W, T1.X, BS:VEC_120/SCL_212
885; EG-NEXT:     ADD_INT * T0.W, PV.W, literal.y,
886; EG-NEXT:    2(2.802597e-45), 32(4.484155e-44)
887; EG-NEXT:     CNDE_INT T0.X, T1.X, PS, PV.W,
888; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, PV.Z,
889; EG-NEXT:     LSHR * T1.X, PV.W, literal.x,
890; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
891;
892; GFX10-LABEL: v_cttz_i64_trunc:
893; GFX10:       ; %bb.0:
894; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
895; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 3, v0
896; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
897; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
898; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
899; GFX10-NEXT:    global_load_dwordx2 v[1:2], v1, s[2:3]
900; GFX10-NEXT:    s_waitcnt vmcnt(0)
901; GFX10-NEXT:    v_ffbl_b32_e32 v2, v2
902; GFX10-NEXT:    v_ffbl_b32_e32 v1, v1
903; GFX10-NEXT:    v_add_nc_u32_e64 v2, v2, 32 clamp
904; GFX10-NEXT:    v_min3_u32 v1, v1, v2, 64
905; GFX10-NEXT:    global_store_dword v0, v1, s[0:1]
906; GFX10-NEXT:    s_endpgm
907;
908; GFX10-GISEL-LABEL: v_cttz_i64_trunc:
909; GFX10-GISEL:       ; %bb.0:
910; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
911; GFX10-GISEL-NEXT:    v_lshlrev_b32_e32 v1, 3, v0
912; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
913; GFX10-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
914; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
915; GFX10-GISEL-NEXT:    global_load_dwordx2 v[1:2], v1, s[2:3]
916; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0)
917; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v3, v2
918; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v4, v1
919; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
920; GFX10-GISEL-NEXT:    v_add_nc_u32_e32 v3, 32, v3
921; GFX10-GISEL-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc_lo
922; GFX10-GISEL-NEXT:    v_cmp_eq_u64_e32 vcc_lo, 0, v[1:2]
923; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v1, v3, 64, vcc_lo
924; GFX10-GISEL-NEXT:    global_store_dword v0, v1, s[0:1]
925; GFX10-GISEL-NEXT:    s_endpgm
926  %tid = call i32 @llvm.amdgcn.workitem.id.x()
927  %in.gep = getelementptr i64, i64 addrspace(1)* %in, i32 %tid
928  %out.gep = getelementptr i32, i32 addrspace(1)* %out, i32 %tid
929  %val = load i64, i64 addrspace(1)* %in.gep
930  %cttz = call i64 @llvm.cttz.i64(i64 %val, i1 false)
931  %trunc = trunc i64 %cttz to i32
932  store i32 %trunc, i32 addrspace(1)* %out.gep
933  ret void
934}
935
936define amdgpu_kernel void @v_cttz_i32_sel_eq_neg1(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalias %valptr) nounwind {
937; SI-LABEL: v_cttz_i32_sel_eq_neg1:
938; SI:       ; %bb.0:
939; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xb
940; SI-NEXT:    s_mov_b32 s3, 0xf000
941; SI-NEXT:    s_mov_b32 s6, 0
942; SI-NEXT:    s_mov_b32 s7, s3
943; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
944; SI-NEXT:    v_mov_b32_e32 v1, 0
945; SI-NEXT:    s_waitcnt lgkmcnt(0)
946; SI-NEXT:    buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
947; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
948; SI-NEXT:    s_mov_b32 s2, -1
949; SI-NEXT:    s_waitcnt vmcnt(0)
950; SI-NEXT:    v_ffbl_b32_e32 v0, v0
951; SI-NEXT:    s_waitcnt lgkmcnt(0)
952; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
953; SI-NEXT:    s_endpgm
954;
955; VI-LABEL: v_cttz_i32_sel_eq_neg1:
956; VI:       ; %bb.0:
957; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
958; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
959; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
960; VI-NEXT:    s_mov_b32 s7, 0xf000
961; VI-NEXT:    s_mov_b32 s6, -1
962; VI-NEXT:    s_waitcnt lgkmcnt(0)
963; VI-NEXT:    v_mov_b32_e32 v1, s1
964; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
965; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
966; VI-NEXT:    flat_load_dword v0, v[0:1]
967; VI-NEXT:    s_waitcnt vmcnt(0)
968; VI-NEXT:    v_ffbl_b32_e32 v0, v0
969; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
970; VI-NEXT:    s_endpgm
971;
972; EG-LABEL: v_cttz_i32_sel_eq_neg1:
973; EG:       ; %bb.0:
974; EG-NEXT:    ALU 2, @8, KC0[CB0:0-32], KC1[]
975; EG-NEXT:    TEX 0 @6
976; EG-NEXT:    ALU 5, @11, KC0[CB0:0-32], KC1[]
977; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
978; EG-NEXT:    CF_END
979; EG-NEXT:    PAD
980; EG-NEXT:    Fetch clause starting at 6:
981; EG-NEXT:     VTX_READ_32 T0.X, T0.X, 0, #1
982; EG-NEXT:    ALU clause starting at 8:
983; EG-NEXT:     LSHL * T0.W, T0.X, literal.x,
984; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
985; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, PV.W,
986; EG-NEXT:    ALU clause starting at 11:
987; EG-NEXT:     FFBL_INT * T0.W, T0.X,
988; EG-NEXT:     CNDE_INT * T0.W, T0.X, literal.x, PV.W,
989; EG-NEXT:    32(4.484155e-44), 0(0.000000e+00)
990; EG-NEXT:     CNDE_INT T0.X, T0.X, literal.x, PV.W,
991; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.y,
992; EG-NEXT:    -1(nan), 2(2.802597e-45)
993;
994; GFX10-LABEL: v_cttz_i32_sel_eq_neg1:
995; GFX10:       ; %bb.0:
996; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
997; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
998; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
999; GFX10-NEXT:    v_mov_b32_e32 v1, 0
1000; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
1001; GFX10-NEXT:    global_load_dword v0, v0, s[2:3]
1002; GFX10-NEXT:    s_waitcnt vmcnt(0)
1003; GFX10-NEXT:    v_ffbl_b32_e32 v0, v0
1004; GFX10-NEXT:    global_store_dword v1, v0, s[0:1]
1005; GFX10-NEXT:    s_endpgm
1006;
1007; GFX10-GISEL-LABEL: v_cttz_i32_sel_eq_neg1:
1008; GFX10-GISEL:       ; %bb.0:
1009; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
1010; GFX10-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1011; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1012; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
1013; GFX10-GISEL-NEXT:    global_load_dword v0, v0, s[2:3]
1014; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0)
1015; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v1, v0
1016; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
1017; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v0, v1, 32, vcc_lo
1018; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, 0
1019; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v0, v0, -1, vcc_lo
1020; GFX10-GISEL-NEXT:    global_store_dword v1, v0, s[0:1]
1021; GFX10-GISEL-NEXT:    s_endpgm
1022  %tid = call i32 @llvm.amdgcn.workitem.id.x()
1023  %in.gep = getelementptr i32, i32 addrspace(1)* %valptr, i32 %tid
1024  %val = load i32, i32 addrspace(1)* %in.gep
1025  %cttz = call i32 @llvm.cttz.i32(i32 %val, i1 false) nounwind readnone
1026  %cmp = icmp eq i32 %val, 0
1027  %sel = select i1 %cmp, i32 -1, i32 %cttz
1028  store i32 %sel, i32 addrspace(1)* %out
1029  ret void
1030}
1031
1032define amdgpu_kernel void @v_cttz_i32_sel_ne_neg1(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalias %valptr) nounwind {
1033; SI-LABEL: v_cttz_i32_sel_ne_neg1:
1034; SI:       ; %bb.0:
1035; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xb
1036; SI-NEXT:    s_mov_b32 s3, 0xf000
1037; SI-NEXT:    s_mov_b32 s6, 0
1038; SI-NEXT:    s_mov_b32 s7, s3
1039; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1040; SI-NEXT:    v_mov_b32_e32 v1, 0
1041; SI-NEXT:    s_waitcnt lgkmcnt(0)
1042; SI-NEXT:    buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
1043; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
1044; SI-NEXT:    s_mov_b32 s2, -1
1045; SI-NEXT:    s_waitcnt vmcnt(0)
1046; SI-NEXT:    v_ffbl_b32_e32 v0, v0
1047; SI-NEXT:    s_waitcnt lgkmcnt(0)
1048; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1049; SI-NEXT:    s_endpgm
1050;
1051; VI-LABEL: v_cttz_i32_sel_ne_neg1:
1052; VI:       ; %bb.0:
1053; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
1054; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
1055; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1056; VI-NEXT:    s_mov_b32 s7, 0xf000
1057; VI-NEXT:    s_mov_b32 s6, -1
1058; VI-NEXT:    s_waitcnt lgkmcnt(0)
1059; VI-NEXT:    v_mov_b32_e32 v1, s1
1060; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
1061; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
1062; VI-NEXT:    flat_load_dword v0, v[0:1]
1063; VI-NEXT:    s_waitcnt vmcnt(0)
1064; VI-NEXT:    v_ffbl_b32_e32 v0, v0
1065; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
1066; VI-NEXT:    s_endpgm
1067;
1068; EG-LABEL: v_cttz_i32_sel_ne_neg1:
1069; EG:       ; %bb.0:
1070; EG-NEXT:    ALU 2, @8, KC0[CB0:0-32], KC1[]
1071; EG-NEXT:    TEX 0 @6
1072; EG-NEXT:    ALU 5, @11, KC0[CB0:0-32], KC1[]
1073; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
1074; EG-NEXT:    CF_END
1075; EG-NEXT:    PAD
1076; EG-NEXT:    Fetch clause starting at 6:
1077; EG-NEXT:     VTX_READ_32 T0.X, T0.X, 0, #1
1078; EG-NEXT:    ALU clause starting at 8:
1079; EG-NEXT:     LSHL * T0.W, T0.X, literal.x,
1080; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
1081; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, PV.W,
1082; EG-NEXT:    ALU clause starting at 11:
1083; EG-NEXT:     FFBL_INT * T0.W, T0.X,
1084; EG-NEXT:     CNDE_INT * T0.W, T0.X, literal.x, PV.W,
1085; EG-NEXT:    32(4.484155e-44), 0(0.000000e+00)
1086; EG-NEXT:     CNDE_INT T0.X, T0.X, literal.x, PV.W,
1087; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.y,
1088; EG-NEXT:    -1(nan), 2(2.802597e-45)
1089;
1090; GFX10-LABEL: v_cttz_i32_sel_ne_neg1:
1091; GFX10:       ; %bb.0:
1092; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
1093; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1094; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1095; GFX10-NEXT:    v_mov_b32_e32 v1, 0
1096; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
1097; GFX10-NEXT:    global_load_dword v0, v0, s[2:3]
1098; GFX10-NEXT:    s_waitcnt vmcnt(0)
1099; GFX10-NEXT:    v_ffbl_b32_e32 v0, v0
1100; GFX10-NEXT:    global_store_dword v1, v0, s[0:1]
1101; GFX10-NEXT:    s_endpgm
1102;
1103; GFX10-GISEL-LABEL: v_cttz_i32_sel_ne_neg1:
1104; GFX10-GISEL:       ; %bb.0:
1105; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
1106; GFX10-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1107; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1108; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
1109; GFX10-GISEL-NEXT:    global_load_dword v0, v0, s[2:3]
1110; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0)
1111; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v1, v0
1112; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
1113; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v1, v1, 32, vcc_lo
1114; GFX10-GISEL-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
1115; GFX10-GISEL-NEXT:    v_cndmask_b32_e32 v0, -1, v1, vcc_lo
1116; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, 0
1117; GFX10-GISEL-NEXT:    global_store_dword v1, v0, s[0:1]
1118; GFX10-GISEL-NEXT:    s_endpgm
1119  %tid = call i32 @llvm.amdgcn.workitem.id.x()
1120  %in.gep = getelementptr i32, i32 addrspace(1)* %valptr, i32 %tid
1121  %val = load i32, i32 addrspace(1)* %in.gep
1122  %cttz = call i32 @llvm.cttz.i32(i32 %val, i1 false) nounwind readnone
1123  %cmp = icmp ne i32 %val, 0
1124  %sel = select i1 %cmp, i32 %cttz, i32 -1
1125  store i32 %sel, i32 addrspace(1)* %out
1126  ret void
1127}
1128
1129; TODO: Should be able to eliminate select here as well.
1130define amdgpu_kernel void @v_cttz_i32_sel_eq_bitwidth(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalias %valptr) nounwind {
1131; SI-LABEL: v_cttz_i32_sel_eq_bitwidth:
1132; SI:       ; %bb.0:
1133; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xb
1134; SI-NEXT:    s_mov_b32 s3, 0xf000
1135; SI-NEXT:    s_mov_b32 s6, 0
1136; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1137; SI-NEXT:    v_mov_b32_e32 v1, 0
1138; SI-NEXT:    s_mov_b32 s7, s3
1139; SI-NEXT:    s_waitcnt lgkmcnt(0)
1140; SI-NEXT:    buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
1141; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
1142; SI-NEXT:    s_mov_b32 s2, -1
1143; SI-NEXT:    s_waitcnt vmcnt(0)
1144; SI-NEXT:    v_ffbl_b32_e32 v1, v0
1145; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
1146; SI-NEXT:    v_cndmask_b32_e32 v0, 32, v1, vcc
1147; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 32, v0
1148; SI-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
1149; SI-NEXT:    s_waitcnt lgkmcnt(0)
1150; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1151; SI-NEXT:    s_endpgm
1152;
1153; VI-LABEL: v_cttz_i32_sel_eq_bitwidth:
1154; VI:       ; %bb.0:
1155; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
1156; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
1157; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1158; VI-NEXT:    s_mov_b32 s7, 0xf000
1159; VI-NEXT:    s_mov_b32 s6, -1
1160; VI-NEXT:    s_waitcnt lgkmcnt(0)
1161; VI-NEXT:    v_mov_b32_e32 v1, s1
1162; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
1163; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
1164; VI-NEXT:    flat_load_dword v0, v[0:1]
1165; VI-NEXT:    s_waitcnt vmcnt(0)
1166; VI-NEXT:    v_ffbl_b32_e32 v1, v0
1167; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
1168; VI-NEXT:    v_cndmask_b32_e32 v0, 32, v1, vcc
1169; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 32, v0
1170; VI-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
1171; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
1172; VI-NEXT:    s_endpgm
1173;
1174; EG-LABEL: v_cttz_i32_sel_eq_bitwidth:
1175; EG:       ; %bb.0:
1176; EG-NEXT:    ALU 2, @8, KC0[CB0:0-32], KC1[]
1177; EG-NEXT:    TEX 0 @6
1178; EG-NEXT:    ALU 7, @11, KC0[CB0:0-32], KC1[]
1179; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
1180; EG-NEXT:    CF_END
1181; EG-NEXT:    PAD
1182; EG-NEXT:    Fetch clause starting at 6:
1183; EG-NEXT:     VTX_READ_32 T0.X, T0.X, 0, #1
1184; EG-NEXT:    ALU clause starting at 8:
1185; EG-NEXT:     LSHL * T0.W, T0.X, literal.x,
1186; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
1187; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, PV.W,
1188; EG-NEXT:    ALU clause starting at 11:
1189; EG-NEXT:     FFBL_INT * T0.W, T0.X,
1190; EG-NEXT:     CNDE_INT * T0.W, T0.X, literal.x, PV.W,
1191; EG-NEXT:    32(4.484155e-44), 0(0.000000e+00)
1192; EG-NEXT:     SETE_INT * T1.W, PV.W, literal.x,
1193; EG-NEXT:    32(4.484155e-44), 0(0.000000e+00)
1194; EG-NEXT:     CNDE_INT T0.X, PV.W, T0.W, literal.x,
1195; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.y,
1196; EG-NEXT:    -1(nan), 2(2.802597e-45)
1197;
1198; GFX10-LABEL: v_cttz_i32_sel_eq_bitwidth:
1199; GFX10:       ; %bb.0:
1200; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
1201; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1202; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1203; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
1204; GFX10-NEXT:    global_load_dword v0, v0, s[2:3]
1205; GFX10-NEXT:    s_waitcnt vmcnt(0)
1206; GFX10-NEXT:    v_ffbl_b32_e32 v1, v0
1207; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
1208; GFX10-NEXT:    v_cndmask_b32_e32 v0, 32, v1, vcc_lo
1209; GFX10-NEXT:    v_mov_b32_e32 v1, 0
1210; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 32, v0
1211; GFX10-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc_lo
1212; GFX10-NEXT:    global_store_dword v1, v0, s[0:1]
1213; GFX10-NEXT:    s_endpgm
1214;
1215; GFX10-GISEL-LABEL: v_cttz_i32_sel_eq_bitwidth:
1216; GFX10-GISEL:       ; %bb.0:
1217; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
1218; GFX10-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1219; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1220; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
1221; GFX10-GISEL-NEXT:    global_load_dword v0, v0, s[2:3]
1222; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0)
1223; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v1, v0
1224; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
1225; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v0, v1, 32, vcc_lo
1226; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, 0
1227; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 32, v0
1228; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v0, v0, -1, vcc_lo
1229; GFX10-GISEL-NEXT:    global_store_dword v1, v0, s[0:1]
1230; GFX10-GISEL-NEXT:    s_endpgm
1231  %tid = call i32 @llvm.amdgcn.workitem.id.x()
1232  %in.gep = getelementptr i32, i32 addrspace(1)* %valptr, i32 %tid
1233  %val = load i32, i32 addrspace(1)* %in.gep
1234  %cttz = call i32 @llvm.cttz.i32(i32 %val, i1 false) nounwind readnone
1235  %cmp = icmp eq i32 %cttz, 32
1236  %sel = select i1 %cmp, i32 -1, i32 %cttz
1237  store i32 %sel, i32 addrspace(1)* %out
1238  ret void
1239}
1240
1241define amdgpu_kernel void @v_cttz_i32_sel_ne_bitwidth(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalias %valptr) nounwind {
1242; SI-LABEL: v_cttz_i32_sel_ne_bitwidth:
1243; SI:       ; %bb.0:
1244; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xb
1245; SI-NEXT:    s_mov_b32 s3, 0xf000
1246; SI-NEXT:    s_mov_b32 s6, 0
1247; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1248; SI-NEXT:    v_mov_b32_e32 v1, 0
1249; SI-NEXT:    s_mov_b32 s7, s3
1250; SI-NEXT:    s_waitcnt lgkmcnt(0)
1251; SI-NEXT:    buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
1252; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
1253; SI-NEXT:    s_mov_b32 s2, -1
1254; SI-NEXT:    s_waitcnt vmcnt(0)
1255; SI-NEXT:    v_ffbl_b32_e32 v1, v0
1256; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
1257; SI-NEXT:    v_cndmask_b32_e32 v0, 32, v1, vcc
1258; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 32, v0
1259; SI-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
1260; SI-NEXT:    s_waitcnt lgkmcnt(0)
1261; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1262; SI-NEXT:    s_endpgm
1263;
1264; VI-LABEL: v_cttz_i32_sel_ne_bitwidth:
1265; VI:       ; %bb.0:
1266; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
1267; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
1268; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1269; VI-NEXT:    s_mov_b32 s7, 0xf000
1270; VI-NEXT:    s_mov_b32 s6, -1
1271; VI-NEXT:    s_waitcnt lgkmcnt(0)
1272; VI-NEXT:    v_mov_b32_e32 v1, s1
1273; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
1274; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
1275; VI-NEXT:    flat_load_dword v0, v[0:1]
1276; VI-NEXT:    s_waitcnt vmcnt(0)
1277; VI-NEXT:    v_ffbl_b32_e32 v1, v0
1278; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
1279; VI-NEXT:    v_cndmask_b32_e32 v0, 32, v1, vcc
1280; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 32, v0
1281; VI-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
1282; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
1283; VI-NEXT:    s_endpgm
1284;
1285; EG-LABEL: v_cttz_i32_sel_ne_bitwidth:
1286; EG:       ; %bb.0:
1287; EG-NEXT:    ALU 2, @8, KC0[CB0:0-32], KC1[]
1288; EG-NEXT:    TEX 0 @6
1289; EG-NEXT:    ALU 7, @11, KC0[CB0:0-32], KC1[]
1290; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
1291; EG-NEXT:    CF_END
1292; EG-NEXT:    PAD
1293; EG-NEXT:    Fetch clause starting at 6:
1294; EG-NEXT:     VTX_READ_32 T0.X, T0.X, 0, #1
1295; EG-NEXT:    ALU clause starting at 8:
1296; EG-NEXT:     LSHL * T0.W, T0.X, literal.x,
1297; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
1298; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, PV.W,
1299; EG-NEXT:    ALU clause starting at 11:
1300; EG-NEXT:     FFBL_INT * T0.W, T0.X,
1301; EG-NEXT:     CNDE_INT * T0.W, T0.X, literal.x, PV.W,
1302; EG-NEXT:    32(4.484155e-44), 0(0.000000e+00)
1303; EG-NEXT:     SETNE_INT * T1.W, PV.W, literal.x,
1304; EG-NEXT:    32(4.484155e-44), 0(0.000000e+00)
1305; EG-NEXT:     CNDE_INT T0.X, PV.W, literal.x, T0.W,
1306; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.y,
1307; EG-NEXT:    -1(nan), 2(2.802597e-45)
1308;
1309; GFX10-LABEL: v_cttz_i32_sel_ne_bitwidth:
1310; GFX10:       ; %bb.0:
1311; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
1312; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1313; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1314; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
1315; GFX10-NEXT:    global_load_dword v0, v0, s[2:3]
1316; GFX10-NEXT:    s_waitcnt vmcnt(0)
1317; GFX10-NEXT:    v_ffbl_b32_e32 v1, v0
1318; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
1319; GFX10-NEXT:    v_cndmask_b32_e32 v0, 32, v1, vcc_lo
1320; GFX10-NEXT:    v_mov_b32_e32 v1, 0
1321; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 32, v0
1322; GFX10-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc_lo
1323; GFX10-NEXT:    global_store_dword v1, v0, s[0:1]
1324; GFX10-NEXT:    s_endpgm
1325;
1326; GFX10-GISEL-LABEL: v_cttz_i32_sel_ne_bitwidth:
1327; GFX10-GISEL:       ; %bb.0:
1328; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
1329; GFX10-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1330; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1331; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
1332; GFX10-GISEL-NEXT:    global_load_dword v0, v0, s[2:3]
1333; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0)
1334; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v1, v0
1335; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
1336; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v0, v1, 32, vcc_lo
1337; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, 0
1338; GFX10-GISEL-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 32, v0
1339; GFX10-GISEL-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc_lo
1340; GFX10-GISEL-NEXT:    global_store_dword v1, v0, s[0:1]
1341; GFX10-GISEL-NEXT:    s_endpgm
1342  %tid = call i32 @llvm.amdgcn.workitem.id.x()
1343  %in.gep = getelementptr i32, i32 addrspace(1)* %valptr, i32 %tid
1344  %val = load i32, i32 addrspace(1)* %in.gep
1345  %cttz = call i32 @llvm.cttz.i32(i32 %val, i1 false) nounwind readnone
1346  %cmp = icmp ne i32 %cttz, 32
1347  %sel = select i1 %cmp, i32 %cttz, i32 -1
1348  store i32 %sel, i32 addrspace(1)* %out
1349  ret void
1350}
1351
1352 define amdgpu_kernel void @v_cttz_i8_sel_eq_neg1(i8 addrspace(1)* noalias %out, i8 addrspace(1)* noalias %valptr) nounwind {
1353; SI-LABEL: v_cttz_i8_sel_eq_neg1:
1354; SI:       ; %bb.0:
1355; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xb
1356; SI-NEXT:    s_mov_b32 s3, 0xf000
1357; SI-NEXT:    v_mov_b32_e32 v1, 0
1358; SI-NEXT:    s_mov_b32 s6, 0
1359; SI-NEXT:    s_mov_b32 s7, s3
1360; SI-NEXT:    s_waitcnt lgkmcnt(0)
1361; SI-NEXT:    buffer_load_ubyte v0, v[0:1], s[4:7], 0 addr64
1362; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
1363; SI-NEXT:    s_mov_b32 s2, -1
1364; SI-NEXT:    s_waitcnt vmcnt(0)
1365; SI-NEXT:    v_ffbl_b32_e32 v0, v0
1366; SI-NEXT:    s_waitcnt lgkmcnt(0)
1367; SI-NEXT:    buffer_store_byte v0, off, s[0:3], 0
1368; SI-NEXT:    s_endpgm
1369;
1370; VI-LABEL: v_cttz_i8_sel_eq_neg1:
1371; VI:       ; %bb.0:
1372; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
1373; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
1374; VI-NEXT:    s_mov_b32 s7, 0xf000
1375; VI-NEXT:    s_mov_b32 s6, -1
1376; VI-NEXT:    s_waitcnt lgkmcnt(0)
1377; VI-NEXT:    v_mov_b32_e32 v1, s1
1378; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
1379; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
1380; VI-NEXT:    flat_load_ubyte v0, v[0:1]
1381; VI-NEXT:    s_waitcnt vmcnt(0)
1382; VI-NEXT:    v_ffbl_b32_e32 v0, v0
1383; VI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
1384; VI-NEXT:    s_endpgm
1385;
1386; EG-LABEL: v_cttz_i8_sel_eq_neg1:
1387; EG:       ; %bb.0:
1388; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
1389; EG-NEXT:    TEX 0 @6
1390; EG-NEXT:    ALU 12, @9, KC0[CB0:0-32], KC1[]
1391; EG-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
1392; EG-NEXT:    CF_END
1393; EG-NEXT:    PAD
1394; EG-NEXT:    Fetch clause starting at 6:
1395; EG-NEXT:     VTX_READ_8 T0.X, T0.X, 0, #1
1396; EG-NEXT:    ALU clause starting at 8:
1397; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, T0.X,
1398; EG-NEXT:    ALU clause starting at 9:
1399; EG-NEXT:     FFBL_INT T0.W, T0.X,
1400; EG-NEXT:     AND_INT * T1.W, KC0[2].Y, literal.x,
1401; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
1402; EG-NEXT:     AND_INT T0.W, PV.W, literal.x,
1403; EG-NEXT:     LSHL * T1.W, PS, literal.y,
1404; EG-NEXT:    255(3.573311e-43), 3(4.203895e-45)
1405; EG-NEXT:     LSHL T0.X, PV.W, PS,
1406; EG-NEXT:     LSHL * T0.W, literal.x, PS,
1407; EG-NEXT:    255(3.573311e-43), 0(0.000000e+00)
1408; EG-NEXT:     MOV T0.Y, 0.0,
1409; EG-NEXT:     MOV * T0.Z, 0.0,
1410; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
1411; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
1412;
1413; GFX10-LABEL: v_cttz_i8_sel_eq_neg1:
1414; GFX10:       ; %bb.0:
1415; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
1416; GFX10-NEXT:    v_mov_b32_e32 v1, 0
1417; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1418; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
1419; GFX10-NEXT:    global_load_ubyte v0, v0, s[2:3]
1420; GFX10-NEXT:    s_waitcnt vmcnt(0)
1421; GFX10-NEXT:    v_ffbl_b32_e32 v0, v0
1422; GFX10-NEXT:    global_store_byte v1, v0, s[0:1]
1423; GFX10-NEXT:    s_endpgm
1424;
1425; GFX10-GISEL-LABEL: v_cttz_i8_sel_eq_neg1:
1426; GFX10-GISEL:       ; %bb.0:
1427; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
1428; GFX10-GISEL-NEXT:    v_ashrrev_i32_e32 v3, 31, v0
1429; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1430; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
1431; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, s2
1432; GFX10-GISEL-NEXT:    v_mov_b32_e32 v2, s3
1433; GFX10-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, v1, v0
1434; GFX10-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v2, v3, vcc_lo
1435; GFX10-GISEL-NEXT:    global_load_ubyte v0, v[0:1], off
1436; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0)
1437; GFX10-GISEL-NEXT:    v_and_b32_e32 v0, 0xff, v0
1438; GFX10-GISEL-NEXT:    v_or_b32_e32 v1, 0x100, v0
1439; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
1440; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v1, v1
1441; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v0, v1, -1, vcc_lo
1442; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, 0
1443; GFX10-GISEL-NEXT:    global_store_byte v1, v0, s[0:1]
1444; GFX10-GISEL-NEXT:    s_endpgm
1445  %tid = call i32 @llvm.amdgcn.workitem.id.x()
1446  %valptr.gep = getelementptr i8, i8 addrspace(1)* %valptr, i32 %tid
1447  %val = load i8, i8 addrspace(1)* %valptr.gep
1448  %cttz = call i8 @llvm.cttz.i8(i8 %val, i1 false) nounwind readnone
1449  %cmp = icmp eq i8 %val, 0
1450  %sel = select i1 %cmp, i8 -1, i8 %cttz
1451  store i8 %sel, i8 addrspace(1)* %out
1452  ret void
1453}
1454
1455 define amdgpu_kernel void @v_cttz_i16_sel_eq_neg1(i16 addrspace(1)* noalias %out, i16 addrspace(1)* noalias %valptr) nounwind {
1456; SI-LABEL: v_cttz_i16_sel_eq_neg1:
1457; SI:       ; %bb.0:
1458; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xb
1459; SI-NEXT:    s_mov_b32 s3, 0xf000
1460; SI-NEXT:    s_mov_b32 s2, -1
1461; SI-NEXT:    s_mov_b32 s6, s2
1462; SI-NEXT:    s_mov_b32 s7, s3
1463; SI-NEXT:    s_waitcnt lgkmcnt(0)
1464; SI-NEXT:    buffer_load_ushort v0, off, s[4:7], 0
1465; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
1466; SI-NEXT:    s_waitcnt vmcnt(0)
1467; SI-NEXT:    v_ffbl_b32_e32 v0, v0
1468; SI-NEXT:    s_waitcnt lgkmcnt(0)
1469; SI-NEXT:    buffer_store_short v0, off, s[0:3], 0
1470; SI-NEXT:    s_endpgm
1471;
1472; VI-LABEL: v_cttz_i16_sel_eq_neg1:
1473; VI:       ; %bb.0:
1474; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
1475; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
1476; VI-NEXT:    s_mov_b32 s7, 0xf000
1477; VI-NEXT:    s_mov_b32 s6, -1
1478; VI-NEXT:    s_mov_b32 s2, s6
1479; VI-NEXT:    s_mov_b32 s3, s7
1480; VI-NEXT:    s_waitcnt lgkmcnt(0)
1481; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], 0
1482; VI-NEXT:    v_mov_b32_e32 v1, 0xffff
1483; VI-NEXT:    s_waitcnt vmcnt(0)
1484; VI-NEXT:    v_or_b32_e32 v2, 0x10000, v0
1485; VI-NEXT:    v_ffbl_b32_e32 v3, v2
1486; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
1487; VI-NEXT:    v_cndmask_b32_e32 v2, 32, v3, vcc
1488; VI-NEXT:    v_cmp_ne_u16_e32 vcc, 0, v0
1489; VI-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
1490; VI-NEXT:    buffer_store_short v0, off, s[4:7], 0
1491; VI-NEXT:    s_endpgm
1492;
1493; EG-LABEL: v_cttz_i16_sel_eq_neg1:
1494; EG:       ; %bb.0:
1495; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
1496; EG-NEXT:    TEX 0 @6
1497; EG-NEXT:    ALU 12, @9, KC0[CB0:0-32], KC1[]
1498; EG-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
1499; EG-NEXT:    CF_END
1500; EG-NEXT:    PAD
1501; EG-NEXT:    Fetch clause starting at 6:
1502; EG-NEXT:     VTX_READ_16 T0.X, T0.X, 0, #1
1503; EG-NEXT:    ALU clause starting at 8:
1504; EG-NEXT:     MOV * T0.X, KC0[2].Z,
1505; EG-NEXT:    ALU clause starting at 9:
1506; EG-NEXT:     FFBL_INT T0.W, T0.X,
1507; EG-NEXT:     AND_INT * T1.W, KC0[2].Y, literal.x,
1508; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
1509; EG-NEXT:     AND_INT T0.W, PV.W, literal.x,
1510; EG-NEXT:     LSHL * T1.W, PS, literal.y,
1511; EG-NEXT:    65535(9.183409e-41), 3(4.203895e-45)
1512; EG-NEXT:     LSHL T0.X, PV.W, PS,
1513; EG-NEXT:     LSHL * T0.W, literal.x, PS,
1514; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
1515; EG-NEXT:     MOV T0.Y, 0.0,
1516; EG-NEXT:     MOV * T0.Z, 0.0,
1517; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
1518; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
1519;
1520; GFX10-LABEL: v_cttz_i16_sel_eq_neg1:
1521; GFX10:       ; %bb.0:
1522; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
1523; GFX10-NEXT:    v_mov_b32_e32 v0, 0
1524; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1525; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
1526; GFX10-NEXT:    global_load_ushort v1, v0, s[2:3]
1527; GFX10-NEXT:    s_waitcnt vmcnt(0)
1528; GFX10-NEXT:    v_or_b32_e32 v2, 0x10000, v1
1529; GFX10-NEXT:    v_ffbl_b32_e32 v3, v2
1530; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v2
1531; GFX10-NEXT:    v_cndmask_b32_e32 v2, 32, v3, vcc_lo
1532; GFX10-NEXT:    v_cmp_ne_u16_e32 vcc_lo, 0, v1
1533; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0xffff, v2, vcc_lo
1534; GFX10-NEXT:    global_store_short v0, v1, s[0:1]
1535; GFX10-NEXT:    s_endpgm
1536;
1537; GFX10-GISEL-LABEL: v_cttz_i16_sel_eq_neg1:
1538; GFX10-GISEL:       ; %bb.0:
1539; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
1540; GFX10-GISEL-NEXT:    v_mov_b32_e32 v0, 0
1541; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1542; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
1543; GFX10-GISEL-NEXT:    global_load_ushort v1, v0, s[2:3]
1544; GFX10-GISEL-NEXT:    s_waitcnt_depctr 0xffe3
1545; GFX10-GISEL-NEXT:    s_mov_b32 s2, 0xffff
1546; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0)
1547; GFX10-GISEL-NEXT:    v_or_b32_e32 v2, 0x10000, v1
1548; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
1549; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v2, v2
1550; GFX10-GISEL-NEXT:    v_and_b32_e32 v2, s2, v2
1551; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v1, v2, s2, vcc_lo
1552; GFX10-GISEL-NEXT:    global_store_short v0, v1, s[0:1]
1553; GFX10-GISEL-NEXT:    s_endpgm
1554  %val = load i16, i16 addrspace(1)* %valptr
1555  %cttz = call i16 @llvm.cttz.i16(i16 %val, i1 false) nounwind readnone
1556  %cmp = icmp eq i16 %val, 0
1557  %sel = select i1 %cmp, i16 -1, i16 %cttz
1558  store i16 %sel, i16 addrspace(1)* %out
1559  ret void
1560}
1561
1562; FIXME: Need to handle non-uniform case for function below (load without gep).
1563define amdgpu_kernel void @v_cttz_i7_sel_eq_neg1(i7 addrspace(1)* noalias %out, i7 addrspace(1)* noalias %valptr) nounwind {
1564; SI-LABEL: v_cttz_i7_sel_eq_neg1:
1565; SI:       ; %bb.0:
1566; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xb
1567; SI-NEXT:    s_mov_b32 s3, 0xf000
1568; SI-NEXT:    v_mov_b32_e32 v1, 0
1569; SI-NEXT:    s_mov_b32 s6, 0
1570; SI-NEXT:    s_mov_b32 s7, s3
1571; SI-NEXT:    s_waitcnt lgkmcnt(0)
1572; SI-NEXT:    buffer_load_ubyte v0, v[0:1], s[4:7], 0 addr64
1573; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
1574; SI-NEXT:    s_mov_b32 s2, -1
1575; SI-NEXT:    s_waitcnt vmcnt(0)
1576; SI-NEXT:    v_ffbl_b32_e32 v0, v0
1577; SI-NEXT:    v_and_b32_e32 v0, 0x7f, v0
1578; SI-NEXT:    s_waitcnt lgkmcnt(0)
1579; SI-NEXT:    buffer_store_byte v0, off, s[0:3], 0
1580; SI-NEXT:    s_endpgm
1581;
1582; VI-LABEL: v_cttz_i7_sel_eq_neg1:
1583; VI:       ; %bb.0:
1584; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
1585; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
1586; VI-NEXT:    s_mov_b32 s7, 0xf000
1587; VI-NEXT:    s_mov_b32 s6, -1
1588; VI-NEXT:    s_waitcnt lgkmcnt(0)
1589; VI-NEXT:    v_mov_b32_e32 v1, s1
1590; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
1591; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
1592; VI-NEXT:    flat_load_ubyte v0, v[0:1]
1593; VI-NEXT:    s_waitcnt vmcnt(0)
1594; VI-NEXT:    v_ffbl_b32_e32 v0, v0
1595; VI-NEXT:    v_and_b32_e32 v0, 0x7f, v0
1596; VI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
1597; VI-NEXT:    s_endpgm
1598;
1599; EG-LABEL: v_cttz_i7_sel_eq_neg1:
1600; EG:       ; %bb.0:
1601; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
1602; EG-NEXT:    TEX 0 @6
1603; EG-NEXT:    ALU 12, @9, KC0[CB0:0-32], KC1[]
1604; EG-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
1605; EG-NEXT:    CF_END
1606; EG-NEXT:    PAD
1607; EG-NEXT:    Fetch clause starting at 6:
1608; EG-NEXT:     VTX_READ_8 T0.X, T0.X, 0, #1
1609; EG-NEXT:    ALU clause starting at 8:
1610; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, T0.X,
1611; EG-NEXT:    ALU clause starting at 9:
1612; EG-NEXT:     FFBL_INT T0.W, T0.X,
1613; EG-NEXT:     AND_INT * T1.W, KC0[2].Y, literal.x,
1614; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
1615; EG-NEXT:     AND_INT T0.W, PV.W, literal.x,
1616; EG-NEXT:     LSHL * T1.W, PS, literal.y,
1617; EG-NEXT:    127(1.779649e-43), 3(4.203895e-45)
1618; EG-NEXT:     LSHL T0.X, PV.W, PS,
1619; EG-NEXT:     LSHL * T0.W, literal.x, PS,
1620; EG-NEXT:    255(3.573311e-43), 0(0.000000e+00)
1621; EG-NEXT:     MOV T0.Y, 0.0,
1622; EG-NEXT:     MOV * T0.Z, 0.0,
1623; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
1624; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
1625;
1626; GFX10-LABEL: v_cttz_i7_sel_eq_neg1:
1627; GFX10:       ; %bb.0:
1628; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
1629; GFX10-NEXT:    v_mov_b32_e32 v1, 0
1630; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1631; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
1632; GFX10-NEXT:    global_load_ubyte v0, v0, s[2:3]
1633; GFX10-NEXT:    s_waitcnt vmcnt(0)
1634; GFX10-NEXT:    v_ffbl_b32_e32 v0, v0
1635; GFX10-NEXT:    v_and_b32_e32 v0, 0x7f, v0
1636; GFX10-NEXT:    global_store_byte v1, v0, s[0:1]
1637; GFX10-NEXT:    s_endpgm
1638;
1639; GFX10-GISEL-LABEL: v_cttz_i7_sel_eq_neg1:
1640; GFX10-GISEL:       ; %bb.0:
1641; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
1642; GFX10-GISEL-NEXT:    v_ashrrev_i32_e32 v3, 31, v0
1643; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1644; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
1645; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, s2
1646; GFX10-GISEL-NEXT:    v_mov_b32_e32 v2, s3
1647; GFX10-GISEL-NEXT:    s_movk_i32 s2, 0x7f
1648; GFX10-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, v1, v0
1649; GFX10-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v2, v3, vcc_lo
1650; GFX10-GISEL-NEXT:    global_load_ubyte v0, v[0:1], off
1651; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0)
1652; GFX10-GISEL-NEXT:    v_and_or_b32 v1, v0, s2, 0x80
1653; GFX10-GISEL-NEXT:    v_and_b32_e32 v0, s2, v0
1654; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v1, v1
1655; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
1656; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v0, v1, 0x7f, vcc_lo
1657; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, 0
1658; GFX10-GISEL-NEXT:    v_and_b32_e32 v0, s2, v0
1659; GFX10-GISEL-NEXT:    global_store_byte v1, v0, s[0:1]
1660; GFX10-GISEL-NEXT:    s_endpgm
1661  %tid = call i32 @llvm.amdgcn.workitem.id.x()
1662  %valptr.gep = getelementptr i7, i7 addrspace(1)* %valptr, i32 %tid
1663  %val = load i7, i7 addrspace(1)* %valptr.gep
1664  %cttz = call i7 @llvm.cttz.i7(i7 %val, i1 false) nounwind readnone
1665  %cmp = icmp eq i7 %val, 0
1666  %sel = select i1 %cmp, i7 -1, i7 %cttz
1667  store i7 %sel, i7 addrspace(1)* %out
1668  ret void
1669}
1670