1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc < %s -march=amdgcn -verify-machineinstrs | FileCheck %s -enable-var-scope --check-prefix=SI
3; RUN: llc < %s -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs | FileCheck %s -enable-var-scope --check-prefix=VI
4; RUN: llc < %s -march=r600 -mcpu=cypress -verify-machineinstrs | FileCheck %s -enable-var-scope --check-prefix=EG
5; RUN: llc < %s -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs | FileCheck %s -enable-var-scope --check-prefix=GFX10
6; RUN: llc < %s -global-isel -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs | FileCheck %s -enable-var-scope --check-prefix=GFX10-GISEL
7
8declare i7 @llvm.cttz.i7(i7, i1) nounwind readnone
9declare i8 @llvm.cttz.i8(i8, i1) nounwind readnone
10declare i16 @llvm.cttz.i16(i16, i1) nounwind readnone
11
12declare i32 @llvm.cttz.i32(i32, i1) nounwind readnone
13declare <2 x i32> @llvm.cttz.v2i32(<2 x i32>, i1) nounwind readnone
14declare <4 x i32> @llvm.cttz.v4i32(<4 x i32>, i1) nounwind readnone
15
16declare i64 @llvm.cttz.i64(i64, i1) nounwind readnone
17declare <2 x i64> @llvm.cttz.v2i64(<2 x i64>, i1) nounwind readnone
18declare <4 x i64> @llvm.cttz.v4i64(<4 x i64>, i1) nounwind readnone
19
20declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
21
22define amdgpu_kernel void @s_cttz_i32(i32 addrspace(1)* noalias %out, i32 %val) nounwind {
23; SI-LABEL: s_cttz_i32:
24; SI:       ; %bb.0:
25; SI-NEXT:    s_load_dword s4, s[0:1], 0xb
26; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
27; SI-NEXT:    s_mov_b32 s3, 0xf000
28; SI-NEXT:    s_waitcnt lgkmcnt(0)
29; SI-NEXT:    s_ff1_i32_b32 s5, s4
30; SI-NEXT:    s_mov_b32 s2, -1
31; SI-NEXT:    v_mov_b32_e32 v0, s5
32; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s4, 0
33; SI-NEXT:    v_cndmask_b32_e32 v0, 32, v0, vcc
34; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
35; SI-NEXT:    s_endpgm
36;
37; VI-LABEL: s_cttz_i32:
38; VI:       ; %bb.0:
39; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
40; VI-NEXT:    s_load_dword s0, s[0:1], 0x2c
41; VI-NEXT:    s_mov_b32 s7, 0xf000
42; VI-NEXT:    s_mov_b32 s6, -1
43; VI-NEXT:    s_waitcnt lgkmcnt(0)
44; VI-NEXT:    s_ff1_i32_b32 s1, s0
45; VI-NEXT:    s_cmp_lg_u32 s0, 0
46; VI-NEXT:    s_cselect_b32 s0, s1, 32
47; VI-NEXT:    v_mov_b32_e32 v0, s0
48; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
49; VI-NEXT:    s_endpgm
50;
51; EG-LABEL: s_cttz_i32:
52; EG:       ; %bb.0:
53; EG-NEXT:    ALU 3, @4, KC0[CB0:0-32], KC1[]
54; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
55; EG-NEXT:    CF_END
56; EG-NEXT:    PAD
57; EG-NEXT:    ALU clause starting at 4:
58; EG-NEXT:     FFBL_INT * T0.W, KC0[2].Z,
59; EG-NEXT:     CNDE_INT T0.X, KC0[2].Z, literal.x, PV.W,
60; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.y,
61; EG-NEXT:    32(4.484155e-44), 2(2.802597e-45)
62;
63; GFX10-LABEL: s_cttz_i32:
64; GFX10:       ; %bb.0:
65; GFX10-NEXT:    s_clause 0x1
66; GFX10-NEXT:    s_load_dword s4, s[0:1], 0x2c
67; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
68; GFX10-NEXT:    v_mov_b32_e32 v0, 0
69; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
70; GFX10-NEXT:    s_ff1_i32_b32 s0, s4
71; GFX10-NEXT:    s_cmp_lg_u32 s4, 0
72; GFX10-NEXT:    s_cselect_b32 s0, s0, 32
73; GFX10-NEXT:    v_mov_b32_e32 v1, s0
74; GFX10-NEXT:    global_store_dword v0, v1, s[2:3]
75; GFX10-NEXT:    s_endpgm
76;
77; GFX10-GISEL-LABEL: s_cttz_i32:
78; GFX10-GISEL:       ; %bb.0:
79; GFX10-GISEL-NEXT:    s_clause 0x1
80; GFX10-GISEL-NEXT:    s_load_dword s4, s[0:1], 0x2c
81; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
82; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, 0
83; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
84; GFX10-GISEL-NEXT:    s_ff1_i32_b32 s0, s4
85; GFX10-GISEL-NEXT:    s_cmp_eq_u32 s4, 0
86; GFX10-GISEL-NEXT:    s_cselect_b32 s0, 32, s0
87; GFX10-GISEL-NEXT:    v_mov_b32_e32 v0, s0
88; GFX10-GISEL-NEXT:    global_store_dword v1, v0, s[2:3]
89; GFX10-GISEL-NEXT:    s_endpgm
90  %cttz = call i32 @llvm.cttz.i32(i32 %val, i1 false) nounwind readnone
91  store i32 %cttz, i32 addrspace(1)* %out, align 4
92  ret void
93}
94
95define amdgpu_kernel void @v_cttz_i32(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalias %valptr) nounwind {
96; SI-LABEL: v_cttz_i32:
97; SI:       ; %bb.0:
98; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xb
99; SI-NEXT:    s_mov_b32 s3, 0xf000
100; SI-NEXT:    s_mov_b32 s6, 0
101; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
102; SI-NEXT:    v_mov_b32_e32 v1, 0
103; SI-NEXT:    s_mov_b32 s7, s3
104; SI-NEXT:    s_waitcnt lgkmcnt(0)
105; SI-NEXT:    buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
106; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
107; SI-NEXT:    s_mov_b32 s2, -1
108; SI-NEXT:    s_waitcnt vmcnt(0)
109; SI-NEXT:    v_ffbl_b32_e32 v1, v0
110; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
111; SI-NEXT:    v_cndmask_b32_e32 v0, 32, v1, vcc
112; SI-NEXT:    s_waitcnt lgkmcnt(0)
113; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
114; SI-NEXT:    s_endpgm
115;
116; VI-LABEL: v_cttz_i32:
117; VI:       ; %bb.0:
118; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
119; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
120; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
121; VI-NEXT:    s_mov_b32 s7, 0xf000
122; VI-NEXT:    s_mov_b32 s6, -1
123; VI-NEXT:    s_waitcnt lgkmcnt(0)
124; VI-NEXT:    v_mov_b32_e32 v1, s1
125; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
126; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
127; VI-NEXT:    flat_load_dword v0, v[0:1]
128; VI-NEXT:    s_waitcnt vmcnt(0)
129; VI-NEXT:    v_ffbl_b32_e32 v1, v0
130; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
131; VI-NEXT:    v_cndmask_b32_e32 v0, 32, v1, vcc
132; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
133; VI-NEXT:    s_endpgm
134;
135; EG-LABEL: v_cttz_i32:
136; EG:       ; %bb.0:
137; EG-NEXT:    ALU 2, @8, KC0[CB0:0-32], KC1[]
138; EG-NEXT:    TEX 0 @6
139; EG-NEXT:    ALU 3, @11, KC0[CB0:0-32], KC1[]
140; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
141; EG-NEXT:    CF_END
142; EG-NEXT:    PAD
143; EG-NEXT:    Fetch clause starting at 6:
144; EG-NEXT:     VTX_READ_32 T0.X, T0.X, 0, #1
145; EG-NEXT:    ALU clause starting at 8:
146; EG-NEXT:     LSHL * T0.W, T0.X, literal.x,
147; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
148; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, PV.W,
149; EG-NEXT:    ALU clause starting at 11:
150; EG-NEXT:     FFBL_INT * T0.W, T0.X,
151; EG-NEXT:     CNDE_INT T0.X, T0.X, literal.x, PV.W,
152; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.y,
153; EG-NEXT:    32(4.484155e-44), 2(2.802597e-45)
154;
155; GFX10-LABEL: v_cttz_i32:
156; GFX10:       ; %bb.0:
157; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
158; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
159; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
160; GFX10-NEXT:    v_mov_b32_e32 v2, 0
161; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
162; GFX10-NEXT:    global_load_dword v0, v0, s[2:3]
163; GFX10-NEXT:    s_waitcnt vmcnt(0)
164; GFX10-NEXT:    v_ffbl_b32_e32 v1, v0
165; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
166; GFX10-NEXT:    v_cndmask_b32_e32 v0, 32, v1, vcc_lo
167; GFX10-NEXT:    global_store_dword v2, v0, s[0:1]
168; GFX10-NEXT:    s_endpgm
169;
170; GFX10-GISEL-LABEL: v_cttz_i32:
171; GFX10-GISEL:       ; %bb.0:
172; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
173; GFX10-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
174; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
175; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
176; GFX10-GISEL-NEXT:    global_load_dword v0, v0, s[2:3]
177; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0)
178; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v1, v0
179; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
180; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v0, v1, 32, vcc_lo
181; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, 0
182; GFX10-GISEL-NEXT:    global_store_dword v1, v0, s[0:1]
183; GFX10-GISEL-NEXT:    s_endpgm
184  %tid = call i32 @llvm.amdgcn.workitem.id.x()
185  %in.gep = getelementptr i32, i32 addrspace(1)* %valptr, i32 %tid
186  %val = load i32, i32 addrspace(1)* %in.gep, align 4
187  %cttz = call i32 @llvm.cttz.i32(i32 %val, i1 false) nounwind readnone
188  store i32 %cttz, i32 addrspace(1)* %out, align 4
189  ret void
190}
191
192define amdgpu_kernel void @v_cttz_v2i32(<2 x i32> addrspace(1)* noalias %out, <2 x i32> addrspace(1)* noalias %valptr) nounwind {
193; SI-LABEL: v_cttz_v2i32:
194; SI:       ; %bb.0:
195; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xb
196; SI-NEXT:    s_mov_b32 s3, 0xf000
197; SI-NEXT:    s_mov_b32 s6, 0
198; SI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
199; SI-NEXT:    v_mov_b32_e32 v1, 0
200; SI-NEXT:    s_mov_b32 s7, s3
201; SI-NEXT:    s_waitcnt lgkmcnt(0)
202; SI-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
203; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
204; SI-NEXT:    s_mov_b32 s2, -1
205; SI-NEXT:    s_waitcnt vmcnt(0)
206; SI-NEXT:    v_ffbl_b32_e32 v2, v1
207; SI-NEXT:    v_ffbl_b32_e32 v3, v0
208; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
209; SI-NEXT:    v_cndmask_b32_e32 v1, 32, v2, vcc
210; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
211; SI-NEXT:    v_cndmask_b32_e32 v0, 32, v3, vcc
212; SI-NEXT:    s_waitcnt lgkmcnt(0)
213; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
214; SI-NEXT:    s_endpgm
215;
216; VI-LABEL: v_cttz_v2i32:
217; VI:       ; %bb.0:
218; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
219; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
220; VI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
221; VI-NEXT:    s_mov_b32 s7, 0xf000
222; VI-NEXT:    s_mov_b32 s6, -1
223; VI-NEXT:    s_waitcnt lgkmcnt(0)
224; VI-NEXT:    v_mov_b32_e32 v1, s1
225; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
226; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
227; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
228; VI-NEXT:    s_waitcnt vmcnt(0)
229; VI-NEXT:    v_ffbl_b32_e32 v2, v1
230; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
231; VI-NEXT:    v_cndmask_b32_e32 v1, 32, v2, vcc
232; VI-NEXT:    v_ffbl_b32_e32 v3, v0
233; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
234; VI-NEXT:    v_cndmask_b32_e32 v0, 32, v3, vcc
235; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
236; VI-NEXT:    s_endpgm
237;
238; EG-LABEL: v_cttz_v2i32:
239; EG:       ; %bb.0:
240; EG-NEXT:    ALU 2, @8, KC0[CB0:0-32], KC1[]
241; EG-NEXT:    TEX 0 @6
242; EG-NEXT:    ALU 6, @11, KC0[CB0:0-32], KC1[]
243; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
244; EG-NEXT:    CF_END
245; EG-NEXT:    PAD
246; EG-NEXT:    Fetch clause starting at 6:
247; EG-NEXT:     VTX_READ_64 T0.XY, T0.X, 0, #1
248; EG-NEXT:    ALU clause starting at 8:
249; EG-NEXT:     LSHL * T0.W, T0.X, literal.x,
250; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
251; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, PV.W,
252; EG-NEXT:    ALU clause starting at 11:
253; EG-NEXT:     FFBL_INT * T0.W, T0.Y,
254; EG-NEXT:     CNDE_INT T0.Y, T0.Y, literal.x, PV.W,
255; EG-NEXT:     FFBL_INT * T0.W, T0.X,
256; EG-NEXT:    32(4.484155e-44), 0(0.000000e+00)
257; EG-NEXT:     CNDE_INT T0.X, T0.X, literal.x, PV.W,
258; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.y,
259; EG-NEXT:    32(4.484155e-44), 2(2.802597e-45)
260;
261; GFX10-LABEL: v_cttz_v2i32:
262; GFX10:       ; %bb.0:
263; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
264; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
265; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
266; GFX10-NEXT:    v_mov_b32_e32 v4, 0
267; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
268; GFX10-NEXT:    global_load_dwordx2 v[0:1], v0, s[2:3]
269; GFX10-NEXT:    s_waitcnt vmcnt(0)
270; GFX10-NEXT:    v_ffbl_b32_e32 v2, v1
271; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v1
272; GFX10-NEXT:    v_ffbl_b32_e32 v3, v0
273; GFX10-NEXT:    v_cndmask_b32_e32 v1, 32, v2, vcc_lo
274; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
275; GFX10-NEXT:    v_cndmask_b32_e32 v0, 32, v3, vcc_lo
276; GFX10-NEXT:    global_store_dwordx2 v4, v[0:1], s[0:1]
277; GFX10-NEXT:    s_endpgm
278;
279; GFX10-GISEL-LABEL: v_cttz_v2i32:
280; GFX10-GISEL:       ; %bb.0:
281; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
282; GFX10-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
283; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
284; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
285; GFX10-GISEL-NEXT:    global_load_dwordx2 v[0:1], v0, s[2:3]
286; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0)
287; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v2, v0
288; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
289; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v3, v1
290; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v0, v2, 32, vcc_lo
291; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
292; GFX10-GISEL-NEXT:    v_mov_b32_e32 v2, 0
293; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v1, v3, 32, vcc_lo
294; GFX10-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
295; GFX10-GISEL-NEXT:    s_endpgm
296  %tid = call i32 @llvm.amdgcn.workitem.id.x()
297  %in.gep = getelementptr <2 x i32>, <2 x i32> addrspace(1)* %valptr, i32 %tid
298  %val = load <2 x i32>, <2 x i32> addrspace(1)* %in.gep, align 8
299  %cttz = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %val, i1 false) nounwind readnone
300  store <2 x i32> %cttz, <2 x i32> addrspace(1)* %out, align 8
301  ret void
302}
303
304define amdgpu_kernel void @v_cttz_v4i32(<4 x i32> addrspace(1)* noalias %out, <4 x i32> addrspace(1)* noalias %valptr) nounwind {
305; SI-LABEL: v_cttz_v4i32:
306; SI:       ; %bb.0:
307; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xb
308; SI-NEXT:    s_mov_b32 s3, 0xf000
309; SI-NEXT:    s_mov_b32 s6, 0
310; SI-NEXT:    v_lshlrev_b32_e32 v0, 4, v0
311; SI-NEXT:    v_mov_b32_e32 v1, 0
312; SI-NEXT:    s_mov_b32 s7, s3
313; SI-NEXT:    s_waitcnt lgkmcnt(0)
314; SI-NEXT:    buffer_load_dwordx4 v[0:3], v[0:1], s[4:7], 0 addr64
315; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
316; SI-NEXT:    s_mov_b32 s2, -1
317; SI-NEXT:    s_waitcnt vmcnt(0)
318; SI-NEXT:    v_ffbl_b32_e32 v4, v3
319; SI-NEXT:    v_ffbl_b32_e32 v5, v2
320; SI-NEXT:    v_ffbl_b32_e32 v6, v1
321; SI-NEXT:    v_ffbl_b32_e32 v7, v0
322; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v3
323; SI-NEXT:    v_cndmask_b32_e32 v3, 32, v4, vcc
324; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
325; SI-NEXT:    v_cndmask_b32_e32 v2, 32, v5, vcc
326; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
327; SI-NEXT:    v_cndmask_b32_e32 v1, 32, v6, vcc
328; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
329; SI-NEXT:    v_cndmask_b32_e32 v0, 32, v7, vcc
330; SI-NEXT:    s_waitcnt lgkmcnt(0)
331; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
332; SI-NEXT:    s_endpgm
333;
334; VI-LABEL: v_cttz_v4i32:
335; VI:       ; %bb.0:
336; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
337; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
338; VI-NEXT:    v_lshlrev_b32_e32 v0, 4, v0
339; VI-NEXT:    s_mov_b32 s7, 0xf000
340; VI-NEXT:    s_mov_b32 s6, -1
341; VI-NEXT:    s_waitcnt lgkmcnt(0)
342; VI-NEXT:    v_mov_b32_e32 v1, s1
343; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
344; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
345; VI-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
346; VI-NEXT:    s_waitcnt vmcnt(0)
347; VI-NEXT:    v_ffbl_b32_e32 v4, v3
348; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v3
349; VI-NEXT:    v_cndmask_b32_e32 v3, 32, v4, vcc
350; VI-NEXT:    v_ffbl_b32_e32 v5, v2
351; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
352; VI-NEXT:    v_cndmask_b32_e32 v2, 32, v5, vcc
353; VI-NEXT:    v_ffbl_b32_e32 v6, v1
354; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
355; VI-NEXT:    v_cndmask_b32_e32 v1, 32, v6, vcc
356; VI-NEXT:    v_ffbl_b32_e32 v7, v0
357; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
358; VI-NEXT:    v_cndmask_b32_e32 v0, 32, v7, vcc
359; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
360; VI-NEXT:    s_endpgm
361;
362; EG-LABEL: v_cttz_v4i32:
363; EG:       ; %bb.0:
364; EG-NEXT:    ALU 2, @8, KC0[CB0:0-32], KC1[]
365; EG-NEXT:    TEX 0 @6
366; EG-NEXT:    ALU 12, @11, KC0[CB0:0-32], KC1[]
367; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XYZW, T1.X, 1
368; EG-NEXT:    CF_END
369; EG-NEXT:    PAD
370; EG-NEXT:    Fetch clause starting at 6:
371; EG-NEXT:     VTX_READ_128 T0.XYZW, T0.X, 0, #1
372; EG-NEXT:    ALU clause starting at 8:
373; EG-NEXT:     LSHL * T0.W, T0.X, literal.x,
374; EG-NEXT:    4(5.605194e-45), 0(0.000000e+00)
375; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, PV.W,
376; EG-NEXT:    ALU clause starting at 11:
377; EG-NEXT:     FFBL_INT * T1.W, T0.W,
378; EG-NEXT:     FFBL_INT T2.W, T0.Z,
379; EG-NEXT:     CNDE_INT * T0.W, T0.W, literal.x, PV.W, BS:VEC_021/SCL_122
380; EG-NEXT:    32(4.484155e-44), 0(0.000000e+00)
381; EG-NEXT:     CNDE_INT T0.Z, T0.Z, literal.x, PV.W,
382; EG-NEXT:     FFBL_INT * T1.W, T0.Y,
383; EG-NEXT:    32(4.484155e-44), 0(0.000000e+00)
384; EG-NEXT:     CNDE_INT T0.Y, T0.Y, literal.x, PV.W,
385; EG-NEXT:     FFBL_INT * T1.W, T0.X,
386; EG-NEXT:    32(4.484155e-44), 0(0.000000e+00)
387; EG-NEXT:     CNDE_INT T0.X, T0.X, literal.x, PV.W,
388; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.y,
389; EG-NEXT:    32(4.484155e-44), 2(2.802597e-45)
390;
391; GFX10-LABEL: v_cttz_v4i32:
392; GFX10:       ; %bb.0:
393; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
394; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 4, v0
395; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
396; GFX10-NEXT:    v_mov_b32_e32 v4, 0
397; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
398; GFX10-NEXT:    global_load_dwordx4 v[0:3], v0, s[2:3]
399; GFX10-NEXT:    s_waitcnt vmcnt(0)
400; GFX10-NEXT:    v_ffbl_b32_e32 v5, v3
401; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v3
402; GFX10-NEXT:    v_ffbl_b32_e32 v6, v2
403; GFX10-NEXT:    v_ffbl_b32_e32 v7, v1
404; GFX10-NEXT:    v_ffbl_b32_e32 v8, v0
405; GFX10-NEXT:    v_cndmask_b32_e32 v3, 32, v5, vcc_lo
406; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v2
407; GFX10-NEXT:    v_cndmask_b32_e32 v2, 32, v6, vcc_lo
408; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v1
409; GFX10-NEXT:    v_cndmask_b32_e32 v1, 32, v7, vcc_lo
410; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
411; GFX10-NEXT:    v_cndmask_b32_e32 v0, 32, v8, vcc_lo
412; GFX10-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
413; GFX10-NEXT:    s_endpgm
414;
415; GFX10-GISEL-LABEL: v_cttz_v4i32:
416; GFX10-GISEL:       ; %bb.0:
417; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
418; GFX10-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 4, v0
419; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
420; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
421; GFX10-GISEL-NEXT:    global_load_dwordx4 v[0:3], v0, s[2:3]
422; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0)
423; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v4, v0
424; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
425; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v5, v1
426; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v6, v2
427; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v7, v3
428; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v0, v4, 32, vcc_lo
429; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
430; GFX10-GISEL-NEXT:    v_mov_b32_e32 v4, 0
431; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v1, v5, 32, vcc_lo
432; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v2
433; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v2, v6, 32, vcc_lo
434; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
435; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v3, v7, 32, vcc_lo
436; GFX10-GISEL-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
437; GFX10-GISEL-NEXT:    s_endpgm
438  %tid = call i32 @llvm.amdgcn.workitem.id.x()
439  %in.gep = getelementptr <4 x i32>, <4 x i32> addrspace(1)* %valptr, i32 %tid
440  %val = load <4 x i32>, <4 x i32> addrspace(1)* %in.gep, align 16
441  %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %val, i1 false) nounwind readnone
442  store <4 x i32> %cttz, <4 x i32> addrspace(1)* %out, align 16
443  ret void
444}
445
446define amdgpu_kernel void @v_cttz_i8(i8 addrspace(1)* noalias %out, i8 addrspace(1)* noalias %valptr) nounwind {
447; SI-LABEL: v_cttz_i8:
448; SI:       ; %bb.0:
449; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xb
450; SI-NEXT:    s_mov_b32 s3, 0xf000
451; SI-NEXT:    s_mov_b32 s2, -1
452; SI-NEXT:    s_mov_b32 s6, s2
453; SI-NEXT:    s_mov_b32 s7, s3
454; SI-NEXT:    s_waitcnt lgkmcnt(0)
455; SI-NEXT:    buffer_load_ubyte v0, off, s[4:7], 0
456; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
457; SI-NEXT:    s_waitcnt vmcnt(0)
458; SI-NEXT:    v_or_b32_e32 v0, 0x100, v0
459; SI-NEXT:    v_ffbl_b32_e32 v0, v0
460; SI-NEXT:    s_waitcnt lgkmcnt(0)
461; SI-NEXT:    buffer_store_byte v0, off, s[0:3], 0
462; SI-NEXT:    s_endpgm
463;
464; VI-LABEL: v_cttz_i8:
465; VI:       ; %bb.0:
466; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
467; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
468; VI-NEXT:    s_mov_b32 s7, 0xf000
469; VI-NEXT:    s_mov_b32 s6, -1
470; VI-NEXT:    s_mov_b32 s2, s6
471; VI-NEXT:    s_mov_b32 s3, s7
472; VI-NEXT:    s_waitcnt lgkmcnt(0)
473; VI-NEXT:    buffer_load_ubyte v0, off, s[0:3], 0
474; VI-NEXT:    s_waitcnt vmcnt(0)
475; VI-NEXT:    v_or_b32_e32 v0, 0x100, v0
476; VI-NEXT:    v_ffbl_b32_e32 v0, v0
477; VI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
478; VI-NEXT:    s_endpgm
479;
480; EG-LABEL: v_cttz_i8:
481; EG:       ; %bb.0:
482; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
483; EG-NEXT:    TEX 0 @6
484; EG-NEXT:    ALU 14, @9, KC0[CB0:0-32], KC1[]
485; EG-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
486; EG-NEXT:    CF_END
487; EG-NEXT:    PAD
488; EG-NEXT:    Fetch clause starting at 6:
489; EG-NEXT:     VTX_READ_8 T0.X, T0.X, 0, #1
490; EG-NEXT:    ALU clause starting at 8:
491; EG-NEXT:     MOV * T0.X, KC0[2].Z,
492; EG-NEXT:    ALU clause starting at 9:
493; EG-NEXT:     OR_INT * T0.W, T0.X, literal.x,
494; EG-NEXT:    256(3.587324e-43), 0(0.000000e+00)
495; EG-NEXT:     FFBL_INT T0.W, PV.W,
496; EG-NEXT:     AND_INT * T1.W, KC0[2].Y, literal.x,
497; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
498; EG-NEXT:     AND_INT T0.W, PV.W, literal.x,
499; EG-NEXT:     LSHL * T1.W, PS, literal.y,
500; EG-NEXT:    255(3.573311e-43), 3(4.203895e-45)
501; EG-NEXT:     LSHL T0.X, PV.W, PS,
502; EG-NEXT:     LSHL * T0.W, literal.x, PS,
503; EG-NEXT:    255(3.573311e-43), 0(0.000000e+00)
504; EG-NEXT:     MOV T0.Y, 0.0,
505; EG-NEXT:     MOV * T0.Z, 0.0,
506; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
507; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
508;
509; GFX10-LABEL: v_cttz_i8:
510; GFX10:       ; %bb.0:
511; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
512; GFX10-NEXT:    v_mov_b32_e32 v0, 0
513; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
514; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
515; GFX10-NEXT:    global_load_ubyte v1, v0, s[2:3]
516; GFX10-NEXT:    s_waitcnt vmcnt(0)
517; GFX10-NEXT:    v_or_b32_e32 v1, 0x100, v1
518; GFX10-NEXT:    v_ffbl_b32_e32 v1, v1
519; GFX10-NEXT:    global_store_byte v0, v1, s[0:1]
520; GFX10-NEXT:    s_endpgm
521;
522; GFX10-GISEL-LABEL: v_cttz_i8:
523; GFX10-GISEL:       ; %bb.0:
524; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
525; GFX10-GISEL-NEXT:    v_mov_b32_e32 v0, 0
526; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
527; GFX10-GISEL-NEXT:    v_mov_b32_e32 v2, 0x100
528; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
529; GFX10-GISEL-NEXT:    global_load_ubyte v1, v0, s[2:3]
530; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0)
531; GFX10-GISEL-NEXT:    v_and_or_b32 v1, 0xff, v1, v2
532; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v1, v1
533; GFX10-GISEL-NEXT:    global_store_byte v0, v1, s[0:1]
534; GFX10-GISEL-NEXT:    s_endpgm
535  %val = load i8, i8 addrspace(1)* %valptr
536  %cttz = call i8 @llvm.cttz.i8(i8 %val, i1 false) nounwind readnone
537  store i8 %cttz, i8 addrspace(1)* %out
538  ret void
539}
540
541define amdgpu_kernel void @s_cttz_i64(i64 addrspace(1)* noalias %out, [8 x i32], i64 %val) nounwind {
542; SI-LABEL: s_cttz_i64:
543; SI:       ; %bb.0:
544; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x13
545; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
546; SI-NEXT:    s_mov_b32 s3, 0xf000
547; SI-NEXT:    s_mov_b32 s2, -1
548; SI-NEXT:    s_waitcnt lgkmcnt(0)
549; SI-NEXT:    s_ff1_i32_b32 s6, s5
550; SI-NEXT:    s_ff1_i32_b32 s7, s4
551; SI-NEXT:    s_add_i32 s6, s6, 32
552; SI-NEXT:    s_or_b32 s5, s5, s4
553; SI-NEXT:    v_mov_b32_e32 v0, s7
554; SI-NEXT:    v_mov_b32_e32 v1, s6
555; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s4, 0
556; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
557; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s5, 0
558; SI-NEXT:    v_cndmask_b32_e32 v0, 64, v0, vcc
559; SI-NEXT:    v_mov_b32_e32 v1, 0
560; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
561; SI-NEXT:    s_endpgm
562;
563; VI-LABEL: s_cttz_i64:
564; VI:       ; %bb.0:
565; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
566; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x4c
567; VI-NEXT:    s_mov_b32 s7, 0xf000
568; VI-NEXT:    s_mov_b32 s6, -1
569; VI-NEXT:    v_mov_b32_e32 v1, 0
570; VI-NEXT:    s_waitcnt lgkmcnt(0)
571; VI-NEXT:    s_ff1_i32_b32 s2, s1
572; VI-NEXT:    s_add_i32 s2, s2, 32
573; VI-NEXT:    s_ff1_i32_b32 s3, s0
574; VI-NEXT:    s_cmp_eq_u32 s0, 0
575; VI-NEXT:    s_cselect_b32 s2, s2, s3
576; VI-NEXT:    s_or_b32 s0, s1, s0
577; VI-NEXT:    s_cmp_lg_u32 s0, 0
578; VI-NEXT:    s_cselect_b32 s0, s2, 64
579; VI-NEXT:    v_mov_b32_e32 v0, s0
580; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
581; VI-NEXT:    s_endpgm
582;
583; EG-LABEL: s_cttz_i64:
584; EG:       ; %bb.0:
585; EG-NEXT:    ALU 9, @4, KC0[CB0:0-32], KC1[]
586; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
587; EG-NEXT:    CF_END
588; EG-NEXT:    PAD
589; EG-NEXT:    ALU clause starting at 4:
590; EG-NEXT:     FFBL_INT * T0.W, KC0[5].X,
591; EG-NEXT:     CNDE_INT * T0.W, KC0[5].X, literal.x, PV.W,
592; EG-NEXT:    32(4.484155e-44), 0(0.000000e+00)
593; EG-NEXT:     FFBL_INT T1.W, KC0[4].W,
594; EG-NEXT:     ADD_INT * T0.W, PV.W, literal.x,
595; EG-NEXT:    32(4.484155e-44), 0(0.000000e+00)
596; EG-NEXT:     CNDE_INT T0.X, KC0[4].W, PS, PV.W,
597; EG-NEXT:     MOV T0.Y, 0.0,
598; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
599; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
600;
601; GFX10-LABEL: s_cttz_i64:
602; GFX10:       ; %bb.0:
603; GFX10-NEXT:    s_clause 0x1
604; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x4c
605; GFX10-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
606; GFX10-NEXT:    v_mov_b32_e32 v1, 0
607; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
608; GFX10-NEXT:    s_ff1_i32_b32 s0, s3
609; GFX10-NEXT:    s_ff1_i32_b32 s1, s2
610; GFX10-NEXT:    s_add_i32 s0, s0, 32
611; GFX10-NEXT:    s_cmp_eq_u32 s2, 0
612; GFX10-NEXT:    s_cselect_b32 s0, s0, s1
613; GFX10-NEXT:    s_or_b32 s1, s3, s2
614; GFX10-NEXT:    s_cmp_lg_u32 s1, 0
615; GFX10-NEXT:    s_cselect_b32 s0, s0, 64
616; GFX10-NEXT:    v_mov_b32_e32 v0, s0
617; GFX10-NEXT:    global_store_dwordx2 v1, v[0:1], s[4:5]
618; GFX10-NEXT:    s_endpgm
619;
620; GFX10-GISEL-LABEL: s_cttz_i64:
621; GFX10-GISEL:       ; %bb.0:
622; GFX10-GISEL-NEXT:    s_clause 0x1
623; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x4c
624; GFX10-GISEL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
625; GFX10-GISEL-NEXT:    v_mov_b32_e32 v2, 0
626; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
627; GFX10-GISEL-NEXT:    s_ff1_i32_b64 s0, s[2:3]
628; GFX10-GISEL-NEXT:    s_cmp_eq_u64 s[2:3], 0
629; GFX10-GISEL-NEXT:    s_cselect_b32 s0, 64, s0
630; GFX10-GISEL-NEXT:    s_bfe_u64 s[0:1], s[0:1], 0x200000
631; GFX10-GISEL-NEXT:    v_mov_b32_e32 v0, s0
632; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, s1
633; GFX10-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[4:5]
634; GFX10-GISEL-NEXT:    s_endpgm
635  %cttz = call i64 @llvm.cttz.i64(i64 %val, i1 false)
636  store i64 %cttz, i64 addrspace(1)* %out
637  ret void
638}
639
640define amdgpu_kernel void @s_cttz_i64_trunc(i32 addrspace(1)* noalias %out, i64 %val) nounwind {
641; SI-LABEL: s_cttz_i64_trunc:
642; SI:       ; %bb.0:
643; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xb
644; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
645; SI-NEXT:    s_mov_b32 s3, 0xf000
646; SI-NEXT:    s_mov_b32 s2, -1
647; SI-NEXT:    s_waitcnt lgkmcnt(0)
648; SI-NEXT:    s_ff1_i32_b32 s6, s5
649; SI-NEXT:    s_ff1_i32_b32 s7, s4
650; SI-NEXT:    s_add_i32 s6, s6, 32
651; SI-NEXT:    s_or_b32 s5, s5, s4
652; SI-NEXT:    v_mov_b32_e32 v0, s7
653; SI-NEXT:    v_mov_b32_e32 v1, s6
654; SI-NEXT:    v_cmp_eq_u32_e64 vcc, s4, 0
655; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
656; SI-NEXT:    v_cmp_ne_u32_e64 vcc, s5, 0
657; SI-NEXT:    v_cndmask_b32_e32 v0, 64, v0, vcc
658; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
659; SI-NEXT:    s_endpgm
660;
661; VI-LABEL: s_cttz_i64_trunc:
662; VI:       ; %bb.0:
663; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
664; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
665; VI-NEXT:    s_mov_b32 s7, 0xf000
666; VI-NEXT:    s_mov_b32 s6, -1
667; VI-NEXT:    s_waitcnt lgkmcnt(0)
668; VI-NEXT:    s_ff1_i32_b32 s2, s1
669; VI-NEXT:    s_add_i32 s2, s2, 32
670; VI-NEXT:    s_ff1_i32_b32 s3, s0
671; VI-NEXT:    s_cmp_eq_u32 s0, 0
672; VI-NEXT:    s_cselect_b32 s2, s2, s3
673; VI-NEXT:    s_or_b32 s0, s1, s0
674; VI-NEXT:    s_cmp_lg_u32 s0, 0
675; VI-NEXT:    s_cselect_b32 s0, s2, 64
676; VI-NEXT:    v_mov_b32_e32 v0, s0
677; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
678; VI-NEXT:    s_endpgm
679;
680; EG-LABEL: s_cttz_i64_trunc:
681; EG:       ; %bb.0:
682; EG-NEXT:    ALU 8, @4, KC0[CB0:0-32], KC1[]
683; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
684; EG-NEXT:    CF_END
685; EG-NEXT:    PAD
686; EG-NEXT:    ALU clause starting at 4:
687; EG-NEXT:     FFBL_INT * T0.W, KC0[3].X,
688; EG-NEXT:     CNDE_INT * T0.W, KC0[3].X, literal.x, PV.W,
689; EG-NEXT:    32(4.484155e-44), 0(0.000000e+00)
690; EG-NEXT:     FFBL_INT T1.W, KC0[2].W,
691; EG-NEXT:     ADD_INT * T0.W, PV.W, literal.x,
692; EG-NEXT:    32(4.484155e-44), 0(0.000000e+00)
693; EG-NEXT:     CNDE_INT T0.X, KC0[2].W, PS, PV.W,
694; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
695; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
696;
697; GFX10-LABEL: s_cttz_i64_trunc:
698; GFX10:       ; %bb.0:
699; GFX10-NEXT:    s_clause 0x1
700; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
701; GFX10-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
702; GFX10-NEXT:    v_mov_b32_e32 v0, 0
703; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
704; GFX10-NEXT:    s_ff1_i32_b32 s0, s3
705; GFX10-NEXT:    s_ff1_i32_b32 s1, s2
706; GFX10-NEXT:    s_add_i32 s0, s0, 32
707; GFX10-NEXT:    s_cmp_eq_u32 s2, 0
708; GFX10-NEXT:    s_cselect_b32 s0, s0, s1
709; GFX10-NEXT:    s_or_b32 s1, s3, s2
710; GFX10-NEXT:    s_cmp_lg_u32 s1, 0
711; GFX10-NEXT:    s_cselect_b32 s0, s0, 64
712; GFX10-NEXT:    v_mov_b32_e32 v1, s0
713; GFX10-NEXT:    global_store_dword v0, v1, s[4:5]
714; GFX10-NEXT:    s_endpgm
715;
716; GFX10-GISEL-LABEL: s_cttz_i64_trunc:
717; GFX10-GISEL:       ; %bb.0:
718; GFX10-GISEL-NEXT:    s_clause 0x1
719; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
720; GFX10-GISEL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
721; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, 0
722; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
723; GFX10-GISEL-NEXT:    s_ff1_i32_b64 s0, s[2:3]
724; GFX10-GISEL-NEXT:    s_cmp_eq_u64 s[2:3], 0
725; GFX10-GISEL-NEXT:    s_cselect_b32 s0, 64, s0
726; GFX10-GISEL-NEXT:    v_mov_b32_e32 v0, s0
727; GFX10-GISEL-NEXT:    global_store_dword v1, v0, s[4:5]
728; GFX10-GISEL-NEXT:    s_endpgm
729  %cttz = call i64 @llvm.cttz.i64(i64 %val, i1 false)
730  %trunc = trunc i64 %cttz to i32
731  store i32 %trunc, i32 addrspace(1)* %out
732  ret void
733}
734
735define amdgpu_kernel void @v_cttz_i64(i64 addrspace(1)* noalias %out, i64 addrspace(1)* noalias %in) nounwind {
736; SI-LABEL: v_cttz_i64:
737; SI:       ; %bb.0:
738; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xb
739; SI-NEXT:    s_mov_b32 s7, 0xf000
740; SI-NEXT:    s_mov_b32 s6, 0
741; SI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
742; SI-NEXT:    v_mov_b32_e32 v1, 0
743; SI-NEXT:    s_waitcnt lgkmcnt(0)
744; SI-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64
745; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
746; SI-NEXT:    s_waitcnt vmcnt(0)
747; SI-NEXT:    v_ffbl_b32_e32 v4, v3
748; SI-NEXT:    v_ffbl_b32_e32 v5, v2
749; SI-NEXT:    v_or_b32_e32 v3, v3, v2
750; SI-NEXT:    v_add_i32_e32 v4, vcc, 32, v4
751; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
752; SI-NEXT:    v_cndmask_b32_e32 v2, v5, v4, vcc
753; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v3
754; SI-NEXT:    v_cndmask_b32_e32 v2, 64, v2, vcc
755; SI-NEXT:    v_mov_b32_e32 v3, v1
756; SI-NEXT:    s_waitcnt lgkmcnt(0)
757; SI-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64
758; SI-NEXT:    s_endpgm
759;
760; VI-LABEL: v_cttz_i64:
761; VI:       ; %bb.0:
762; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
763; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
764; VI-NEXT:    v_lshlrev_b32_e32 v3, 3, v0
765; VI-NEXT:    v_mov_b32_e32 v4, 0
766; VI-NEXT:    v_mov_b32_e32 v2, 0
767; VI-NEXT:    s_waitcnt lgkmcnt(0)
768; VI-NEXT:    v_mov_b32_e32 v5, s3
769; VI-NEXT:    v_mov_b32_e32 v1, s1
770; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v3
771; VI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v4, vcc
772; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
773; VI-NEXT:    v_add_u32_e32 v3, vcc, s2, v3
774; VI-NEXT:    v_addc_u32_e32 v4, vcc, v5, v4, vcc
775; VI-NEXT:    s_waitcnt vmcnt(0)
776; VI-NEXT:    v_ffbl_b32_e32 v5, v1
777; VI-NEXT:    v_add_u32_e32 v5, vcc, 32, v5
778; VI-NEXT:    v_ffbl_b32_e32 v6, v0
779; VI-NEXT:    v_or_b32_e32 v1, v1, v0
780; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
781; VI-NEXT:    v_cndmask_b32_e32 v0, v6, v5, vcc
782; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
783; VI-NEXT:    v_cndmask_b32_e32 v1, 64, v0, vcc
784; VI-NEXT:    flat_store_dwordx2 v[3:4], v[1:2]
785; VI-NEXT:    s_endpgm
786;
787; EG-LABEL: v_cttz_i64:
788; EG:       ; %bb.0:
789; EG-NEXT:    ALU 2, @8, KC0[CB0:0-32], KC1[]
790; EG-NEXT:    TEX 0 @6
791; EG-NEXT:    ALU 10, @11, KC0[CB0:0-32], KC1[]
792; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
793; EG-NEXT:    CF_END
794; EG-NEXT:    PAD
795; EG-NEXT:    Fetch clause starting at 6:
796; EG-NEXT:     VTX_READ_64 T0.XY, T0.X, 0, #1
797; EG-NEXT:    ALU clause starting at 8:
798; EG-NEXT:     LSHL * T0.W, T0.X, literal.x,
799; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
800; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, PV.W,
801; EG-NEXT:    ALU clause starting at 11:
802; EG-NEXT:     FFBL_INT * T1.W, T0.Y,
803; EG-NEXT:     CNDE_INT * T1.W, T0.Y, literal.x, PV.W,
804; EG-NEXT:    32(4.484155e-44), 0(0.000000e+00)
805; EG-NEXT:     FFBL_INT T2.W, T0.X,
806; EG-NEXT:     ADD_INT * T1.W, PV.W, literal.x,
807; EG-NEXT:    32(4.484155e-44), 0(0.000000e+00)
808; EG-NEXT:     CNDE_INT T0.X, T0.X, PS, PV.W,
809; EG-NEXT:     MOV T0.Y, 0.0,
810; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, T0.W,
811; EG-NEXT:     LSHR * T1.X, PV.W, literal.x,
812; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
813;
814; GFX10-LABEL: v_cttz_i64:
815; GFX10:       ; %bb.0:
816; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
817; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
818; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
819; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
820; GFX10-NEXT:    global_load_dwordx2 v[0:1], v2, s[2:3]
821; GFX10-NEXT:    s_waitcnt vmcnt(0)
822; GFX10-NEXT:    v_ffbl_b32_e32 v3, v1
823; GFX10-NEXT:    v_ffbl_b32_e32 v4, v0
824; GFX10-NEXT:    v_or_b32_e32 v1, v1, v0
825; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
826; GFX10-NEXT:    v_add_nc_u32_e32 v3, 32, v3
827; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v3, vcc_lo
828; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v1
829; GFX10-NEXT:    v_mov_b32_e32 v1, 0
830; GFX10-NEXT:    v_cndmask_b32_e32 v0, 64, v0, vcc_lo
831; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
832; GFX10-NEXT:    s_endpgm
833;
834; GFX10-GISEL-LABEL: v_cttz_i64:
835; GFX10-GISEL:       ; %bb.0:
836; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
837; GFX10-GISEL-NEXT:    v_lshlrev_b32_e32 v3, 3, v0
838; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
839; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
840; GFX10-GISEL-NEXT:    global_load_dwordx2 v[0:1], v3, s[2:3]
841; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0)
842; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v2, v1
843; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v4, v0
844; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
845; GFX10-GISEL-NEXT:    v_add_nc_u32_e32 v2, 32, v2
846; GFX10-GISEL-NEXT:    v_cndmask_b32_e32 v4, v4, v2, vcc_lo
847; GFX10-GISEL-NEXT:    v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]
848; GFX10-GISEL-NEXT:    v_mov_b32_e32 v2, 0
849; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v1, v4, 64, vcc_lo
850; GFX10-GISEL-NEXT:    global_store_dwordx2 v3, v[1:2], s[0:1]
851; GFX10-GISEL-NEXT:    s_endpgm
852  %tid = call i32 @llvm.amdgcn.workitem.id.x()
853  %in.gep = getelementptr i64, i64 addrspace(1)* %in, i32 %tid
854  %out.gep = getelementptr i64, i64 addrspace(1)* %out, i32 %tid
855  %val = load i64, i64 addrspace(1)* %in.gep
856  %cttz = call i64 @llvm.cttz.i64(i64 %val, i1 false)
857  store i64 %cttz, i64 addrspace(1)* %out.gep
858  ret void
859}
860
861define amdgpu_kernel void @v_cttz_i64_trunc(i32 addrspace(1)* noalias %out, i64 addrspace(1)* noalias %in) nounwind {
862; SI-LABEL: v_cttz_i64_trunc:
863; SI:       ; %bb.0:
864; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xb
865; SI-NEXT:    s_mov_b32 s7, 0xf000
866; SI-NEXT:    s_mov_b32 s6, 0
867; SI-NEXT:    v_lshlrev_b32_e32 v1, 3, v0
868; SI-NEXT:    v_mov_b32_e32 v2, 0
869; SI-NEXT:    s_waitcnt lgkmcnt(0)
870; SI-NEXT:    buffer_load_dwordx2 v[3:4], v[1:2], s[4:7], 0 addr64
871; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
872; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
873; SI-NEXT:    s_waitcnt vmcnt(0)
874; SI-NEXT:    v_ffbl_b32_e32 v0, v4
875; SI-NEXT:    v_ffbl_b32_e32 v5, v3
876; SI-NEXT:    v_or_b32_e32 v4, v4, v3
877; SI-NEXT:    v_add_i32_e32 v0, vcc, 32, v0
878; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
879; SI-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
880; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
881; SI-NEXT:    v_cndmask_b32_e32 v0, 64, v0, vcc
882; SI-NEXT:    s_waitcnt lgkmcnt(0)
883; SI-NEXT:    buffer_store_dword v0, v[1:2], s[4:7], 0 addr64
884; SI-NEXT:    s_endpgm
885;
886; VI-LABEL: v_cttz_i64_trunc:
887; VI:       ; %bb.0:
888; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
889; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
890; VI-NEXT:    v_lshlrev_b32_e32 v1, 3, v0
891; VI-NEXT:    v_mov_b32_e32 v4, 0
892; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
893; VI-NEXT:    s_waitcnt lgkmcnt(0)
894; VI-NEXT:    v_mov_b32_e32 v5, s3
895; VI-NEXT:    v_mov_b32_e32 v2, s1
896; VI-NEXT:    v_add_u32_e32 v1, vcc, s0, v1
897; VI-NEXT:    v_addc_u32_e32 v2, vcc, v2, v4, vcc
898; VI-NEXT:    flat_load_dwordx2 v[1:2], v[1:2]
899; VI-NEXT:    v_add_u32_e32 v3, vcc, s2, v0
900; VI-NEXT:    v_addc_u32_e32 v4, vcc, v5, v4, vcc
901; VI-NEXT:    s_waitcnt vmcnt(0)
902; VI-NEXT:    v_ffbl_b32_e32 v0, v2
903; VI-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
904; VI-NEXT:    v_ffbl_b32_e32 v5, v1
905; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
906; VI-NEXT:    v_or_b32_e32 v2, v2, v1
907; VI-NEXT:    v_cndmask_b32_e32 v0, v5, v0, vcc
908; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
909; VI-NEXT:    v_cndmask_b32_e32 v0, 64, v0, vcc
910; VI-NEXT:    flat_store_dword v[3:4], v0
911; VI-NEXT:    s_endpgm
912;
913; EG-LABEL: v_cttz_i64_trunc:
914; EG:       ; %bb.0:
915; EG-NEXT:    ALU 2, @8, KC0[CB0:0-32], KC1[]
916; EG-NEXT:    TEX 0 @6
917; EG-NEXT:    ALU 10, @11, KC0[CB0:0-32], KC1[]
918; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
919; EG-NEXT:    CF_END
920; EG-NEXT:    PAD
921; EG-NEXT:    Fetch clause starting at 6:
922; EG-NEXT:     VTX_READ_64 T1.XY, T1.X, 0, #1
923; EG-NEXT:    ALU clause starting at 8:
924; EG-NEXT:     LSHL * T0.W, T0.X, literal.x,
925; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
926; EG-NEXT:     ADD_INT * T1.X, KC0[2].Z, PV.W,
927; EG-NEXT:    ALU clause starting at 11:
928; EG-NEXT:     FFBL_INT * T0.W, T1.Y,
929; EG-NEXT:     CNDE_INT * T0.W, T1.Y, literal.x, PV.W,
930; EG-NEXT:    32(4.484155e-44), 0(0.000000e+00)
931; EG-NEXT:     LSHL T0.Z, T0.X, literal.x,
932; EG-NEXT:     FFBL_INT T1.W, T1.X, BS:VEC_120/SCL_212
933; EG-NEXT:     ADD_INT * T0.W, PV.W, literal.y,
934; EG-NEXT:    2(2.802597e-45), 32(4.484155e-44)
935; EG-NEXT:     CNDE_INT T0.X, T1.X, PS, PV.W,
936; EG-NEXT:     ADD_INT * T0.W, KC0[2].Y, PV.Z,
937; EG-NEXT:     LSHR * T1.X, PV.W, literal.x,
938; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
939;
940; GFX10-LABEL: v_cttz_i64_trunc:
941; GFX10:       ; %bb.0:
942; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
943; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 3, v0
944; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
945; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
946; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
947; GFX10-NEXT:    global_load_dwordx2 v[1:2], v1, s[2:3]
948; GFX10-NEXT:    s_waitcnt vmcnt(0)
949; GFX10-NEXT:    v_ffbl_b32_e32 v3, v2
950; GFX10-NEXT:    v_ffbl_b32_e32 v4, v1
951; GFX10-NEXT:    v_or_b32_e32 v2, v2, v1
952; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
953; GFX10-NEXT:    v_add_nc_u32_e32 v3, 32, v3
954; GFX10-NEXT:    v_cndmask_b32_e32 v1, v4, v3, vcc_lo
955; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v2
956; GFX10-NEXT:    v_cndmask_b32_e32 v1, 64, v1, vcc_lo
957; GFX10-NEXT:    global_store_dword v0, v1, s[0:1]
958; GFX10-NEXT:    s_endpgm
959;
960; GFX10-GISEL-LABEL: v_cttz_i64_trunc:
961; GFX10-GISEL:       ; %bb.0:
962; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
963; GFX10-GISEL-NEXT:    v_lshlrev_b32_e32 v1, 3, v0
964; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
965; GFX10-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
966; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
967; GFX10-GISEL-NEXT:    global_load_dwordx2 v[1:2], v1, s[2:3]
968; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0)
969; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v3, v2
970; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v4, v1
971; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
972; GFX10-GISEL-NEXT:    v_add_nc_u32_e32 v3, 32, v3
973; GFX10-GISEL-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc_lo
974; GFX10-GISEL-NEXT:    v_cmp_eq_u64_e32 vcc_lo, 0, v[1:2]
975; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v1, v3, 64, vcc_lo
976; GFX10-GISEL-NEXT:    global_store_dword v0, v1, s[0:1]
977; GFX10-GISEL-NEXT:    s_endpgm
978  %tid = call i32 @llvm.amdgcn.workitem.id.x()
979  %in.gep = getelementptr i64, i64 addrspace(1)* %in, i32 %tid
980  %out.gep = getelementptr i32, i32 addrspace(1)* %out, i32 %tid
981  %val = load i64, i64 addrspace(1)* %in.gep
982  %cttz = call i64 @llvm.cttz.i64(i64 %val, i1 false)
983  %trunc = trunc i64 %cttz to i32
984  store i32 %trunc, i32 addrspace(1)* %out.gep
985  ret void
986}
987
988define amdgpu_kernel void @v_cttz_i32_sel_eq_neg1(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalias %valptr) nounwind {
989; SI-LABEL: v_cttz_i32_sel_eq_neg1:
990; SI:       ; %bb.0:
991; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xb
992; SI-NEXT:    s_mov_b32 s3, 0xf000
993; SI-NEXT:    s_mov_b32 s6, 0
994; SI-NEXT:    s_mov_b32 s7, s3
995; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
996; SI-NEXT:    v_mov_b32_e32 v1, 0
997; SI-NEXT:    s_waitcnt lgkmcnt(0)
998; SI-NEXT:    buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
999; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
1000; SI-NEXT:    s_mov_b32 s2, -1
1001; SI-NEXT:    s_waitcnt vmcnt(0)
1002; SI-NEXT:    v_ffbl_b32_e32 v0, v0
1003; SI-NEXT:    s_waitcnt lgkmcnt(0)
1004; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1005; SI-NEXT:    s_endpgm
1006;
1007; VI-LABEL: v_cttz_i32_sel_eq_neg1:
1008; VI:       ; %bb.0:
1009; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
1010; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
1011; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1012; VI-NEXT:    s_mov_b32 s7, 0xf000
1013; VI-NEXT:    s_mov_b32 s6, -1
1014; VI-NEXT:    s_waitcnt lgkmcnt(0)
1015; VI-NEXT:    v_mov_b32_e32 v1, s1
1016; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
1017; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
1018; VI-NEXT:    flat_load_dword v0, v[0:1]
1019; VI-NEXT:    s_waitcnt vmcnt(0)
1020; VI-NEXT:    v_ffbl_b32_e32 v0, v0
1021; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
1022; VI-NEXT:    s_endpgm
1023;
1024; EG-LABEL: v_cttz_i32_sel_eq_neg1:
1025; EG:       ; %bb.0:
1026; EG-NEXT:    ALU 2, @8, KC0[CB0:0-32], KC1[]
1027; EG-NEXT:    TEX 0 @6
1028; EG-NEXT:    ALU 5, @11, KC0[CB0:0-32], KC1[]
1029; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
1030; EG-NEXT:    CF_END
1031; EG-NEXT:    PAD
1032; EG-NEXT:    Fetch clause starting at 6:
1033; EG-NEXT:     VTX_READ_32 T0.X, T0.X, 0, #1
1034; EG-NEXT:    ALU clause starting at 8:
1035; EG-NEXT:     LSHL * T0.W, T0.X, literal.x,
1036; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
1037; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, PV.W,
1038; EG-NEXT:    ALU clause starting at 11:
1039; EG-NEXT:     FFBL_INT * T0.W, T0.X,
1040; EG-NEXT:     CNDE_INT * T0.W, T0.X, literal.x, PV.W,
1041; EG-NEXT:    32(4.484155e-44), 0(0.000000e+00)
1042; EG-NEXT:     CNDE_INT T0.X, T0.X, literal.x, PV.W,
1043; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.y,
1044; EG-NEXT:    -1(nan), 2(2.802597e-45)
1045;
1046; GFX10-LABEL: v_cttz_i32_sel_eq_neg1:
1047; GFX10:       ; %bb.0:
1048; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
1049; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1050; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1051; GFX10-NEXT:    v_mov_b32_e32 v1, 0
1052; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
1053; GFX10-NEXT:    global_load_dword v0, v0, s[2:3]
1054; GFX10-NEXT:    s_waitcnt vmcnt(0)
1055; GFX10-NEXT:    v_ffbl_b32_e32 v0, v0
1056; GFX10-NEXT:    global_store_dword v1, v0, s[0:1]
1057; GFX10-NEXT:    s_endpgm
1058;
1059; GFX10-GISEL-LABEL: v_cttz_i32_sel_eq_neg1:
1060; GFX10-GISEL:       ; %bb.0:
1061; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
1062; GFX10-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1063; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1064; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
1065; GFX10-GISEL-NEXT:    global_load_dword v0, v0, s[2:3]
1066; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0)
1067; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v1, v0
1068; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
1069; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v0, v1, 32, vcc_lo
1070; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, 0
1071; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v0, v0, -1, vcc_lo
1072; GFX10-GISEL-NEXT:    global_store_dword v1, v0, s[0:1]
1073; GFX10-GISEL-NEXT:    s_endpgm
1074  %tid = call i32 @llvm.amdgcn.workitem.id.x()
1075  %in.gep = getelementptr i32, i32 addrspace(1)* %valptr, i32 %tid
1076  %val = load i32, i32 addrspace(1)* %in.gep
1077  %cttz = call i32 @llvm.cttz.i32(i32 %val, i1 false) nounwind readnone
1078  %cmp = icmp eq i32 %val, 0
1079  %sel = select i1 %cmp, i32 -1, i32 %cttz
1080  store i32 %sel, i32 addrspace(1)* %out
1081  ret void
1082}
1083
1084define amdgpu_kernel void @v_cttz_i32_sel_ne_neg1(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalias %valptr) nounwind {
1085; SI-LABEL: v_cttz_i32_sel_ne_neg1:
1086; SI:       ; %bb.0:
1087; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xb
1088; SI-NEXT:    s_mov_b32 s3, 0xf000
1089; SI-NEXT:    s_mov_b32 s6, 0
1090; SI-NEXT:    s_mov_b32 s7, s3
1091; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1092; SI-NEXT:    v_mov_b32_e32 v1, 0
1093; SI-NEXT:    s_waitcnt lgkmcnt(0)
1094; SI-NEXT:    buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
1095; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
1096; SI-NEXT:    s_mov_b32 s2, -1
1097; SI-NEXT:    s_waitcnt vmcnt(0)
1098; SI-NEXT:    v_ffbl_b32_e32 v0, v0
1099; SI-NEXT:    s_waitcnt lgkmcnt(0)
1100; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1101; SI-NEXT:    s_endpgm
1102;
1103; VI-LABEL: v_cttz_i32_sel_ne_neg1:
1104; VI:       ; %bb.0:
1105; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
1106; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
1107; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1108; VI-NEXT:    s_mov_b32 s7, 0xf000
1109; VI-NEXT:    s_mov_b32 s6, -1
1110; VI-NEXT:    s_waitcnt lgkmcnt(0)
1111; VI-NEXT:    v_mov_b32_e32 v1, s1
1112; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
1113; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
1114; VI-NEXT:    flat_load_dword v0, v[0:1]
1115; VI-NEXT:    s_waitcnt vmcnt(0)
1116; VI-NEXT:    v_ffbl_b32_e32 v0, v0
1117; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
1118; VI-NEXT:    s_endpgm
1119;
1120; EG-LABEL: v_cttz_i32_sel_ne_neg1:
1121; EG:       ; %bb.0:
1122; EG-NEXT:    ALU 2, @8, KC0[CB0:0-32], KC1[]
1123; EG-NEXT:    TEX 0 @6
1124; EG-NEXT:    ALU 5, @11, KC0[CB0:0-32], KC1[]
1125; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
1126; EG-NEXT:    CF_END
1127; EG-NEXT:    PAD
1128; EG-NEXT:    Fetch clause starting at 6:
1129; EG-NEXT:     VTX_READ_32 T0.X, T0.X, 0, #1
1130; EG-NEXT:    ALU clause starting at 8:
1131; EG-NEXT:     LSHL * T0.W, T0.X, literal.x,
1132; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
1133; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, PV.W,
1134; EG-NEXT:    ALU clause starting at 11:
1135; EG-NEXT:     FFBL_INT * T0.W, T0.X,
1136; EG-NEXT:     CNDE_INT * T0.W, T0.X, literal.x, PV.W,
1137; EG-NEXT:    32(4.484155e-44), 0(0.000000e+00)
1138; EG-NEXT:     CNDE_INT T0.X, T0.X, literal.x, PV.W,
1139; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.y,
1140; EG-NEXT:    -1(nan), 2(2.802597e-45)
1141;
1142; GFX10-LABEL: v_cttz_i32_sel_ne_neg1:
1143; GFX10:       ; %bb.0:
1144; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
1145; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1146; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1147; GFX10-NEXT:    v_mov_b32_e32 v1, 0
1148; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
1149; GFX10-NEXT:    global_load_dword v0, v0, s[2:3]
1150; GFX10-NEXT:    s_waitcnt vmcnt(0)
1151; GFX10-NEXT:    v_ffbl_b32_e32 v0, v0
1152; GFX10-NEXT:    global_store_dword v1, v0, s[0:1]
1153; GFX10-NEXT:    s_endpgm
1154;
1155; GFX10-GISEL-LABEL: v_cttz_i32_sel_ne_neg1:
1156; GFX10-GISEL:       ; %bb.0:
1157; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
1158; GFX10-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1159; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1160; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
1161; GFX10-GISEL-NEXT:    global_load_dword v0, v0, s[2:3]
1162; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0)
1163; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v1, v0
1164; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
1165; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v1, v1, 32, vcc_lo
1166; GFX10-GISEL-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
1167; GFX10-GISEL-NEXT:    v_cndmask_b32_e32 v0, -1, v1, vcc_lo
1168; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, 0
1169; GFX10-GISEL-NEXT:    global_store_dword v1, v0, s[0:1]
1170; GFX10-GISEL-NEXT:    s_endpgm
1171  %tid = call i32 @llvm.amdgcn.workitem.id.x()
1172  %in.gep = getelementptr i32, i32 addrspace(1)* %valptr, i32 %tid
1173  %val = load i32, i32 addrspace(1)* %in.gep
1174  %cttz = call i32 @llvm.cttz.i32(i32 %val, i1 false) nounwind readnone
1175  %cmp = icmp ne i32 %val, 0
1176  %sel = select i1 %cmp, i32 %cttz, i32 -1
1177  store i32 %sel, i32 addrspace(1)* %out
1178  ret void
1179}
1180
1181; TODO: Should be able to eliminate select here as well.
1182define amdgpu_kernel void @v_cttz_i32_sel_eq_bitwidth(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalias %valptr) nounwind {
1183; SI-LABEL: v_cttz_i32_sel_eq_bitwidth:
1184; SI:       ; %bb.0:
1185; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xb
1186; SI-NEXT:    s_mov_b32 s3, 0xf000
1187; SI-NEXT:    s_mov_b32 s6, 0
1188; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1189; SI-NEXT:    v_mov_b32_e32 v1, 0
1190; SI-NEXT:    s_mov_b32 s7, s3
1191; SI-NEXT:    s_waitcnt lgkmcnt(0)
1192; SI-NEXT:    buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
1193; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
1194; SI-NEXT:    s_mov_b32 s2, -1
1195; SI-NEXT:    s_waitcnt vmcnt(0)
1196; SI-NEXT:    v_ffbl_b32_e32 v1, v0
1197; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
1198; SI-NEXT:    v_cndmask_b32_e32 v0, 32, v1, vcc
1199; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 32, v0
1200; SI-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
1201; SI-NEXT:    s_waitcnt lgkmcnt(0)
1202; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1203; SI-NEXT:    s_endpgm
1204;
1205; VI-LABEL: v_cttz_i32_sel_eq_bitwidth:
1206; VI:       ; %bb.0:
1207; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
1208; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
1209; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1210; VI-NEXT:    s_mov_b32 s7, 0xf000
1211; VI-NEXT:    s_mov_b32 s6, -1
1212; VI-NEXT:    s_waitcnt lgkmcnt(0)
1213; VI-NEXT:    v_mov_b32_e32 v1, s1
1214; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
1215; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
1216; VI-NEXT:    flat_load_dword v0, v[0:1]
1217; VI-NEXT:    s_waitcnt vmcnt(0)
1218; VI-NEXT:    v_ffbl_b32_e32 v1, v0
1219; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
1220; VI-NEXT:    v_cndmask_b32_e32 v0, 32, v1, vcc
1221; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 32, v0
1222; VI-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
1223; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
1224; VI-NEXT:    s_endpgm
1225;
1226; EG-LABEL: v_cttz_i32_sel_eq_bitwidth:
1227; EG:       ; %bb.0:
1228; EG-NEXT:    ALU 2, @8, KC0[CB0:0-32], KC1[]
1229; EG-NEXT:    TEX 0 @6
1230; EG-NEXT:    ALU 7, @11, KC0[CB0:0-32], KC1[]
1231; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
1232; EG-NEXT:    CF_END
1233; EG-NEXT:    PAD
1234; EG-NEXT:    Fetch clause starting at 6:
1235; EG-NEXT:     VTX_READ_32 T0.X, T0.X, 0, #1
1236; EG-NEXT:    ALU clause starting at 8:
1237; EG-NEXT:     LSHL * T0.W, T0.X, literal.x,
1238; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
1239; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, PV.W,
1240; EG-NEXT:    ALU clause starting at 11:
1241; EG-NEXT:     FFBL_INT * T0.W, T0.X,
1242; EG-NEXT:     CNDE_INT * T0.W, T0.X, literal.x, PV.W,
1243; EG-NEXT:    32(4.484155e-44), 0(0.000000e+00)
1244; EG-NEXT:     SETE_INT * T1.W, PV.W, literal.x,
1245; EG-NEXT:    32(4.484155e-44), 0(0.000000e+00)
1246; EG-NEXT:     CNDE_INT T0.X, PV.W, T0.W, literal.x,
1247; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.y,
1248; EG-NEXT:    -1(nan), 2(2.802597e-45)
1249;
1250; GFX10-LABEL: v_cttz_i32_sel_eq_bitwidth:
1251; GFX10:       ; %bb.0:
1252; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
1253; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1254; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1255; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
1256; GFX10-NEXT:    global_load_dword v0, v0, s[2:3]
1257; GFX10-NEXT:    s_waitcnt vmcnt(0)
1258; GFX10-NEXT:    v_ffbl_b32_e32 v1, v0
1259; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
1260; GFX10-NEXT:    v_cndmask_b32_e32 v0, 32, v1, vcc_lo
1261; GFX10-NEXT:    v_mov_b32_e32 v1, 0
1262; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 32, v0
1263; GFX10-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc_lo
1264; GFX10-NEXT:    global_store_dword v1, v0, s[0:1]
1265; GFX10-NEXT:    s_endpgm
1266;
1267; GFX10-GISEL-LABEL: v_cttz_i32_sel_eq_bitwidth:
1268; GFX10-GISEL:       ; %bb.0:
1269; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
1270; GFX10-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1271; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1272; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
1273; GFX10-GISEL-NEXT:    global_load_dword v0, v0, s[2:3]
1274; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0)
1275; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v1, v0
1276; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
1277; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v0, v1, 32, vcc_lo
1278; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, 0
1279; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 32, v0
1280; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v0, v0, -1, vcc_lo
1281; GFX10-GISEL-NEXT:    global_store_dword v1, v0, s[0:1]
1282; GFX10-GISEL-NEXT:    s_endpgm
1283  %tid = call i32 @llvm.amdgcn.workitem.id.x()
1284  %in.gep = getelementptr i32, i32 addrspace(1)* %valptr, i32 %tid
1285  %val = load i32, i32 addrspace(1)* %in.gep
1286  %cttz = call i32 @llvm.cttz.i32(i32 %val, i1 false) nounwind readnone
1287  %cmp = icmp eq i32 %cttz, 32
1288  %sel = select i1 %cmp, i32 -1, i32 %cttz
1289  store i32 %sel, i32 addrspace(1)* %out
1290  ret void
1291}
1292
1293define amdgpu_kernel void @v_cttz_i32_sel_ne_bitwidth(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalias %valptr) nounwind {
1294; SI-LABEL: v_cttz_i32_sel_ne_bitwidth:
1295; SI:       ; %bb.0:
1296; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xb
1297; SI-NEXT:    s_mov_b32 s3, 0xf000
1298; SI-NEXT:    s_mov_b32 s6, 0
1299; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1300; SI-NEXT:    v_mov_b32_e32 v1, 0
1301; SI-NEXT:    s_mov_b32 s7, s3
1302; SI-NEXT:    s_waitcnt lgkmcnt(0)
1303; SI-NEXT:    buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
1304; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
1305; SI-NEXT:    s_mov_b32 s2, -1
1306; SI-NEXT:    s_waitcnt vmcnt(0)
1307; SI-NEXT:    v_ffbl_b32_e32 v1, v0
1308; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
1309; SI-NEXT:    v_cndmask_b32_e32 v0, 32, v1, vcc
1310; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 32, v0
1311; SI-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
1312; SI-NEXT:    s_waitcnt lgkmcnt(0)
1313; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1314; SI-NEXT:    s_endpgm
1315;
1316; VI-LABEL: v_cttz_i32_sel_ne_bitwidth:
1317; VI:       ; %bb.0:
1318; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
1319; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
1320; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1321; VI-NEXT:    s_mov_b32 s7, 0xf000
1322; VI-NEXT:    s_mov_b32 s6, -1
1323; VI-NEXT:    s_waitcnt lgkmcnt(0)
1324; VI-NEXT:    v_mov_b32_e32 v1, s1
1325; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
1326; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
1327; VI-NEXT:    flat_load_dword v0, v[0:1]
1328; VI-NEXT:    s_waitcnt vmcnt(0)
1329; VI-NEXT:    v_ffbl_b32_e32 v1, v0
1330; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
1331; VI-NEXT:    v_cndmask_b32_e32 v0, 32, v1, vcc
1332; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 32, v0
1333; VI-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
1334; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
1335; VI-NEXT:    s_endpgm
1336;
1337; EG-LABEL: v_cttz_i32_sel_ne_bitwidth:
1338; EG:       ; %bb.0:
1339; EG-NEXT:    ALU 2, @8, KC0[CB0:0-32], KC1[]
1340; EG-NEXT:    TEX 0 @6
1341; EG-NEXT:    ALU 7, @11, KC0[CB0:0-32], KC1[]
1342; EG-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
1343; EG-NEXT:    CF_END
1344; EG-NEXT:    PAD
1345; EG-NEXT:    Fetch clause starting at 6:
1346; EG-NEXT:     VTX_READ_32 T0.X, T0.X, 0, #1
1347; EG-NEXT:    ALU clause starting at 8:
1348; EG-NEXT:     LSHL * T0.W, T0.X, literal.x,
1349; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
1350; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, PV.W,
1351; EG-NEXT:    ALU clause starting at 11:
1352; EG-NEXT:     FFBL_INT * T0.W, T0.X,
1353; EG-NEXT:     CNDE_INT * T0.W, T0.X, literal.x, PV.W,
1354; EG-NEXT:    32(4.484155e-44), 0(0.000000e+00)
1355; EG-NEXT:     SETNE_INT * T1.W, PV.W, literal.x,
1356; EG-NEXT:    32(4.484155e-44), 0(0.000000e+00)
1357; EG-NEXT:     CNDE_INT T0.X, PV.W, literal.x, T0.W,
1358; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.y,
1359; EG-NEXT:    -1(nan), 2(2.802597e-45)
1360;
1361; GFX10-LABEL: v_cttz_i32_sel_ne_bitwidth:
1362; GFX10:       ; %bb.0:
1363; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
1364; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1365; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1366; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
1367; GFX10-NEXT:    global_load_dword v0, v0, s[2:3]
1368; GFX10-NEXT:    s_waitcnt vmcnt(0)
1369; GFX10-NEXT:    v_ffbl_b32_e32 v1, v0
1370; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
1371; GFX10-NEXT:    v_cndmask_b32_e32 v0, 32, v1, vcc_lo
1372; GFX10-NEXT:    v_mov_b32_e32 v1, 0
1373; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 32, v0
1374; GFX10-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc_lo
1375; GFX10-NEXT:    global_store_dword v1, v0, s[0:1]
1376; GFX10-NEXT:    s_endpgm
1377;
1378; GFX10-GISEL-LABEL: v_cttz_i32_sel_ne_bitwidth:
1379; GFX10-GISEL:       ; %bb.0:
1380; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
1381; GFX10-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
1382; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1383; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
1384; GFX10-GISEL-NEXT:    global_load_dword v0, v0, s[2:3]
1385; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0)
1386; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v1, v0
1387; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
1388; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v0, v1, 32, vcc_lo
1389; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, 0
1390; GFX10-GISEL-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 32, v0
1391; GFX10-GISEL-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc_lo
1392; GFX10-GISEL-NEXT:    global_store_dword v1, v0, s[0:1]
1393; GFX10-GISEL-NEXT:    s_endpgm
1394  %tid = call i32 @llvm.amdgcn.workitem.id.x()
1395  %in.gep = getelementptr i32, i32 addrspace(1)* %valptr, i32 %tid
1396  %val = load i32, i32 addrspace(1)* %in.gep
1397  %cttz = call i32 @llvm.cttz.i32(i32 %val, i1 false) nounwind readnone
1398  %cmp = icmp ne i32 %cttz, 32
1399  %sel = select i1 %cmp, i32 %cttz, i32 -1
1400  store i32 %sel, i32 addrspace(1)* %out
1401  ret void
1402}
1403
1404 define amdgpu_kernel void @v_cttz_i8_sel_eq_neg1(i8 addrspace(1)* noalias %out, i8 addrspace(1)* noalias %valptr) nounwind {
1405; SI-LABEL: v_cttz_i8_sel_eq_neg1:
1406; SI:       ; %bb.0:
1407; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xb
1408; SI-NEXT:    s_mov_b32 s3, 0xf000
1409; SI-NEXT:    v_mov_b32_e32 v1, 0
1410; SI-NEXT:    s_mov_b32 s6, 0
1411; SI-NEXT:    s_mov_b32 s7, s3
1412; SI-NEXT:    s_waitcnt lgkmcnt(0)
1413; SI-NEXT:    buffer_load_ubyte v0, v[0:1], s[4:7], 0 addr64
1414; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
1415; SI-NEXT:    s_mov_b32 s2, -1
1416; SI-NEXT:    s_waitcnt vmcnt(0)
1417; SI-NEXT:    v_ffbl_b32_e32 v0, v0
1418; SI-NEXT:    s_waitcnt lgkmcnt(0)
1419; SI-NEXT:    buffer_store_byte v0, off, s[0:3], 0
1420; SI-NEXT:    s_endpgm
1421;
1422; VI-LABEL: v_cttz_i8_sel_eq_neg1:
1423; VI:       ; %bb.0:
1424; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
1425; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
1426; VI-NEXT:    s_mov_b32 s7, 0xf000
1427; VI-NEXT:    s_mov_b32 s6, -1
1428; VI-NEXT:    s_waitcnt lgkmcnt(0)
1429; VI-NEXT:    v_mov_b32_e32 v1, s1
1430; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
1431; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
1432; VI-NEXT:    flat_load_ubyte v0, v[0:1]
1433; VI-NEXT:    s_waitcnt vmcnt(0)
1434; VI-NEXT:    v_ffbl_b32_e32 v0, v0
1435; VI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
1436; VI-NEXT:    s_endpgm
1437;
1438; EG-LABEL: v_cttz_i8_sel_eq_neg1:
1439; EG:       ; %bb.0:
1440; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
1441; EG-NEXT:    TEX 0 @6
1442; EG-NEXT:    ALU 12, @9, KC0[CB0:0-32], KC1[]
1443; EG-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
1444; EG-NEXT:    CF_END
1445; EG-NEXT:    PAD
1446; EG-NEXT:    Fetch clause starting at 6:
1447; EG-NEXT:     VTX_READ_8 T0.X, T0.X, 0, #1
1448; EG-NEXT:    ALU clause starting at 8:
1449; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, T0.X,
1450; EG-NEXT:    ALU clause starting at 9:
1451; EG-NEXT:     FFBL_INT T0.W, T0.X,
1452; EG-NEXT:     AND_INT * T1.W, KC0[2].Y, literal.x,
1453; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
1454; EG-NEXT:     AND_INT T0.W, PV.W, literal.x,
1455; EG-NEXT:     LSHL * T1.W, PS, literal.y,
1456; EG-NEXT:    255(3.573311e-43), 3(4.203895e-45)
1457; EG-NEXT:     LSHL T0.X, PV.W, PS,
1458; EG-NEXT:     LSHL * T0.W, literal.x, PS,
1459; EG-NEXT:    255(3.573311e-43), 0(0.000000e+00)
1460; EG-NEXT:     MOV T0.Y, 0.0,
1461; EG-NEXT:     MOV * T0.Z, 0.0,
1462; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
1463; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
1464;
1465; GFX10-LABEL: v_cttz_i8_sel_eq_neg1:
1466; GFX10:       ; %bb.0:
1467; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
1468; GFX10-NEXT:    v_mov_b32_e32 v1, 0
1469; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1470; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
1471; GFX10-NEXT:    global_load_ubyte v0, v0, s[2:3]
1472; GFX10-NEXT:    s_waitcnt vmcnt(0)
1473; GFX10-NEXT:    v_ffbl_b32_e32 v0, v0
1474; GFX10-NEXT:    global_store_byte v1, v0, s[0:1]
1475; GFX10-NEXT:    s_endpgm
1476;
1477; GFX10-GISEL-LABEL: v_cttz_i8_sel_eq_neg1:
1478; GFX10-GISEL:       ; %bb.0:
1479; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
1480; GFX10-GISEL-NEXT:    v_ashrrev_i32_e32 v3, 31, v0
1481; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1482; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
1483; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, s2
1484; GFX10-GISEL-NEXT:    v_mov_b32_e32 v2, s3
1485; GFX10-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, v1, v0
1486; GFX10-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v2, v3, vcc_lo
1487; GFX10-GISEL-NEXT:    global_load_ubyte v0, v[0:1], off
1488; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0)
1489; GFX10-GISEL-NEXT:    v_and_b32_e32 v0, 0xff, v0
1490; GFX10-GISEL-NEXT:    v_or_b32_e32 v1, 0x100, v0
1491; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
1492; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v1, v1
1493; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v0, v1, -1, vcc_lo
1494; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, 0
1495; GFX10-GISEL-NEXT:    global_store_byte v1, v0, s[0:1]
1496; GFX10-GISEL-NEXT:    s_endpgm
1497  %tid = call i32 @llvm.amdgcn.workitem.id.x()
1498  %valptr.gep = getelementptr i8, i8 addrspace(1)* %valptr, i32 %tid
1499  %val = load i8, i8 addrspace(1)* %valptr.gep
1500  %cttz = call i8 @llvm.cttz.i8(i8 %val, i1 false) nounwind readnone
1501  %cmp = icmp eq i8 %val, 0
1502  %sel = select i1 %cmp, i8 -1, i8 %cttz
1503  store i8 %sel, i8 addrspace(1)* %out
1504  ret void
1505}
1506
1507 define amdgpu_kernel void @v_cttz_i16_sel_eq_neg1(i16 addrspace(1)* noalias %out, i16 addrspace(1)* noalias %valptr) nounwind {
1508; SI-LABEL: v_cttz_i16_sel_eq_neg1:
1509; SI:       ; %bb.0:
1510; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xb
1511; SI-NEXT:    s_mov_b32 s3, 0xf000
1512; SI-NEXT:    s_mov_b32 s2, -1
1513; SI-NEXT:    s_mov_b32 s6, s2
1514; SI-NEXT:    s_mov_b32 s7, s3
1515; SI-NEXT:    s_waitcnt lgkmcnt(0)
1516; SI-NEXT:    buffer_load_ushort v0, off, s[4:7], 0
1517; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
1518; SI-NEXT:    s_waitcnt vmcnt(0)
1519; SI-NEXT:    v_ffbl_b32_e32 v0, v0
1520; SI-NEXT:    s_waitcnt lgkmcnt(0)
1521; SI-NEXT:    buffer_store_short v0, off, s[0:3], 0
1522; SI-NEXT:    s_endpgm
1523;
1524; VI-LABEL: v_cttz_i16_sel_eq_neg1:
1525; VI:       ; %bb.0:
1526; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
1527; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
1528; VI-NEXT:    s_mov_b32 s7, 0xf000
1529; VI-NEXT:    s_mov_b32 s6, -1
1530; VI-NEXT:    s_mov_b32 s2, s6
1531; VI-NEXT:    s_mov_b32 s3, s7
1532; VI-NEXT:    s_waitcnt lgkmcnt(0)
1533; VI-NEXT:    buffer_load_ushort v0, off, s[0:3], 0
1534; VI-NEXT:    v_mov_b32_e32 v1, 0xffff
1535; VI-NEXT:    s_waitcnt vmcnt(0)
1536; VI-NEXT:    v_or_b32_e32 v2, 0x10000, v0
1537; VI-NEXT:    v_ffbl_b32_e32 v3, v2
1538; VI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
1539; VI-NEXT:    v_cndmask_b32_e32 v2, 32, v3, vcc
1540; VI-NEXT:    v_cmp_ne_u16_e32 vcc, 0, v0
1541; VI-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
1542; VI-NEXT:    buffer_store_short v0, off, s[4:7], 0
1543; VI-NEXT:    s_endpgm
1544;
1545; EG-LABEL: v_cttz_i16_sel_eq_neg1:
1546; EG:       ; %bb.0:
1547; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
1548; EG-NEXT:    TEX 0 @6
1549; EG-NEXT:    ALU 12, @9, KC0[CB0:0-32], KC1[]
1550; EG-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
1551; EG-NEXT:    CF_END
1552; EG-NEXT:    PAD
1553; EG-NEXT:    Fetch clause starting at 6:
1554; EG-NEXT:     VTX_READ_16 T0.X, T0.X, 0, #1
1555; EG-NEXT:    ALU clause starting at 8:
1556; EG-NEXT:     MOV * T0.X, KC0[2].Z,
1557; EG-NEXT:    ALU clause starting at 9:
1558; EG-NEXT:     FFBL_INT T0.W, T0.X,
1559; EG-NEXT:     AND_INT * T1.W, KC0[2].Y, literal.x,
1560; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
1561; EG-NEXT:     AND_INT T0.W, PV.W, literal.x,
1562; EG-NEXT:     LSHL * T1.W, PS, literal.y,
1563; EG-NEXT:    65535(9.183409e-41), 3(4.203895e-45)
1564; EG-NEXT:     LSHL T0.X, PV.W, PS,
1565; EG-NEXT:     LSHL * T0.W, literal.x, PS,
1566; EG-NEXT:    65535(9.183409e-41), 0(0.000000e+00)
1567; EG-NEXT:     MOV T0.Y, 0.0,
1568; EG-NEXT:     MOV * T0.Z, 0.0,
1569; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
1570; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
1571;
1572; GFX10-LABEL: v_cttz_i16_sel_eq_neg1:
1573; GFX10:       ; %bb.0:
1574; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
1575; GFX10-NEXT:    v_mov_b32_e32 v0, 0
1576; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1577; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
1578; GFX10-NEXT:    global_load_ushort v1, v0, s[2:3]
1579; GFX10-NEXT:    s_waitcnt vmcnt(0)
1580; GFX10-NEXT:    v_or_b32_e32 v2, 0x10000, v1
1581; GFX10-NEXT:    v_ffbl_b32_e32 v3, v2
1582; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v2
1583; GFX10-NEXT:    v_cndmask_b32_e32 v2, 32, v3, vcc_lo
1584; GFX10-NEXT:    v_cmp_ne_u16_e32 vcc_lo, 0, v1
1585; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0xffff, v2, vcc_lo
1586; GFX10-NEXT:    global_store_short v0, v1, s[0:1]
1587; GFX10-NEXT:    s_endpgm
1588;
1589; GFX10-GISEL-LABEL: v_cttz_i16_sel_eq_neg1:
1590; GFX10-GISEL:       ; %bb.0:
1591; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
1592; GFX10-GISEL-NEXT:    v_mov_b32_e32 v0, 0
1593; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1594; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
1595; GFX10-GISEL-NEXT:    global_load_ushort v1, v0, s[2:3]
1596; GFX10-GISEL-NEXT:    s_waitcnt_depctr 0xffe3
1597; GFX10-GISEL-NEXT:    s_mov_b32 s2, 0xffff
1598; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0)
1599; GFX10-GISEL-NEXT:    v_or_b32_e32 v2, 0x10000, v1
1600; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
1601; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v2, v2
1602; GFX10-GISEL-NEXT:    v_and_b32_e32 v2, s2, v2
1603; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v1, v2, s2, vcc_lo
1604; GFX10-GISEL-NEXT:    global_store_short v0, v1, s[0:1]
1605; GFX10-GISEL-NEXT:    s_endpgm
1606  %val = load i16, i16 addrspace(1)* %valptr
1607  %cttz = call i16 @llvm.cttz.i16(i16 %val, i1 false) nounwind readnone
1608  %cmp = icmp eq i16 %val, 0
1609  %sel = select i1 %cmp, i16 -1, i16 %cttz
1610  store i16 %sel, i16 addrspace(1)* %out
1611  ret void
1612}
1613
1614; FIXME: Need to handle non-uniform case for function below (load without gep).
1615define amdgpu_kernel void @v_cttz_i7_sel_eq_neg1(i7 addrspace(1)* noalias %out, i7 addrspace(1)* noalias %valptr) nounwind {
1616; SI-LABEL: v_cttz_i7_sel_eq_neg1:
1617; SI:       ; %bb.0:
1618; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xb
1619; SI-NEXT:    s_mov_b32 s3, 0xf000
1620; SI-NEXT:    v_mov_b32_e32 v1, 0
1621; SI-NEXT:    s_mov_b32 s6, 0
1622; SI-NEXT:    s_mov_b32 s7, s3
1623; SI-NEXT:    s_waitcnt lgkmcnt(0)
1624; SI-NEXT:    buffer_load_ubyte v0, v[0:1], s[4:7], 0 addr64
1625; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
1626; SI-NEXT:    s_mov_b32 s2, -1
1627; SI-NEXT:    s_waitcnt vmcnt(0)
1628; SI-NEXT:    v_ffbl_b32_e32 v0, v0
1629; SI-NEXT:    v_and_b32_e32 v0, 0x7f, v0
1630; SI-NEXT:    s_waitcnt lgkmcnt(0)
1631; SI-NEXT:    buffer_store_byte v0, off, s[0:3], 0
1632; SI-NEXT:    s_endpgm
1633;
1634; VI-LABEL: v_cttz_i7_sel_eq_neg1:
1635; VI:       ; %bb.0:
1636; VI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
1637; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
1638; VI-NEXT:    s_mov_b32 s7, 0xf000
1639; VI-NEXT:    s_mov_b32 s6, -1
1640; VI-NEXT:    s_waitcnt lgkmcnt(0)
1641; VI-NEXT:    v_mov_b32_e32 v1, s1
1642; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
1643; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
1644; VI-NEXT:    flat_load_ubyte v0, v[0:1]
1645; VI-NEXT:    s_waitcnt vmcnt(0)
1646; VI-NEXT:    v_ffbl_b32_e32 v0, v0
1647; VI-NEXT:    v_and_b32_e32 v0, 0x7f, v0
1648; VI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
1649; VI-NEXT:    s_endpgm
1650;
1651; EG-LABEL: v_cttz_i7_sel_eq_neg1:
1652; EG:       ; %bb.0:
1653; EG-NEXT:    ALU 0, @8, KC0[CB0:0-32], KC1[]
1654; EG-NEXT:    TEX 0 @6
1655; EG-NEXT:    ALU 12, @9, KC0[CB0:0-32], KC1[]
1656; EG-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
1657; EG-NEXT:    CF_END
1658; EG-NEXT:    PAD
1659; EG-NEXT:    Fetch clause starting at 6:
1660; EG-NEXT:     VTX_READ_8 T0.X, T0.X, 0, #1
1661; EG-NEXT:    ALU clause starting at 8:
1662; EG-NEXT:     ADD_INT * T0.X, KC0[2].Z, T0.X,
1663; EG-NEXT:    ALU clause starting at 9:
1664; EG-NEXT:     FFBL_INT T0.W, T0.X,
1665; EG-NEXT:     AND_INT * T1.W, KC0[2].Y, literal.x,
1666; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
1667; EG-NEXT:     AND_INT T0.W, PV.W, literal.x,
1668; EG-NEXT:     LSHL * T1.W, PS, literal.y,
1669; EG-NEXT:    127(1.779649e-43), 3(4.203895e-45)
1670; EG-NEXT:     LSHL T0.X, PV.W, PS,
1671; EG-NEXT:     LSHL * T0.W, literal.x, PS,
1672; EG-NEXT:    255(3.573311e-43), 0(0.000000e+00)
1673; EG-NEXT:     MOV T0.Y, 0.0,
1674; EG-NEXT:     MOV * T0.Z, 0.0,
1675; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
1676; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
1677;
1678; GFX10-LABEL: v_cttz_i7_sel_eq_neg1:
1679; GFX10:       ; %bb.0:
1680; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
1681; GFX10-NEXT:    v_mov_b32_e32 v1, 0
1682; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1683; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
1684; GFX10-NEXT:    global_load_ubyte v0, v0, s[2:3]
1685; GFX10-NEXT:    s_waitcnt vmcnt(0)
1686; GFX10-NEXT:    v_ffbl_b32_e32 v0, v0
1687; GFX10-NEXT:    v_and_b32_e32 v0, 0x7f, v0
1688; GFX10-NEXT:    global_store_byte v1, v0, s[0:1]
1689; GFX10-NEXT:    s_endpgm
1690;
1691; GFX10-GISEL-LABEL: v_cttz_i7_sel_eq_neg1:
1692; GFX10-GISEL:       ; %bb.0:
1693; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
1694; GFX10-GISEL-NEXT:    v_ashrrev_i32_e32 v3, 31, v0
1695; GFX10-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1696; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
1697; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, s2
1698; GFX10-GISEL-NEXT:    v_mov_b32_e32 v2, s3
1699; GFX10-GISEL-NEXT:    s_movk_i32 s2, 0x7f
1700; GFX10-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, v1, v0
1701; GFX10-GISEL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v2, v3, vcc_lo
1702; GFX10-GISEL-NEXT:    global_load_ubyte v0, v[0:1], off
1703; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0)
1704; GFX10-GISEL-NEXT:    v_and_or_b32 v1, v0, s2, 0x80
1705; GFX10-GISEL-NEXT:    v_and_b32_e32 v0, s2, v0
1706; GFX10-GISEL-NEXT:    v_ffbl_b32_e32 v1, v1
1707; GFX10-GISEL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
1708; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v0, v1, 0x7f, vcc_lo
1709; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, 0
1710; GFX10-GISEL-NEXT:    v_and_b32_e32 v0, s2, v0
1711; GFX10-GISEL-NEXT:    global_store_byte v1, v0, s[0:1]
1712; GFX10-GISEL-NEXT:    s_endpgm
1713  %tid = call i32 @llvm.amdgcn.workitem.id.x()
1714  %valptr.gep = getelementptr i7, i7 addrspace(1)* %valptr, i32 %tid
1715  %val = load i7, i7 addrspace(1)* %valptr.gep
1716  %cttz = call i7 @llvm.cttz.i7(i7 %val, i1 false) nounwind readnone
1717  %cmp = icmp eq i7 %val, 0
1718  %sel = select i1 %cmp, i7 -1, i7 %cttz
1719  store i7 %sel, i7 addrspace(1)* %out
1720  ret void
1721}
1722