1; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-NOSDWA -check-prefix=FUNC %s 2; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-SDWA -check-prefix=FUNC %s 3; RUN: llc -march=r600 -mcpu=cypress -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=EG -check-prefix=FUNC %s 4 5declare i7 @llvm.cttz.i7(i7, i1) nounwind readnone 6declare i8 @llvm.cttz.i8(i8, i1) nounwind readnone 7declare i16 @llvm.cttz.i16(i16, i1) nounwind readnone 8declare i32 @llvm.cttz.i32(i32, i1) nounwind readnone 9declare i64 @llvm.cttz.i64(i64, i1) nounwind readnone 10declare <2 x i32> @llvm.cttz.v2i32(<2 x i32>, i1) nounwind readnone 11declare <4 x i32> @llvm.cttz.v4i32(<4 x i32>, i1) nounwind readnone 12declare i32 @llvm.r600.read.tidig.x() nounwind readnone 13 14; FUNC-LABEL: {{^}}s_cttz_zero_undef_i32: 15; SI: s_load_dword [[VAL:s[0-9]+]], 16; SI: s_ff1_i32_b32 [[SRESULT:s[0-9]+]], [[VAL]] 17; SI: v_mov_b32_e32 [[VRESULT:v[0-9]+]], [[SRESULT]] 18; SI: buffer_store_dword [[VRESULT]], 19; SI: s_endpgm 20; EG: MEM_RAT_CACHELESS STORE_RAW [[RESULT:T[0-9]+\.[XYZW]]] 21; EG: FFBL_INT {{\*? *}}[[RESULT]] 22define amdgpu_kernel void @s_cttz_zero_undef_i32(i32 addrspace(1)* noalias %out, i32 %val) nounwind { 23 %cttz = call i32 @llvm.cttz.i32(i32 %val, i1 true) nounwind readnone 24 store i32 %cttz, i32 addrspace(1)* %out, align 4 25 ret void 26} 27 28; FUNC-LABEL: {{^}}v_cttz_zero_undef_i32: 29; SI: {{buffer|flat}}_load_dword [[VAL:v[0-9]+]], 30; SI: v_ffbl_b32_e32 [[RESULT:v[0-9]+]], [[VAL]] 31; SI: buffer_store_dword [[RESULT]], 32; SI: s_endpgm 33; EG: MEM_RAT_CACHELESS STORE_RAW [[RESULT:T[0-9]+\.[XYZW]]] 34; EG: FFBL_INT {{\*? *}}[[RESULT]] 35define amdgpu_kernel void @v_cttz_zero_undef_i32(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalias %valptr) nounwind { 36 %tid = call i32 @llvm.r600.read.tidig.x() 37 %in.gep = getelementptr i32, i32 addrspace(1)* %valptr, i32 %tid 38 %val = load i32, i32 addrspace(1)* %in.gep, align 4 39 %cttz = call i32 @llvm.cttz.i32(i32 %val, i1 true) nounwind readnone 40 store i32 %cttz, i32 addrspace(1)* %out, align 4 41 ret void 42} 43 44; FUNC-LABEL: {{^}}v_cttz_zero_undef_v2i32: 45; SI: {{buffer|flat}}_load_dwordx2 46; SI: v_ffbl_b32_e32 47; SI: v_ffbl_b32_e32 48; SI: buffer_store_dwordx2 49; SI: s_endpgm 50; EG: MEM_RAT_CACHELESS STORE_RAW [[RESULT:T[0-9]+]]{{\.[XYZW]}} 51; EG: FFBL_INT {{\*? *}}[[RESULT]] 52; EG: FFBL_INT {{\*? *}}[[RESULT]] 53define amdgpu_kernel void @v_cttz_zero_undef_v2i32(<2 x i32> addrspace(1)* noalias %out, <2 x i32> addrspace(1)* noalias %valptr) nounwind { 54 %tid = call i32 @llvm.r600.read.tidig.x() 55 %in.gep = getelementptr <2 x i32>, <2 x i32> addrspace(1)* %valptr, i32 %tid 56 %val = load <2 x i32>, <2 x i32> addrspace(1)* %in.gep, align 8 57 %cttz = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %val, i1 true) nounwind readnone 58 store <2 x i32> %cttz, <2 x i32> addrspace(1)* %out, align 8 59 ret void 60} 61 62; FUNC-LABEL: {{^}}v_cttz_zero_undef_v4i32: 63; SI: {{buffer|flat}}_load_dwordx4 64; SI: v_ffbl_b32_e32 65; SI: v_ffbl_b32_e32 66; SI: v_ffbl_b32_e32 67; SI: v_ffbl_b32_e32 68; SI: buffer_store_dwordx4 69; SI: s_endpgm 70; EG: MEM_RAT_CACHELESS STORE_RAW [[RESULT:T[0-9]+]]{{\.[XYZW]}} 71; EG: FFBL_INT {{\*? *}}[[RESULT]] 72; EG: FFBL_INT {{\*? *}}[[RESULT]] 73; EG: FFBL_INT {{\*? *}}[[RESULT]] 74; EG: FFBL_INT {{\*? *}}[[RESULT]] 75define amdgpu_kernel void @v_cttz_zero_undef_v4i32(<4 x i32> addrspace(1)* noalias %out, <4 x i32> addrspace(1)* noalias %valptr) nounwind { 76 %tid = call i32 @llvm.r600.read.tidig.x() 77 %in.gep = getelementptr <4 x i32>, <4 x i32> addrspace(1)* %valptr, i32 %tid 78 %val = load <4 x i32>, <4 x i32> addrspace(1)* %in.gep, align 16 79 %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %val, i1 true) nounwind readnone 80 store <4 x i32> %cttz, <4 x i32> addrspace(1)* %out, align 16 81 ret void 82} 83 84; FUNC-LABEL: {{^}}s_cttz_zero_undef_i8_with_select: 85; SI: s_ff1_i32_b32 s{{[0-9]+}}, s{{[0-9]+}} 86; EG: MEM_RAT MSKOR 87; EG: FFBL_INT 88define amdgpu_kernel void @s_cttz_zero_undef_i8_with_select(i8 addrspace(1)* noalias %out, i8 %val) nounwind { 89 %cttz = tail call i8 @llvm.cttz.i8(i8 %val, i1 true) nounwind readnone 90 %cttz_ret = icmp ne i8 %val, 0 91 %ret = select i1 %cttz_ret, i8 %cttz, i8 32 92 store i8 %cttz, i8 addrspace(1)* %out, align 4 93 ret void 94} 95 96; FUNC-LABEL: {{^}}s_cttz_zero_undef_i16_with_select: 97; SI: s_ff1_i32_b32 s{{[0-9]+}}, s{{[0-9]+}} 98; EG: MEM_RAT MSKOR 99; EG: FFBL_INT 100define amdgpu_kernel void @s_cttz_zero_undef_i16_with_select(i16 addrspace(1)* noalias %out, i16 %val) nounwind { 101 %cttz = tail call i16 @llvm.cttz.i16(i16 %val, i1 true) nounwind readnone 102 %cttz_ret = icmp ne i16 %val, 0 103 %ret = select i1 %cttz_ret, i16 %cttz, i16 32 104 store i16 %cttz, i16 addrspace(1)* %out, align 4 105 ret void 106} 107 108; FUNC-LABEL: {{^}}s_cttz_zero_undef_i32_with_select: 109; SI: s_ff1_i32_b32 110; EG: MEM_RAT_CACHELESS STORE_RAW [[RESULT:T[0-9]+\.[XYZW]]] 111; EG: FFBL_INT {{\*? *}}[[RESULT]] 112define amdgpu_kernel void @s_cttz_zero_undef_i32_with_select(i32 addrspace(1)* noalias %out, i32 %val) nounwind { 113 %cttz = tail call i32 @llvm.cttz.i32(i32 %val, i1 true) nounwind readnone 114 %cttz_ret = icmp ne i32 %val, 0 115 %ret = select i1 %cttz_ret, i32 %cttz, i32 32 116 store i32 %cttz, i32 addrspace(1)* %out, align 4 117 ret void 118} 119 120; FUNC-LABEL: {{^}}s_cttz_zero_undef_i64_with_select: 121; SI: s_ff1_i32_b32 s{{[0-9]+}}, s{{[0-9]+}} 122; SI: s_ff1_i32_b32 s{{[0-9]+}}, s{{[0-9]+}} 123; EG: MEM_RAT_CACHELESS STORE_RAW [[RESULT:T[0-9]+\.[XYZW]]] 124define amdgpu_kernel void @s_cttz_zero_undef_i64_with_select(i64 addrspace(1)* noalias %out, i64 %val) nounwind { 125 %cttz = tail call i64 @llvm.cttz.i64(i64 %val, i1 true) nounwind readnone 126 %cttz_ret = icmp ne i64 %val, 0 127 %ret = select i1 %cttz_ret, i64 %cttz, i64 32 128 store i64 %cttz, i64 addrspace(1)* %out, align 4 129 ret void 130} 131 132; FUNC-LABEL: {{^}}v_cttz_zero_undef_i8_with_select: 133; SI-NOSDWA: v_ffbl_b32_e32 v{{[0-9]+}}, v{{[0-9]+}} 134; SI-SDWA: v_ffbl_b32_sdwa 135; EG: MEM_RAT MSKOR 136define amdgpu_kernel void @v_cttz_zero_undef_i8_with_select(i8 addrspace(1)* noalias %out, i8 addrspace(1)* nocapture readonly %arrayidx) nounwind { 137 %val = load i8, i8 addrspace(1)* %arrayidx, align 1 138 %cttz = tail call i8 @llvm.cttz.i8(i8 %val, i1 true) nounwind readnone 139 %cttz_ret = icmp ne i8 %val, 0 140 %ret = select i1 %cttz_ret, i8 %cttz, i8 32 141 store i8 %ret, i8 addrspace(1)* %out, align 4 142 ret void 143} 144 145; FUNC-LABEL: {{^}}v_cttz_zero_undef_i16_with_select: 146; SI-NOSDWA: v_ffbl_b32_e32 v{{[0-9]+}}, v{{[0-9]+}} 147; SI-SDWA: v_ffbl_b32_sdwa 148; EG: MEM_RAT MSKOR 149define amdgpu_kernel void @v_cttz_zero_undef_i16_with_select(i16 addrspace(1)* noalias %out, i16 addrspace(1)* nocapture readonly %arrayidx) nounwind { 150 %val = load i16, i16 addrspace(1)* %arrayidx, align 1 151 %cttz = tail call i16 @llvm.cttz.i16(i16 %val, i1 true) nounwind readnone 152 %cttz_ret = icmp ne i16 %val, 0 153 %ret = select i1 %cttz_ret, i16 %cttz, i16 32 154 store i16 %ret, i16 addrspace(1)* %out, align 4 155 ret void 156} 157 158; FUNC-LABEL: {{^}}v_cttz_zero_undef_i32_with_select: 159; SI: v_ffbl_b32_e32 v{{[0-9]+}}, v{{[0-9]+}} 160; SI: v_cmp_ne_u32_e32 vcc, 0 161; EG: MEM_RAT_CACHELESS STORE_RAW [[RESULT:T[0-9]+\.[XYZW]]] 162define amdgpu_kernel void @v_cttz_zero_undef_i32_with_select(i32 addrspace(1)* noalias %out, i32 addrspace(1)* nocapture readonly %arrayidx) nounwind { 163 %val = load i32, i32 addrspace(1)* %arrayidx, align 1 164 %cttz = tail call i32 @llvm.cttz.i32(i32 %val, i1 true) nounwind readnone 165 %cttz_ret = icmp ne i32 %val, 0 166 %ret = select i1 %cttz_ret, i32 %cttz, i32 32 167 store i32 %ret, i32 addrspace(1)* %out, align 4 168 ret void 169} 170 171; FUNC-LABEL: {{^}}v_cttz_zero_undef_i64_with_select: 172; SI-NOSDWA: v_or_b32_e32 173; SI-NOSDWA: v_or_b32_e32 174; SI-NOSDWA: v_or_b32_e32 175; SI-SDWA: v_or_b32_sdwa 176; SI-NOSDWA: v_or_b32_e32 177; SI-SDWA: v_or_b32_sdwa 178; SI: v_or_b32_e32 [[VAL1:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} 179; SI: v_or_b32_e32 [[VAL2:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} 180; SI-DAG: v_ffbl_b32_e32 v{{[0-9]+}}, [[VAL1]] 181; SI-DAG: v_ffbl_b32_e32 v{{[0-9]+}}, [[VAL2]] 182; SI: v_cmp_eq_u32_e32 vcc, 0 183; SI: v_cmp_ne_u64_e32 vcc, 0 184; EG: MEM_RAT_CACHELESS STORE_RAW [[RESULT:T[0-9]+\.[XYZW]]] 185define amdgpu_kernel void @v_cttz_zero_undef_i64_with_select(i64 addrspace(1)* noalias %out, i64 addrspace(1)* nocapture readonly %arrayidx) nounwind { 186 %val = load i64, i64 addrspace(1)* %arrayidx, align 1 187 %cttz = tail call i64 @llvm.cttz.i64(i64 %val, i1 true) nounwind readnone 188 %cttz_ret = icmp ne i64 %val, 0 189 %ret = select i1 %cttz_ret, i64 %cttz, i64 32 190 store i64 %ret, i64 addrspace(1)* %out, align 4 191 ret void 192} 193 194; FUNC-LABEL: {{^}}v_cttz_i32_sel_eq_neg1: 195; SI: v_ffbl_b32_e32 v{{[0-9]+}}, [[VAL:v[0-9]+]] 196; SI: v_cmp_ne_u32_e32 vcc, 0, [[VAL]] 197; SI: s_endpgm 198; EG: MEM_RAT_CACHELESS STORE_RAW 199; EG: FFBL_INT 200define amdgpu_kernel void @v_cttz_i32_sel_eq_neg1(i32 addrspace(1)* noalias %out, i32 addrspace(1)* nocapture readonly %arrayidx) nounwind { 201 %val = load i32, i32 addrspace(1)* %arrayidx, align 1 202 %ctlz = call i32 @llvm.cttz.i32(i32 %val, i1 false) nounwind readnone 203 %cmp = icmp eq i32 %val, 0 204 %sel = select i1 %cmp, i32 -1, i32 %ctlz 205 store i32 %sel, i32 addrspace(1)* %out 206 ret void 207} 208 209; FUNC-LABEL: {{^}}v_cttz_i32_sel_ne_neg1: 210; SI: v_ffbl_b32_e32 v{{[0-9]+}}, [[VAL:v[0-9]+]] 211; SI: v_cmp_ne_u32_e32 vcc, 0, [[VAL]] 212; SI: s_endpgm 213; EG: MEM_RAT_CACHELESS STORE_RAW 214; EG: FFBL_INT 215define amdgpu_kernel void @v_cttz_i32_sel_ne_neg1(i32 addrspace(1)* noalias %out, i32 addrspace(1)* nocapture readonly %arrayidx) nounwind { 216 %val = load i32, i32 addrspace(1)* %arrayidx, align 1 217 %ctlz = call i32 @llvm.cttz.i32(i32 %val, i1 false) nounwind readnone 218 %cmp = icmp ne i32 %val, 0 219 %sel = select i1 %cmp, i32 %ctlz, i32 -1 220 store i32 %sel, i32 addrspace(1)* %out 221 ret void 222} 223 224; FUNC-LABEL: {{^}}v_cttz_i32_sel_ne_bitwidth: 225; SI: v_ffbl_b32_e32 v{{[0-9]+}}, v{{[0-9]+}} 226; SI: v_cmp 227; SI: v_cndmask 228; SI: s_endpgm 229; EG: MEM_RAT_CACHELESS STORE_RAW 230; EG: FFBL_INT 231define amdgpu_kernel void @v_cttz_i32_sel_ne_bitwidth(i32 addrspace(1)* noalias %out, i32 addrspace(1)* nocapture readonly %arrayidx) nounwind { 232 %val = load i32, i32 addrspace(1)* %arrayidx, align 1 233 %ctlz = call i32 @llvm.cttz.i32(i32 %val, i1 false) nounwind readnone 234 %cmp = icmp ne i32 %ctlz, 32 235 %sel = select i1 %cmp, i32 %ctlz, i32 -1 236 store i32 %sel, i32 addrspace(1)* %out 237 ret void 238} 239 240; FUNC-LABEL: {{^}}v_cttz_i8_sel_eq_neg1: 241; SI: {{buffer|flat}}_load_ubyte 242; SI: v_ffbl_b32_e32 v{{[0-9]+}}, v{{[0-9]+}} 243; EG: MEM_RAT MSKOR 244; EG: FFBL_INT 245 define amdgpu_kernel void @v_cttz_i8_sel_eq_neg1(i8 addrspace(1)* noalias %out, i8 addrspace(1)* nocapture readonly %arrayidx) nounwind { 246 %val = load i8, i8 addrspace(1)* %arrayidx, align 1 247 %ctlz = call i8 @llvm.cttz.i8(i8 %val, i1 false) nounwind readnone 248 %cmp = icmp eq i8 %val, 0 249 %sel = select i1 %cmp, i8 -1, i8 %ctlz 250 store i8 %sel, i8 addrspace(1)* %out 251 ret void 252} 253 254; FUNC-LABEL: {{^}}v_cttz_i16_sel_eq_neg1: 255; SI: {{buffer|flat}}_load_ubyte 256; SI: v_ffbl_b32_e32 v{{[0-9]+}}, v{{[0-9]+}} 257; SI: buffer_store_short 258; EG: MEM_RAT MSKOR 259; EG: FFBL_INT 260 define amdgpu_kernel void @v_cttz_i16_sel_eq_neg1(i16 addrspace(1)* noalias %out, i16 addrspace(1)* nocapture readonly %arrayidx) nounwind { 261 %val = load i16, i16 addrspace(1)* %arrayidx, align 1 262 %ctlz = call i16 @llvm.cttz.i16(i16 %val, i1 false) nounwind readnone 263 %cmp = icmp eq i16 %val, 0 264 %sel = select i1 %cmp, i16 -1, i16 %ctlz 265 store i16 %sel, i16 addrspace(1)* %out 266 ret void 267} 268 269 270