1; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC -check-prefix=SI %s 2; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC -check-prefix=VI %s 3; RUN: llc -march=r600 -mcpu=cypress -verify-machineinstrs < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s 4 5declare i32 @llvm.ctpop.i32(i32) nounwind readnone 6declare <2 x i32> @llvm.ctpop.v2i32(<2 x i32>) nounwind readnone 7declare <4 x i32> @llvm.ctpop.v4i32(<4 x i32>) nounwind readnone 8declare <8 x i32> @llvm.ctpop.v8i32(<8 x i32>) nounwind readnone 9declare <16 x i32> @llvm.ctpop.v16i32(<16 x i32>) nounwind readnone 10 11declare i32 @llvm.r600.read.tidig.x() nounwind readnone 12 13; FUNC-LABEL: {{^}}s_ctpop_i32: 14; GCN: s_load_dword [[SVAL:s[0-9]+]], 15; GCN: s_bcnt1_i32_b32 [[SRESULT:s[0-9]+]], [[SVAL]] 16; GCN: v_mov_b32_e32 [[VRESULT:v[0-9]+]], [[SRESULT]] 17; GCN: buffer_store_dword [[VRESULT]], 18; GCN: s_endpgm 19 20; EG: BCNT_INT 21define amdgpu_kernel void @s_ctpop_i32(i32 addrspace(1)* noalias %out, i32 %val) nounwind { 22 %ctpop = call i32 @llvm.ctpop.i32(i32 %val) nounwind readnone 23 store i32 %ctpop, i32 addrspace(1)* %out, align 4 24 ret void 25} 26 27; XXX - Why 0 in register? 28; FUNC-LABEL: {{^}}v_ctpop_i32: 29; GCN: {{buffer|flat}}_load_dword [[VAL:v[0-9]+]], 30; GCN: v_bcnt_u32_b32{{(_e64)*}} [[RESULT:v[0-9]+]], [[VAL]], 0 31; GCN: buffer_store_dword [[RESULT]], 32; GCN: s_endpgm 33 34; EG: BCNT_INT 35define amdgpu_kernel void @v_ctpop_i32(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalias %in) nounwind { 36 %tid = call i32 @llvm.r600.read.tidig.x() 37 %in.gep = getelementptr i32, i32 addrspace(1)* %in, i32 %tid 38 %val = load i32, i32 addrspace(1)* %in.gep, align 4 39 %ctpop = call i32 @llvm.ctpop.i32(i32 %val) nounwind readnone 40 store i32 %ctpop, i32 addrspace(1)* %out, align 4 41 ret void 42} 43 44; FUNC-LABEL: {{^}}v_ctpop_add_chain_i32: 45; GCN: {{buffer|flat}}_load_dword [[VAL0:v[0-9]+]], 46; GCN: {{buffer|flat}}_load_dword [[VAL1:v[0-9]+]], 47; GCN: v_bcnt_u32_b32{{(_e64)*}} [[MIDRESULT:v[0-9]+]], [[VAL1]], 0 48; SI: v_bcnt_u32_b32_e32 [[RESULT:v[0-9]+]], [[VAL0]], [[MIDRESULT]] 49; VI: v_bcnt_u32_b32 [[RESULT:v[0-9]+]], [[VAL0]], [[MIDRESULT]] 50; GCN: buffer_store_dword [[RESULT]], 51; GCN: s_endpgm 52 53; EG: BCNT_INT 54; EG: BCNT_INT 55define amdgpu_kernel void @v_ctpop_add_chain_i32(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalias %in0, i32 addrspace(1)* noalias %in1) nounwind { 56 %tid = call i32 @llvm.r600.read.tidig.x() 57 %in0.gep = getelementptr i32, i32 addrspace(1)* %in0, i32 %tid 58 %in1.gep = getelementptr i32, i32 addrspace(1)* %in1, i32 %tid 59 %val0 = load i32, i32 addrspace(1)* %in0.gep, align 4 60 %val1 = load i32, i32 addrspace(1)* %in1.gep, align 4 61 %ctpop0 = call i32 @llvm.ctpop.i32(i32 %val0) nounwind readnone 62 %ctpop1 = call i32 @llvm.ctpop.i32(i32 %val1) nounwind readnone 63 %add = add i32 %ctpop0, %ctpop1 64 store i32 %add, i32 addrspace(1)* %out, align 4 65 ret void 66} 67 68; FUNC-LABEL: {{^}}v_ctpop_add_sgpr_i32: 69; GCN: {{buffer|flat}}_load_dword [[VAL0:v[0-9]+]], 70; GCN: s_waitcnt 71; GCN-NEXT: v_bcnt_u32_b32{{(_e64)*}} [[RESULT:v[0-9]+]], [[VAL0]], s{{[0-9]+}} 72; GCN: buffer_store_dword [[RESULT]], 73; GCN: s_endpgm 74define amdgpu_kernel void @v_ctpop_add_sgpr_i32(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalias %in, i32 %sval) nounwind { 75 %tid = call i32 @llvm.r600.read.tidig.x() 76 %in.gep = getelementptr i32, i32 addrspace(1)* %in, i32 %tid 77 %val = load i32, i32 addrspace(1)* %in.gep, align 4 78 %ctpop = call i32 @llvm.ctpop.i32(i32 %val) nounwind readnone 79 %add = add i32 %ctpop, %sval 80 store i32 %add, i32 addrspace(1)* %out, align 4 81 ret void 82} 83 84; FUNC-LABEL: {{^}}v_ctpop_v2i32: 85; GCN: v_bcnt_u32_b32{{(_e64)*}} 86; GCN: v_bcnt_u32_b32{{(_e64)*}} 87; GCN: s_endpgm 88 89; EG: BCNT_INT 90; EG: BCNT_INT 91define amdgpu_kernel void @v_ctpop_v2i32(<2 x i32> addrspace(1)* noalias %out, <2 x i32> addrspace(1)* noalias %in) nounwind { 92 %tid = call i32 @llvm.r600.read.tidig.x() 93 %in.gep = getelementptr <2 x i32>, <2 x i32> addrspace(1)* %in, i32 %tid 94 %val = load <2 x i32>, <2 x i32> addrspace(1)* %in.gep, align 8 95 %ctpop = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %val) nounwind readnone 96 store <2 x i32> %ctpop, <2 x i32> addrspace(1)* %out, align 8 97 ret void 98} 99 100; FUNC-LABEL: {{^}}v_ctpop_v4i32: 101; GCN: v_bcnt_u32_b32{{(_e64)*}} 102; GCN: v_bcnt_u32_b32{{(_e64)*}} 103; GCN: v_bcnt_u32_b32{{(_e64)*}} 104; GCN: v_bcnt_u32_b32{{(_e64)*}} 105; GCN: s_endpgm 106 107; EG: BCNT_INT 108; EG: BCNT_INT 109; EG: BCNT_INT 110; EG: BCNT_INT 111define amdgpu_kernel void @v_ctpop_v4i32(<4 x i32> addrspace(1)* noalias %out, <4 x i32> addrspace(1)* noalias %in) nounwind { 112 %tid = call i32 @llvm.r600.read.tidig.x() 113 %in.gep = getelementptr <4 x i32>, <4 x i32> addrspace(1)* %in, i32 %tid 114 %val = load <4 x i32>, <4 x i32> addrspace(1)* %in.gep, align 16 115 %ctpop = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %val) nounwind readnone 116 store <4 x i32> %ctpop, <4 x i32> addrspace(1)* %out, align 16 117 ret void 118} 119 120; FUNC-LABEL: {{^}}v_ctpop_v8i32: 121; GCN: v_bcnt_u32_b32{{(_e64)*}} 122; GCN: v_bcnt_u32_b32{{(_e64)*}} 123; GCN: v_bcnt_u32_b32{{(_e64)*}} 124; GCN: v_bcnt_u32_b32{{(_e64)*}} 125; GCN: v_bcnt_u32_b32{{(_e64)*}} 126; GCN: v_bcnt_u32_b32{{(_e64)*}} 127; GCN: v_bcnt_u32_b32{{(_e64)*}} 128; GCN: v_bcnt_u32_b32{{(_e64)*}} 129; GCN: s_endpgm 130 131; EG: BCNT_INT 132; EG: BCNT_INT 133; EG: BCNT_INT 134; EG: BCNT_INT 135; EG: BCNT_INT 136; EG: BCNT_INT 137; EG: BCNT_INT 138; EG: BCNT_INT 139define amdgpu_kernel void @v_ctpop_v8i32(<8 x i32> addrspace(1)* noalias %out, <8 x i32> addrspace(1)* noalias %in) nounwind { 140 %tid = call i32 @llvm.r600.read.tidig.x() 141 %in.gep = getelementptr <8 x i32>, <8 x i32> addrspace(1)* %in, i32 %tid 142 %val = load <8 x i32>, <8 x i32> addrspace(1)* %in.gep, align 32 143 %ctpop = call <8 x i32> @llvm.ctpop.v8i32(<8 x i32> %val) nounwind readnone 144 store <8 x i32> %ctpop, <8 x i32> addrspace(1)* %out, align 32 145 ret void 146} 147 148; FUNC-LABEL: {{^}}v_ctpop_v16i32: 149; GCN: v_bcnt_u32_b32{{(_e64)*}} 150; GCN: v_bcnt_u32_b32{{(_e64)*}} 151; GCN: v_bcnt_u32_b32{{(_e64)*}} 152; GCN: v_bcnt_u32_b32{{(_e64)*}} 153; GCN: v_bcnt_u32_b32{{(_e64)*}} 154; GCN: v_bcnt_u32_b32{{(_e64)*}} 155; GCN: v_bcnt_u32_b32{{(_e64)*}} 156; GCN: v_bcnt_u32_b32{{(_e64)*}} 157; GCN: v_bcnt_u32_b32{{(_e64)*}} 158; GCN: v_bcnt_u32_b32{{(_e64)*}} 159; GCN: v_bcnt_u32_b32{{(_e64)*}} 160; GCN: v_bcnt_u32_b32{{(_e64)*}} 161; GCN: v_bcnt_u32_b32{{(_e64)*}} 162; GCN: v_bcnt_u32_b32{{(_e64)*}} 163; GCN: v_bcnt_u32_b32{{(_e64)*}} 164; GCN: v_bcnt_u32_b32{{(_e64)*}} 165; GCN: s_endpgm 166 167; EG: BCNT_INT 168; EG: BCNT_INT 169; EG: BCNT_INT 170; EG: BCNT_INT 171; EG: BCNT_INT 172; EG: BCNT_INT 173; EG: BCNT_INT 174; EG: BCNT_INT 175; EG: BCNT_INT 176; EG: BCNT_INT 177; EG: BCNT_INT 178; EG: BCNT_INT 179; EG: BCNT_INT 180; EG: BCNT_INT 181; EG: BCNT_INT 182; EG: BCNT_INT 183define amdgpu_kernel void @v_ctpop_v16i32(<16 x i32> addrspace(1)* noalias %out, <16 x i32> addrspace(1)* noalias %in) nounwind { 184 %tid = call i32 @llvm.r600.read.tidig.x() 185 %in.gep = getelementptr <16 x i32>, <16 x i32> addrspace(1)* %in, i32 %tid 186 %val = load <16 x i32>, <16 x i32> addrspace(1)* %in.gep, align 32 187 %ctpop = call <16 x i32> @llvm.ctpop.v16i32(<16 x i32> %val) nounwind readnone 188 store <16 x i32> %ctpop, <16 x i32> addrspace(1)* %out, align 32 189 ret void 190} 191 192; FUNC-LABEL: {{^}}v_ctpop_i32_add_inline_constant: 193; GCN: {{buffer|flat}}_load_dword [[VAL:v[0-9]+]], 194; GCN: v_bcnt_u32_b32{{(_e64)*}} [[RESULT:v[0-9]+]], [[VAL]], 4 195; GCN: buffer_store_dword [[RESULT]], 196; GCN: s_endpgm 197 198; EG: BCNT_INT 199define amdgpu_kernel void @v_ctpop_i32_add_inline_constant(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalias %in) nounwind { 200 %tid = call i32 @llvm.r600.read.tidig.x() 201 %in.gep = getelementptr i32, i32 addrspace(1)* %in, i32 %tid 202 %val = load i32, i32 addrspace(1)* %in.gep, align 4 203 %ctpop = call i32 @llvm.ctpop.i32(i32 %val) nounwind readnone 204 %add = add i32 %ctpop, 4 205 store i32 %add, i32 addrspace(1)* %out, align 4 206 ret void 207} 208 209; FUNC-LABEL: {{^}}v_ctpop_i32_add_inline_constant_inv: 210; GCN: {{buffer|flat}}_load_dword [[VAL:v[0-9]+]], 211; GCN: v_bcnt_u32_b32{{(_e64)*}} [[RESULT:v[0-9]+]], [[VAL]], 4 212; GCN: buffer_store_dword [[RESULT]], 213; GCN: s_endpgm 214 215; EG: BCNT_INT 216define amdgpu_kernel void @v_ctpop_i32_add_inline_constant_inv(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalias %in) nounwind { 217 %tid = call i32 @llvm.r600.read.tidig.x() 218 %in.gep = getelementptr i32, i32 addrspace(1)* %in, i32 %tid 219 %val = load i32, i32 addrspace(1)* %in.gep, align 4 220 %ctpop = call i32 @llvm.ctpop.i32(i32 %val) nounwind readnone 221 %add = add i32 4, %ctpop 222 store i32 %add, i32 addrspace(1)* %out, align 4 223 ret void 224} 225 226; FUNC-LABEL: {{^}}v_ctpop_i32_add_literal: 227; GCN-DAG: {{buffer|flat}}_load_dword [[VAL:v[0-9]+]], 228; GCN-DAG: v_mov_b32_e32 [[LIT:v[0-9]+]], 0x1869f 229; SI: v_bcnt_u32_b32_e32 [[RESULT:v[0-9]+]], [[VAL]], [[LIT]] 230; VI: v_bcnt_u32_b32 [[RESULT:v[0-9]+]], [[VAL]], [[LIT]] 231; GCN: buffer_store_dword [[RESULT]], 232; GCN: s_endpgm 233define amdgpu_kernel void @v_ctpop_i32_add_literal(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalias %in) nounwind { 234 %tid = call i32 @llvm.r600.read.tidig.x() 235 %in.gep = getelementptr i32, i32 addrspace(1)* %in, i32 %tid 236 %val = load i32, i32 addrspace(1)* %in.gep, align 4 237 %ctpop = call i32 @llvm.ctpop.i32(i32 %val) nounwind readnone 238 %add = add i32 %ctpop, 99999 239 store i32 %add, i32 addrspace(1)* %out, align 4 240 ret void 241} 242 243; FUNC-LABEL: {{^}}v_ctpop_i32_add_var: 244; GCN-DAG: {{buffer|flat}}_load_dword [[VAL:v[0-9]+]], 245; GCN-DAG: s_load_dword [[VAR:s[0-9]+]], 246; GCN: v_bcnt_u32_b32{{(_e64)*}} [[RESULT:v[0-9]+]], [[VAL]], [[VAR]] 247; GCN: buffer_store_dword [[RESULT]], 248; GCN: s_endpgm 249 250; EG: BCNT_INT 251define amdgpu_kernel void @v_ctpop_i32_add_var(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalias %in, i32 %const) nounwind { 252 %tid = call i32 @llvm.r600.read.tidig.x() 253 %in.gep = getelementptr i32, i32 addrspace(1)* %in, i32 %tid 254 %val = load i32, i32 addrspace(1)* %in.gep, align 4 255 %ctpop = call i32 @llvm.ctpop.i32(i32 %val) nounwind readnone 256 %add = add i32 %ctpop, %const 257 store i32 %add, i32 addrspace(1)* %out, align 4 258 ret void 259} 260 261; FUNC-LABEL: {{^}}v_ctpop_i32_add_var_inv: 262; GCN-DAG: {{buffer|flat}}_load_dword [[VAL:v[0-9]+]], 263; GCN-DAG: s_load_dword [[VAR:s[0-9]+]], 264; GCN: v_bcnt_u32_b32{{(_e64)*}} [[RESULT:v[0-9]+]], [[VAL]], [[VAR]] 265; GCN: buffer_store_dword [[RESULT]], 266; GCN: s_endpgm 267 268; EG: BCNT_INT 269define amdgpu_kernel void @v_ctpop_i32_add_var_inv(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalias %in, i32 %const) nounwind { 270 %tid = call i32 @llvm.r600.read.tidig.x() 271 %in.gep = getelementptr i32, i32 addrspace(1)* %in, i32 %tid 272 %val = load i32, i32 addrspace(1)* %in.gep, align 4 273 %ctpop = call i32 @llvm.ctpop.i32(i32 %val) nounwind readnone 274 %add = add i32 %const, %ctpop 275 store i32 %add, i32 addrspace(1)* %out, align 4 276 ret void 277} 278 279; FUNC-LABEL: {{^}}v_ctpop_i32_add_vvar_inv: 280; SI: buffer_load_dword [[VAR:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], 0 addr64 281; SI: buffer_load_dword [[VAL:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], 0 addr64 282; SI: v_bcnt_u32_b32_e32 [[RESULT:v[0-9]+]], [[VAR]], [[VAL]] 283; VI: flat_load_dword [[VAL:v[0-9]+]], v[{{[0-9]+:[0-9]+}}] 284; VI: flat_load_dword [[VAR:v[0-9]+]], v[{{[0-9]+:[0-9]+}}] 285; VI: v_bcnt_u32_b32 [[RESULT:v[0-9]+]], [[VAL]], [[VAR]] 286; GCN: buffer_store_dword [[RESULT]], 287; GCN: s_endpgm 288 289; EG: BCNT_INT 290define amdgpu_kernel void @v_ctpop_i32_add_vvar_inv(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalias %in, i32 addrspace(1)* noalias %constptr) nounwind { 291 %tid = call i32 @llvm.r600.read.tidig.x() 292 %in.gep = getelementptr i32, i32 addrspace(1)* %in, i32 %tid 293 %val = load i32, i32 addrspace(1)* %in.gep, align 4 294 %ctpop = call i32 @llvm.ctpop.i32(i32 %val) nounwind readnone 295 %gep = getelementptr i32, i32 addrspace(1)* %constptr, i32 %tid 296 %const = load i32, i32 addrspace(1)* %gep, align 4 297 %add = add i32 %const, %ctpop 298 store i32 %add, i32 addrspace(1)* %out, align 4 299 ret void 300} 301 302; FIXME: We currently disallow SALU instructions in all branches, 303; but there are some cases when the should be allowed. 304 305; FUNC-LABEL: {{^}}ctpop_i32_in_br: 306; SI: s_load_dword [[VAL:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0xd 307; VI: s_load_dword [[VAL:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0x34 308; GCN: s_bcnt1_i32_b32 [[SRESULT:s[0-9]+]], [[VAL]] 309; GCN: v_mov_b32_e32 [[RESULT]], [[SRESULT]] 310; GCN: buffer_store_dword [[RESULT]], 311; GCN: s_endpgm 312; EG: BCNT_INT 313define amdgpu_kernel void @ctpop_i32_in_br(i32 addrspace(1)* %out, i32 addrspace(1)* %in, i32 %ctpop_arg, i32 %cond) { 314entry: 315 %tmp0 = icmp eq i32 %cond, 0 316 br i1 %tmp0, label %if, label %else 317 318if: 319 %tmp2 = call i32 @llvm.ctpop.i32(i32 %ctpop_arg) 320 br label %endif 321 322else: 323 %tmp3 = getelementptr i32, i32 addrspace(1)* %in, i32 1 324 %tmp4 = load i32, i32 addrspace(1)* %tmp3 325 br label %endif 326 327endif: 328 %tmp5 = phi i32 [%tmp2, %if], [%tmp4, %else] 329 store i32 %tmp5, i32 addrspace(1)* %out 330 ret void 331} 332