1; RUN: opt -mtriple=amdgcn-- -mcpu=gfx600 -S -amdgpu-unify-divergent-exit-nodes -verify -structurizecfg -verify -si-annotate-control-flow -simplifycfg-require-and-preserve-domtree=1 %s | FileCheck -check-prefix=IR %s 2; RUN: opt -mtriple=amdgcn-- -mcpu=gfx1100 -mattr=-wavefrontsize32,+wavefrontsize64 -S -amdgpu-unify-divergent-exit-nodes -verify -structurizecfg -verify -si-annotate-control-flow -simplifycfg-require-and-preserve-domtree=1 %s | FileCheck -check-prefix=IR %s 3; RUN: llc -march=amdgcn -verify-machineinstrs -simplifycfg-require-and-preserve-domtree=1 < %s | FileCheck -check-prefix=GCN %s 4 5; Add an extra verifier runs. There were some cases where invalid IR 6; was produced but happened to be fixed by the later passes. 7 8; Make sure divergent control flow with multiple exits from a region 9; is properly handled. UnifyFunctionExitNodes should be run before 10; StructurizeCFG. 11 12; IR-LABEL: @multi_divergent_region_exit_ret_ret( 13; IR: %0 = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %Pivot) 14; IR: %1 = extractvalue { i1, i64 } %0, 0 15; IR: %2 = extractvalue { i1, i64 } %0, 1 16; IR: br i1 %1, label %LeafBlock1, label %Flow 17 18; IR: Flow: 19; IR: %3 = phi i1 [ true, %LeafBlock1 ], [ false, %entry ] 20; IR: %4 = phi i1 [ %SwitchLeaf2, %LeafBlock1 ], [ false, %entry ] 21; IR: %5 = call { i1, i64 } @llvm.amdgcn.else.i64.i64(i64 %2) 22; IR: %6 = extractvalue { i1, i64 } %5, 0 23; IR: %7 = extractvalue { i1, i64 } %5, 1 24; IR: br i1 %6, label %LeafBlock, label %Flow1 25 26; IR: LeafBlock: 27; IR: br label %Flow1 28 29; IR: LeafBlock1: 30; IR: br label %Flow{{$}} 31 32; IR: Flow2: 33; IR: %8 = phi i1 [ false, %exit1 ], [ %12, %Flow1 ] 34; IR: call void @llvm.amdgcn.end.cf.i64(i64 %16) 35; IR: %9 = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %8) 36; IR: %10 = extractvalue { i1, i64 } %9, 0 37; IR: %11 = extractvalue { i1, i64 } %9, 1 38; IR: br i1 %10, label %exit0, label %UnifiedReturnBlock 39 40; IR: exit0: 41; IR: store volatile i32 9, i32 addrspace(1)* undef 42; IR: br label %UnifiedReturnBlock 43 44; IR: Flow1: 45; IR: %12 = phi i1 [ %SwitchLeaf, %LeafBlock ], [ %3, %Flow ] 46; IR: %13 = phi i1 [ %SwitchLeaf.inv, %LeafBlock ], [ %4, %Flow ] 47; IR: call void @llvm.amdgcn.end.cf.i64(i64 %7) 48; IR: %14 = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %13) 49; IR: %15 = extractvalue { i1, i64 } %14, 0 50; IR: %16 = extractvalue { i1, i64 } %14, 1 51; IR: br i1 %15, label %exit1, label %Flow2 52 53; IR: exit1: 54; IR: store volatile i32 17, i32 addrspace(3)* undef 55; IR: br label %Flow2 56 57; IR: UnifiedReturnBlock: 58; IR: call void @llvm.amdgcn.end.cf.i64(i64 %11) 59; IR: ret void 60 61 62; GCN-LABEL: {{^}}multi_divergent_region_exit_ret_ret: 63 64; GCN-DAG: s_mov_b64 [[EXIT1:s\[[0-9]+:[0-9]+\]]], 0 65; GCN-DAG: v_cmp_lt_i32_e32 vcc, 1, 66; GCN-DAG: s_mov_b64 [[EXIT0:s\[[0-9]+:[0-9]+\]]], 0 67; GCN-DAG: s_and_saveexec_b64 68; GCN-DAG: s_xor_b64 69 70; GCN: ; %LeafBlock1 71; GCN-NEXT: s_mov_b64 [[EXIT0]], exec 72; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 2, 73; GCN-NEXT: s_and_b64 [[EXIT1]], vcc, exec 74 75; GCN: ; %Flow 76; GCN-NEXT: s_andn2_saveexec_b64 77 78; GCN: ; %LeafBlock 79; GCN-DAG: v_cmp_eq_u32_e32 vcc, 1, 80; GCN-DAG: v_cmp_ne_u32_e64 [[INV:s\[[0-9]+:[0-9]+\]]], 1, 81; GCN-DAG: s_andn2_b64 [[EXIT0]], [[EXIT0]], exec 82; GCN-DAG: s_andn2_b64 [[EXIT1]], [[EXIT1]], exec 83; GCN-DAG: s_and_b64 [[TMP0:s\[[0-9]+:[0-9]+\]]], vcc, exec 84; GCN-DAG: s_and_b64 [[TMP1:s\[[0-9]+:[0-9]+\]]], [[INV]], exec 85; GCN-DAG: s_or_b64 [[EXIT0]], [[EXIT0]], [[TMP0]] 86; GCN-DAG: s_or_b64 [[EXIT1]], [[EXIT1]], [[TMP1]] 87 88; GCN: ; %Flow4 89; GCN-NEXT: s_or_b64 exec, exec, 90; GCN-NEXT: s_and_saveexec_b64 {{s\[[0-9]+:[0-9]+\]}}, [[EXIT1]] 91; GCN-NEXT: s_xor_b64 92 93; GCN: ; %exit1 94; GCN-DAG: ds_write_b32 95; GCN-DAG: s_andn2_b64 [[EXIT0]], [[EXIT0]], exec 96 97; GCN: ; %Flow5 98; GCN-NEXT: s_or_b64 exec, exec, 99; GCN-NEXT: s_and_saveexec_b64 {{s\[[0-9]+:[0-9]+\]}}, [[EXIT0]] 100 101; GCN: ; %exit0 102; GCN: buffer_store_dword 103 104; GCN: ; %UnifiedReturnBlock 105; GCN-NEXT: s_endpgm 106define amdgpu_kernel void @multi_divergent_region_exit_ret_ret(i32 addrspace(1)* nocapture %arg0, i32 addrspace(1)* nocapture %arg1, i32 addrspace(1)* nocapture %arg2) #0 { 107entry: 108 %tmp = tail call i32 @llvm.amdgcn.workitem.id.x() #1 109 %tmp1 = add i32 0, %tmp 110 %tmp2 = zext i32 %tmp1 to i64 111 %tmp3 = add i64 0, %tmp2 112 %tmp4 = shl i64 %tmp3, 32 113 %tmp5 = ashr exact i64 %tmp4, 32 114 %tmp6 = getelementptr inbounds i32, i32 addrspace(1)* %arg0, i64 %tmp5 115 %tmp7 = load i32, i32 addrspace(1)* %tmp6, align 4 116 %tmp8 = sext i32 %tmp7 to i64 117 %tmp9 = getelementptr inbounds i32, i32 addrspace(1)* %arg1, i64 %tmp8 118 %tmp10 = load i32, i32 addrspace(1)* %tmp9, align 4 119 %tmp13 = zext i32 %tmp10 to i64 120 %tmp14 = getelementptr inbounds i32, i32 addrspace(1)* %arg2, i64 %tmp13 121 %tmp16 = load i32, i32 addrspace(1)* %tmp14, align 16 122 %Pivot = icmp slt i32 %tmp16, 2 123 br i1 %Pivot, label %LeafBlock, label %LeafBlock1 124 125LeafBlock: ; preds = %entry 126 %SwitchLeaf = icmp eq i32 %tmp16, 1 127 br i1 %SwitchLeaf, label %exit0, label %exit1 128 129LeafBlock1: ; preds = %entry 130 %SwitchLeaf2 = icmp eq i32 %tmp16, 2 131 br i1 %SwitchLeaf2, label %exit0, label %exit1 132 133exit0: ; preds = %LeafBlock, %LeafBlock1 134 store volatile i32 9, i32 addrspace(1)* undef 135 ret void 136 137exit1: ; preds = %LeafBlock, %LeafBlock1 138 store volatile i32 17, i32 addrspace(3)* undef 139 ret void 140} 141 142; IR-LABEL: @multi_divergent_region_exit_unreachable_unreachable( 143; IR: %0 = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %Pivot) 144 145; IR: %5 = call { i1, i64 } @llvm.amdgcn.else.i64.i64(i64 %2) 146 147; IR: %8 = phi i1 [ false, %exit1 ], [ %12, %Flow1 ] 148; IR: call void @llvm.amdgcn.end.cf.i64(i64 %16) 149; IR: %9 = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %8) 150; IR: br i1 %10, label %exit0, label %UnifiedUnreachableBlock 151 152 153; IR: UnifiedUnreachableBlock: 154; IR-NEXT: unreachable 155 156 157; FIXME: Probably should insert an s_endpgm anyway. 158; GCN-LABEL: {{^}}multi_divergent_region_exit_unreachable_unreachable: 159; GCN: ; %UnifiedUnreachableBlock 160; GCN-NEXT: .Lfunc_end 161define amdgpu_kernel void @multi_divergent_region_exit_unreachable_unreachable(i32 addrspace(1)* nocapture %arg0, i32 addrspace(1)* nocapture %arg1, i32 addrspace(1)* nocapture %arg2) #0 { 162entry: 163 %tmp = tail call i32 @llvm.amdgcn.workitem.id.x() #1 164 %tmp1 = add i32 0, %tmp 165 %tmp2 = zext i32 %tmp1 to i64 166 %tmp3 = add i64 0, %tmp2 167 %tmp4 = shl i64 %tmp3, 32 168 %tmp5 = ashr exact i64 %tmp4, 32 169 %tmp6 = getelementptr inbounds i32, i32 addrspace(1)* %arg0, i64 %tmp5 170 %tmp7 = load i32, i32 addrspace(1)* %tmp6, align 4 171 %tmp8 = sext i32 %tmp7 to i64 172 %tmp9 = getelementptr inbounds i32, i32 addrspace(1)* %arg1, i64 %tmp8 173 %tmp10 = load i32, i32 addrspace(1)* %tmp9, align 4 174 %tmp13 = zext i32 %tmp10 to i64 175 %tmp14 = getelementptr inbounds i32, i32 addrspace(1)* %arg2, i64 %tmp13 176 %tmp16 = load i32, i32 addrspace(1)* %tmp14, align 16 177 %Pivot = icmp slt i32 %tmp16, 2 178 br i1 %Pivot, label %LeafBlock, label %LeafBlock1 179 180LeafBlock: ; preds = %entry 181 %SwitchLeaf = icmp eq i32 %tmp16, 1 182 br i1 %SwitchLeaf, label %exit0, label %exit1 183 184LeafBlock1: ; preds = %entry 185 %SwitchLeaf2 = icmp eq i32 %tmp16, 2 186 br i1 %SwitchLeaf2, label %exit0, label %exit1 187 188exit0: ; preds = %LeafBlock, %LeafBlock1 189 store volatile i32 9, i32 addrspace(1)* undef 190 unreachable 191 192exit1: ; preds = %LeafBlock, %LeafBlock1 193 store volatile i32 17, i32 addrspace(3)* undef 194 unreachable 195} 196 197; IR-LABEL: @multi_exit_region_divergent_ret_uniform_ret( 198; IR: %divergent.cond0 = icmp sge i32 %tmp16, 2 199; IR: llvm.amdgcn.if 200; IR: br i1 201 202; IR: {{^}}Flow: 203; IR: %3 = phi i1 [ true, %LeafBlock1 ], [ false, %entry ] 204; IR: %4 = phi i1 [ %uniform.cond0, %LeafBlock1 ], [ false, %entry ] 205; IR: %5 = call { i1, i64 } @llvm.amdgcn.else.i64.i64(i64 %2) 206; IR: br i1 %6, label %LeafBlock, label %Flow1 207 208; IR: {{^}}LeafBlock: 209; IR: %divergent.cond1 = icmp eq i32 %tmp16, 1 210; IR: br label %Flow1 211 212; IR: LeafBlock1: 213; IR: %uniform.cond0 = icmp ne i32 %arg3, 2 214; IR: br label %Flow 215 216; IR: Flow2: 217; IR: %8 = phi i1 [ false, %exit1 ], [ %12, %Flow1 ] 218; IR: call void @llvm.amdgcn.end.cf.i64(i64 %16) 219; IR: %9 = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %8) 220; IR: br i1 %10, label %exit0, label %UnifiedReturnBlock 221 222; IR: exit0: 223; IR: store volatile i32 9, i32 addrspace(1)* undef 224; IR: br label %UnifiedReturnBlock 225 226; IR: {{^}}Flow1: 227; IR: %12 = phi i1 [ %divergent.cond1, %LeafBlock ], [ %3, %Flow ] 228; IR: %13 = phi i1 [ %divergent.cond1.inv, %LeafBlock ], [ %4, %Flow ] 229; IR: call void @llvm.amdgcn.end.cf.i64(i64 %7) 230; IR: %14 = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %13) 231; IR: %15 = extractvalue { i1, i64 } %14, 0 232; IR: %16 = extractvalue { i1, i64 } %14, 1 233; IR: br i1 %15, label %exit1, label %Flow2 234 235; IR: exit1: 236; IR: store volatile i32 17, i32 addrspace(3)* undef 237; IR: br label %Flow2 238 239; IR: UnifiedReturnBlock: 240; IR: call void @llvm.amdgcn.end.cf.i64(i64 %11) 241; IR: ret void 242define amdgpu_kernel void @multi_exit_region_divergent_ret_uniform_ret(i32 addrspace(1)* nocapture %arg0, i32 addrspace(1)* nocapture %arg1, i32 addrspace(1)* nocapture %arg2, i32 %arg3) #0 { 243entry: 244 %tmp = tail call i32 @llvm.amdgcn.workitem.id.x() #1 245 %tmp1 = add i32 0, %tmp 246 %tmp2 = zext i32 %tmp1 to i64 247 %tmp3 = add i64 0, %tmp2 248 %tmp4 = shl i64 %tmp3, 32 249 %tmp5 = ashr exact i64 %tmp4, 32 250 %tmp6 = getelementptr inbounds i32, i32 addrspace(1)* %arg0, i64 %tmp5 251 %tmp7 = load i32, i32 addrspace(1)* %tmp6, align 4 252 %tmp8 = sext i32 %tmp7 to i64 253 %tmp9 = getelementptr inbounds i32, i32 addrspace(1)* %arg1, i64 %tmp8 254 %tmp10 = load i32, i32 addrspace(1)* %tmp9, align 4 255 %tmp13 = zext i32 %tmp10 to i64 256 %tmp14 = getelementptr inbounds i32, i32 addrspace(1)* %arg2, i64 %tmp13 257 %tmp16 = load i32, i32 addrspace(1)* %tmp14, align 16 258 %divergent.cond0 = icmp slt i32 %tmp16, 2 259 br i1 %divergent.cond0, label %LeafBlock, label %LeafBlock1 260 261LeafBlock: ; preds = %entry 262 %divergent.cond1 = icmp eq i32 %tmp16, 1 263 br i1 %divergent.cond1, label %exit0, label %exit1 264 265LeafBlock1: ; preds = %entry 266 %uniform.cond0 = icmp eq i32 %arg3, 2 267 br i1 %uniform.cond0, label %exit0, label %exit1 268 269exit0: ; preds = %LeafBlock, %LeafBlock1 270 store volatile i32 9, i32 addrspace(1)* undef 271 ret void 272 273exit1: ; preds = %LeafBlock, %LeafBlock1 274 store volatile i32 17, i32 addrspace(3)* undef 275 ret void 276} 277 278; IR-LABEL: @multi_exit_region_uniform_ret_divergent_ret( 279; IR: %0 = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %Pivot) 280; IR: br i1 %1, label %LeafBlock1, label %Flow 281 282; IR: Flow: 283; IR: %3 = phi i1 [ true, %LeafBlock1 ], [ false, %entry ] 284; IR: %4 = phi i1 [ %SwitchLeaf2, %LeafBlock1 ], [ false, %entry ] 285; IR: %5 = call { i1, i64 } @llvm.amdgcn.else.i64.i64(i64 %2) 286 287; IR: %8 = phi i1 [ false, %exit1 ], [ %12, %Flow1 ] 288; IR: call void @llvm.amdgcn.end.cf.i64(i64 %16) 289; IR: %9 = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %8) 290 291define amdgpu_kernel void @multi_exit_region_uniform_ret_divergent_ret(i32 addrspace(1)* nocapture %arg0, i32 addrspace(1)* nocapture %arg1, i32 addrspace(1)* nocapture %arg2, i32 %arg3) #0 { 292entry: 293 %tmp = tail call i32 @llvm.amdgcn.workitem.id.x() #1 294 %tmp1 = add i32 0, %tmp 295 %tmp2 = zext i32 %tmp1 to i64 296 %tmp3 = add i64 0, %tmp2 297 %tmp4 = shl i64 %tmp3, 32 298 %tmp5 = ashr exact i64 %tmp4, 32 299 %tmp6 = getelementptr inbounds i32, i32 addrspace(1)* %arg0, i64 %tmp5 300 %tmp7 = load i32, i32 addrspace(1)* %tmp6, align 4 301 %tmp8 = sext i32 %tmp7 to i64 302 %tmp9 = getelementptr inbounds i32, i32 addrspace(1)* %arg1, i64 %tmp8 303 %tmp10 = load i32, i32 addrspace(1)* %tmp9, align 4 304 %tmp13 = zext i32 %tmp10 to i64 305 %tmp14 = getelementptr inbounds i32, i32 addrspace(1)* %arg2, i64 %tmp13 306 %tmp16 = load i32, i32 addrspace(1)* %tmp14, align 16 307 %Pivot = icmp slt i32 %tmp16, 2 308 br i1 %Pivot, label %LeafBlock, label %LeafBlock1 309 310LeafBlock: ; preds = %entry 311 %SwitchLeaf = icmp eq i32 %arg3, 1 312 br i1 %SwitchLeaf, label %exit0, label %exit1 313 314LeafBlock1: ; preds = %entry 315 %SwitchLeaf2 = icmp eq i32 %tmp16, 2 316 br i1 %SwitchLeaf2, label %exit0, label %exit1 317 318exit0: ; preds = %LeafBlock, %LeafBlock1 319 store volatile i32 9, i32 addrspace(1)* undef 320 ret void 321 322exit1: ; preds = %LeafBlock, %LeafBlock1 323 store volatile i32 17, i32 addrspace(3)* undef 324 ret void 325} 326 327; IR-LABEL: @multi_divergent_region_exit_ret_ret_return_value( 328; IR: Flow2: 329; IR: %8 = phi i1 [ false, %exit1 ], [ %12, %Flow1 ] 330; IR: call void @llvm.amdgcn.end.cf.i64(i64 %16) 331 332; IR: UnifiedReturnBlock: 333; IR: %UnifiedRetVal = phi float [ 2.000000e+00, %Flow2 ], [ 1.000000e+00, %exit0 ] 334; IR: call void @llvm.amdgcn.end.cf.i64(i64 %11) 335; IR: ret float %UnifiedRetVal 336define amdgpu_ps float @multi_divergent_region_exit_ret_ret_return_value(i32 %vgpr) #0 { 337entry: 338 %Pivot = icmp slt i32 %vgpr, 2 339 br i1 %Pivot, label %LeafBlock, label %LeafBlock1 340 341LeafBlock: ; preds = %entry 342 %SwitchLeaf = icmp eq i32 %vgpr, 1 343 br i1 %SwitchLeaf, label %exit0, label %exit1 344 345LeafBlock1: ; preds = %entry 346 %SwitchLeaf2 = icmp eq i32 %vgpr, 2 347 br i1 %SwitchLeaf2, label %exit0, label %exit1 348 349exit0: ; preds = %LeafBlock, %LeafBlock1 350 store i32 9, i32 addrspace(1)* undef 351 ret float 1.0 352 353exit1: ; preds = %LeafBlock, %LeafBlock1 354 store i32 17, i32 addrspace(3)* undef 355 ret float 2.0 356} 357 358; IR-LABEL: @uniform_branch_to_multi_divergent_region_exit_ret_ret_return_value( 359 360; GCN-LABEL: {{^}}uniform_branch_to_multi_divergent_region_exit_ret_ret_return_value: 361; GCN: s_cmp_gt_i32 s0, 1 362; GCN: s_cbranch_scc0 [[FLOW:.LBB[0-9]+_[0-9]+]] 363 364; GCN: v_cmp_ne_u32_e32 vcc, 7, v0 365 366; GCN: {{^}}[[FLOW]]: 367 368; GCN: s_or_b64 exec, exec 369; GCN: v_mov_b32_e32 v0, 2.0 370; GCN-NOT: s_and_b64 exec, exec 371; GCN: v_mov_b32_e32 v0, 1.0 372 373; GCN: {{^.LBB[0-9]+_[0-9]+}}: ; %UnifiedReturnBlock 374; GCN-NEXT: s_or_b64 exec, exec 375; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) 376; GCN-NEXT: ; return 377 378define amdgpu_ps float @uniform_branch_to_multi_divergent_region_exit_ret_ret_return_value(i32 inreg %sgpr, i32 %vgpr) #0 { 379entry: 380 %uniform.cond = icmp slt i32 %sgpr, 2 381 br i1 %uniform.cond, label %LeafBlock, label %LeafBlock1 382 383LeafBlock: ; preds = %entry 384 %divergent.cond0 = icmp eq i32 %vgpr, 3 385 br i1 %divergent.cond0, label %exit0, label %exit1 386 387LeafBlock1: ; preds = %entry 388 %divergent.cond1 = icmp eq i32 %vgpr, 7 389 br i1 %divergent.cond1, label %exit0, label %exit1 390 391exit0: ; preds = %LeafBlock, %LeafBlock1 392 store i32 9, i32 addrspace(1)* undef 393 ret float 1.0 394 395exit1: ; preds = %LeafBlock, %LeafBlock1 396 store i32 17, i32 addrspace(3)* undef 397 ret float 2.0 398} 399 400; IR-LABEL: @multi_divergent_region_exit_ret_unreachable( 401; IR: %0 = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %Pivot) 402 403; IR: Flow: 404; IR: %3 = phi i1 [ true, %LeafBlock1 ], [ false, %entry ] 405; IR: %4 = phi i1 [ %SwitchLeaf2, %LeafBlock1 ], [ false, %entry ] 406; IR: %5 = call { i1, i64 } @llvm.amdgcn.else.i64.i64(i64 %2) 407 408; IR: Flow2: 409; IR: %8 = phi i1 [ false, %exit1 ], [ %12, %Flow1 ] 410; IR: call void @llvm.amdgcn.end.cf.i64(i64 %16) 411; IR: %9 = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %8) 412; IR: br i1 %10, label %exit0, label %UnifiedReturnBlock 413 414; IR: exit0: 415; IR-NEXT: store volatile i32 17, i32 addrspace(3)* undef 416; IR-NEXT: br label %UnifiedReturnBlock 417 418; IR: Flow1: 419; IR: %12 = phi i1 [ %SwitchLeaf, %LeafBlock ], [ %3, %Flow ] 420; IR: %13 = phi i1 [ %SwitchLeaf.inv, %LeafBlock ], [ %4, %Flow ] 421; IR: call void @llvm.amdgcn.end.cf.i64(i64 %7) 422; IR: %14 = call { i1, i64 } @llvm.amdgcn.if.i64(i1 %13) 423; IR: %15 = extractvalue { i1, i64 } %14, 0 424; IR: %16 = extractvalue { i1, i64 } %14, 1 425; IR: br i1 %15, label %exit1, label %Flow2 426 427; IR: exit1: 428; IR-NEXT: store volatile i32 9, i32 addrspace(1)* undef 429; IR-NEXT: call void @llvm.amdgcn.unreachable() 430; IR-NEXT: br label %Flow2 431 432; IR: UnifiedReturnBlock: 433; IR-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 %11) 434; IR-NEXT: ret void 435define amdgpu_kernel void @multi_divergent_region_exit_ret_unreachable(i32 addrspace(1)* nocapture %arg0, i32 addrspace(1)* nocapture %arg1, i32 addrspace(1)* nocapture %arg2) #0 { 436entry: 437 %tmp = tail call i32 @llvm.amdgcn.workitem.id.x() #1 438 %tmp1 = add i32 0, %tmp 439 %tmp2 = zext i32 %tmp1 to i64 440 %tmp3 = add i64 0, %tmp2 441 %tmp4 = shl i64 %tmp3, 32 442 %tmp5 = ashr exact i64 %tmp4, 32 443 %tmp6 = getelementptr inbounds i32, i32 addrspace(1)* %arg0, i64 %tmp5 444 %tmp7 = load i32, i32 addrspace(1)* %tmp6, align 4 445 %tmp8 = sext i32 %tmp7 to i64 446 %tmp9 = getelementptr inbounds i32, i32 addrspace(1)* %arg1, i64 %tmp8 447 %tmp10 = load i32, i32 addrspace(1)* %tmp9, align 4 448 %tmp13 = zext i32 %tmp10 to i64 449 %tmp14 = getelementptr inbounds i32, i32 addrspace(1)* %arg2, i64 %tmp13 450 %tmp16 = load i32, i32 addrspace(1)* %tmp14, align 16 451 %Pivot = icmp slt i32 %tmp16, 2 452 br i1 %Pivot, label %LeafBlock, label %LeafBlock1 453 454LeafBlock: ; preds = %entry 455 %SwitchLeaf = icmp eq i32 %tmp16, 1 456 br i1 %SwitchLeaf, label %exit0, label %exit1 457 458LeafBlock1: ; preds = %entry 459 %SwitchLeaf2 = icmp eq i32 %tmp16, 2 460 br i1 %SwitchLeaf2, label %exit0, label %exit1 461 462exit0: ; preds = %LeafBlock, %LeafBlock1 463 store volatile i32 17, i32 addrspace(3)* undef 464 ret void 465 466exit1: ; preds = %LeafBlock, %LeafBlock1 467 store volatile i32 9, i32 addrspace(1)* undef 468 unreachable 469} 470 471; The non-uniformity of the branch to the exiting blocks requires 472; looking at transitive predecessors. 473 474; IR-LABEL: @indirect_multi_divergent_region_exit_ret_unreachable( 475 476; IR: exit0: ; preds = %Flow2 477; IR-NEXT: store volatile i32 17, i32 addrspace(3)* undef 478; IR-NEXT: br label %UnifiedReturnBlock 479 480 481; IR: indirect.exit1: 482; IR: %load = load volatile i32, i32 addrspace(1)* undef 483; IR: store volatile i32 %load, i32 addrspace(1)* undef 484; IR: store volatile i32 9, i32 addrspace(1)* undef 485; IR: call void @llvm.amdgcn.unreachable() 486; IR-NEXT: br label %Flow2 487 488; IR: UnifiedReturnBlock: ; preds = %exit0, %Flow2 489; IR-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 %11) 490; IR-NEXT: ret void 491define amdgpu_kernel void @indirect_multi_divergent_region_exit_ret_unreachable(i32 addrspace(1)* nocapture %arg0, i32 addrspace(1)* nocapture %arg1, i32 addrspace(1)* nocapture %arg2) #0 { 492entry: 493 %tmp = tail call i32 @llvm.amdgcn.workitem.id.x() #1 494 %tmp1 = add i32 0, %tmp 495 %tmp2 = zext i32 %tmp1 to i64 496 %tmp3 = add i64 0, %tmp2 497 %tmp4 = shl i64 %tmp3, 32 498 %tmp5 = ashr exact i64 %tmp4, 32 499 %tmp6 = getelementptr inbounds i32, i32 addrspace(1)* %arg0, i64 %tmp5 500 %tmp7 = load i32, i32 addrspace(1)* %tmp6, align 4 501 %tmp8 = sext i32 %tmp7 to i64 502 %tmp9 = getelementptr inbounds i32, i32 addrspace(1)* %arg1, i64 %tmp8 503 %tmp10 = load i32, i32 addrspace(1)* %tmp9, align 4 504 %tmp13 = zext i32 %tmp10 to i64 505 %tmp14 = getelementptr inbounds i32, i32 addrspace(1)* %arg2, i64 %tmp13 506 %tmp16 = load i32, i32 addrspace(1)* %tmp14, align 16 507 %Pivot = icmp slt i32 %tmp16, 2 508 br i1 %Pivot, label %LeafBlock, label %LeafBlock1 509 510LeafBlock: ; preds = %entry 511 %SwitchLeaf = icmp eq i32 %tmp16, 1 512 br i1 %SwitchLeaf, label %exit0, label %indirect.exit1 513 514LeafBlock1: ; preds = %entry 515 %SwitchLeaf2 = icmp eq i32 %tmp16, 2 516 br i1 %SwitchLeaf2, label %exit0, label %indirect.exit1 517 518exit0: ; preds = %LeafBlock, %LeafBlock1 519 store volatile i32 17, i32 addrspace(3)* undef 520 ret void 521 522indirect.exit1: 523 %load = load volatile i32, i32 addrspace(1)* undef 524 store volatile i32 %load, i32 addrspace(1)* undef 525 br label %exit1 526 527exit1: ; preds = %LeafBlock, %LeafBlock1 528 store volatile i32 9, i32 addrspace(1)* undef 529 unreachable 530} 531 532; IR-LABEL: @multi_divergent_region_exit_ret_switch( 533define amdgpu_kernel void @multi_divergent_region_exit_ret_switch(i32 addrspace(1)* nocapture %arg0, i32 addrspace(1)* nocapture %arg1, i32 addrspace(1)* nocapture %arg2) #0 { 534entry: 535 %tmp = tail call i32 @llvm.amdgcn.workitem.id.x() #1 536 %tmp1 = add i32 0, %tmp 537 %tmp2 = zext i32 %tmp1 to i64 538 %tmp3 = add i64 0, %tmp2 539 %tmp4 = shl i64 %tmp3, 32 540 %tmp5 = ashr exact i64 %tmp4, 32 541 %tmp6 = getelementptr inbounds i32, i32 addrspace(1)* %arg0, i64 %tmp5 542 %tmp7 = load i32, i32 addrspace(1)* %tmp6, align 4 543 %tmp8 = sext i32 %tmp7 to i64 544 %tmp9 = getelementptr inbounds i32, i32 addrspace(1)* %arg1, i64 %tmp8 545 %tmp10 = load i32, i32 addrspace(1)* %tmp9, align 4 546 %tmp13 = zext i32 %tmp10 to i64 547 %tmp14 = getelementptr inbounds i32, i32 addrspace(1)* %arg2, i64 %tmp13 548 %tmp16 = load i32, i32 addrspace(1)* %tmp14, align 16 549 switch i32 %tmp16, label %exit1 550 [ i32 1, label %LeafBlock 551 i32 2, label %LeafBlock1 552 i32 3, label %exit0 ] 553 554LeafBlock: ; preds = %entry 555 %SwitchLeaf = icmp eq i32 %tmp16, 1 556 br i1 %SwitchLeaf, label %exit0, label %exit1 557 558LeafBlock1: ; preds = %entry 559 %SwitchLeaf2 = icmp eq i32 %tmp16, 2 560 br i1 %SwitchLeaf2, label %exit0, label %exit1 561 562exit0: ; preds = %LeafBlock, %LeafBlock1 563 store volatile i32 17, i32 addrspace(3)* undef 564 ret void 565 566exit1: ; preds = %LeafBlock, %LeafBlock1 567 store volatile i32 9, i32 addrspace(1)* undef 568 unreachable 569} 570 571; IR-LABEL: @divergent_multi_ret_nest_in_uniform_triangle( 572define amdgpu_kernel void @divergent_multi_ret_nest_in_uniform_triangle(i32 %arg0) #0 { 573entry: 574 %uniform.cond0 = icmp eq i32 %arg0, 4 575 br i1 %uniform.cond0, label %divergent.multi.exit.region, label %uniform.ret 576 577divergent.multi.exit.region: 578 %id.x = tail call i32 @llvm.amdgcn.workitem.id.x() 579 %divergent.cond0 = icmp eq i32 %id.x, 0 580 br i1 %divergent.cond0, label %divergent.ret0, label %divergent.ret1 581 582divergent.ret0: 583 store volatile i32 11, i32 addrspace(3)* undef 584 ret void 585 586divergent.ret1: 587 store volatile i32 42, i32 addrspace(3)* undef 588 ret void 589 590uniform.ret: 591 store volatile i32 9, i32 addrspace(1)* undef 592 ret void 593} 594 595; IR-LABEL: @divergent_complex_multi_ret_nest_in_uniform_triangle( 596define amdgpu_kernel void @divergent_complex_multi_ret_nest_in_uniform_triangle(i32 %arg0) #0 { 597entry: 598 %uniform.cond0 = icmp eq i32 %arg0, 4 599 br i1 %uniform.cond0, label %divergent.multi.exit.region, label %uniform.ret 600 601divergent.multi.exit.region: 602 %id.x = tail call i32 @llvm.amdgcn.workitem.id.x() 603 %divergent.cond0 = icmp eq i32 %id.x, 0 604 br i1 %divergent.cond0, label %divergent.if, label %divergent.ret1 605 606divergent.if: 607 %vgpr0 = load volatile float, float addrspace(1)* undef 608 %divergent.cond1 = fcmp ogt float %vgpr0, 1.0 609 br i1 %divergent.cond1, label %divergent.then, label %divergent.endif 610 611divergent.then: 612 %vgpr1 = load volatile float, float addrspace(1)* undef 613 %divergent.cond2 = fcmp olt float %vgpr1, 4.0 614 store volatile i32 33, i32 addrspace(1)* undef 615 br i1 %divergent.cond2, label %divergent.ret0, label %divergent.endif 616 617divergent.endif: 618 store volatile i32 38, i32 addrspace(1)* undef 619 br label %divergent.ret0 620 621divergent.ret0: 622 store volatile i32 11, i32 addrspace(3)* undef 623 ret void 624 625divergent.ret1: 626 store volatile i32 42, i32 addrspace(3)* undef 627 ret void 628 629uniform.ret: 630 store volatile i32 9, i32 addrspace(1)* undef 631 ret void 632} 633 634; IR-LABEL: @uniform_complex_multi_ret_nest_in_divergent_triangle( 635; IR: Flow1: ; preds = %uniform.ret1, %uniform.multi.exit.region 636; IR: %6 = phi i1 [ false, %uniform.ret1 ], [ true, %uniform.multi.exit.region ] 637; IR: br i1 %6, label %uniform.if, label %Flow2 638 639; IR: Flow: ; preds = %uniform.then, %uniform.if 640; IR: %7 = phi i1 [ %uniform.cond2, %uniform.then ], [ %uniform.cond1.inv, %uniform.if ] 641; IR: br i1 %7, label %uniform.endif, label %uniform.ret0 642 643; IR: UnifiedReturnBlock: ; preds = %Flow3, %Flow2 644; IR-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 %5) 645; IR-NEXT: ret void 646define amdgpu_kernel void @uniform_complex_multi_ret_nest_in_divergent_triangle(i32 %arg0) #0 { 647entry: 648 %id.x = tail call i32 @llvm.amdgcn.workitem.id.x() 649 %divergent.cond0 = icmp eq i32 %id.x, 0 650 br i1 %divergent.cond0, label %uniform.multi.exit.region, label %divergent.ret 651 652uniform.multi.exit.region: 653 %uniform.cond0 = icmp eq i32 %arg0, 4 654 br i1 %uniform.cond0, label %uniform.if, label %uniform.ret1 655 656uniform.if: 657 %sgpr0 = load volatile i32, i32 addrspace(4)* undef 658 %uniform.cond1 = icmp slt i32 %sgpr0, 1 659 br i1 %uniform.cond1, label %uniform.then, label %uniform.endif 660 661uniform.then: 662 %sgpr1 = load volatile i32, i32 addrspace(4)* undef 663 %uniform.cond2 = icmp sge i32 %sgpr1, 4 664 store volatile i32 33, i32 addrspace(1)* undef 665 br i1 %uniform.cond2, label %uniform.ret0, label %uniform.endif 666 667uniform.endif: 668 store volatile i32 38, i32 addrspace(1)* undef 669 br label %uniform.ret0 670 671uniform.ret0: 672 store volatile i32 11, i32 addrspace(3)* undef 673 ret void 674 675uniform.ret1: 676 store volatile i32 42, i32 addrspace(3)* undef 677 ret void 678 679divergent.ret: 680 store volatile i32 9, i32 addrspace(1)* undef 681 ret void 682} 683 684; IR-LABEL: @multi_divergent_unreachable_exit( 685; IR: UnifiedUnreachableBlock: 686; IR-NEXT: call void @llvm.amdgcn.unreachable() 687; IR-NEXT: br label %UnifiedReturnBlock 688 689; IR: UnifiedReturnBlock: 690; IR-NEXT: call void @llvm.amdgcn.end.cf.i64(i64 691; IR-NEXT: ret void 692define amdgpu_kernel void @multi_divergent_unreachable_exit() #0 { 693bb: 694 %tmp = tail call i32 @llvm.amdgcn.workitem.id.x() 695 switch i32 %tmp, label %bb3 [ 696 i32 2, label %bb1 697 i32 0, label %bb2 698 ] 699 700bb1: ; preds = %bb 701 unreachable 702 703bb2: ; preds = %bb 704 unreachable 705 706bb3: ; preds = %bb 707 switch i32 undef, label %bb5 [ 708 i32 2, label %bb4 709 ] 710 711bb4: ; preds = %bb3 712 ret void 713 714bb5: ; preds = %bb3 715 unreachable 716} 717 718; Test that there is an extra export inserted after the normal export, 719; if the normal export is inside a uniformly reached block and there is 720; an infinite loop in the pixel shader. 721 722; IR-LABEL: @uniformly_reached_export 723; IR-NEXT: .entry: 724; IR: br i1 [[CND:%.*]], label %[[LOOP:.*]], label %[[EXP:.*]] 725 726; IR: [[LOOP]]: 727; IR-NEXT: br i1 false, label %DummyReturnBlock, label %[[LOOP]] 728 729; IR: [[EXP]]: 730; IR-NEXT: call void @llvm.amdgcn.exp.compr.v2f16(i32 immarg 0, i32 immarg 15, <2 x half> <half 0xH3C00, half 0xH0000>, <2 x half> <half 0xH0000, half 0xH3C00>, i1 immarg true, i1 immarg true) 731; IR-NEXT: ret void 732 733; IR: DummyReturnBlock: 734; IR-NEXT: ret void 735 736define amdgpu_ps void @uniformly_reached_export(float inreg %tmp25) { 737.entry: 738 %tmp26 = fcmp olt float %tmp25, 0.000000e+00 739 br i1 %tmp26, label %loop, label %bb27 740 741loop: ; preds = %loop, %.entry 742 br label %loop 743 744bb27: ; preds = %.entry 745 call void @llvm.amdgcn.exp.compr.v2f16(i32 immarg 0, i32 immarg 15, <2 x half> <half 0xH3C00, half 0xH0000>, <2 x half> <half 0xH0000, half 0xH3C00>, i1 immarg true, i1 immarg true) 746 ret void 747} 748 749declare void @llvm.amdgcn.exp.compr.v2f16(i32 immarg, i32 immarg, <2 x half>, <2 x half>, i1 immarg, i1 immarg) #0 750declare i32 @llvm.amdgcn.workitem.id.x() #1 751 752attributes #0 = { nounwind } 753attributes #1 = { nounwind readnone } 754