1; RUN: llc -march=amdgcn -mcpu=verde -machine-sink-split-probability-threshold=0 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s 2; RUN: llc -march=amdgcn -mcpu=tonga -machine-sink-split-probability-threshold=0 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s 3 4; GCN-LABEL: {{^}}uniform_if_scc: 5; GCN-DAG: s_cmp_eq_u32 s{{[0-9]+}}, 0 6; GCN-DAG: s_mov_b32 [[S_VAL:s[0-9]+]], 0 7; GCN: s_cbranch_scc1 [[IF_LABEL:[0-9_A-Za-z]+]] 8 9; Fall-through to the else 10; GCN: s_mov_b32 [[S_VAL]], 1 11 12; GCN: [[IF_LABEL]]: 13; GCN: v_mov_b32_e32 [[V_VAL:v[0-9]+]], [[S_VAL]] 14; GCN: buffer_store_dword [[V_VAL]] 15define void @uniform_if_scc(i32 %cond, i32 addrspace(1)* %out) { 16entry: 17 %cmp0 = icmp eq i32 %cond, 0 18 br i1 %cmp0, label %if, label %else 19 20if: 21 br label %done 22 23else: 24 br label %done 25 26done: 27 %value = phi i32 [0, %if], [1, %else] 28 store i32 %value, i32 addrspace(1)* %out 29 ret void 30} 31 32; GCN-LABEL: {{^}}uniform_if_vcc: 33; GCN-DAG: v_cmp_eq_f32_e64 [[COND:vcc|s\[[0-9]+:[0-9]+\]]], s{{[0-9]+}}, 0{{$}} 34; GCN-DAG: s_mov_b32 [[S_VAL:s[0-9]+]], 0 35; GCN: s_cbranch_vccnz [[IF_LABEL:[0-9_A-Za-z]+]] 36 37; Fall-through to the else 38; GCN: s_mov_b32 [[S_VAL]], 1 39 40; GCN: [[IF_LABEL]]: 41; GCN: v_mov_b32_e32 [[V_VAL:v[0-9]+]], [[S_VAL]] 42; GCN: buffer_store_dword [[V_VAL]] 43define void @uniform_if_vcc(float %cond, i32 addrspace(1)* %out) { 44entry: 45 %cmp0 = fcmp oeq float %cond, 0.0 46 br i1 %cmp0, label %if, label %else 47 48if: 49 br label %done 50 51else: 52 br label %done 53 54done: 55 %value = phi i32 [0, %if], [1, %else] 56 store i32 %value, i32 addrspace(1)* %out 57 ret void 58} 59 60; GCN-LABEL: {{^}}uniform_if_swap_br_targets_scc: 61; GCN-DAG: s_cmp_lg_u32 s{{[0-9]+}}, 0 62; GCN-DAG: s_mov_b32 [[S_VAL:s[0-9]+]], 0 63; GCN: s_cbranch_scc1 [[IF_LABEL:[0-9_A-Za-z]+]] 64 65; Fall-through to the else 66; GCN: s_mov_b32 [[S_VAL]], 1 67 68; GCN: [[IF_LABEL]]: 69; GCN: v_mov_b32_e32 [[V_VAL:v[0-9]+]], [[S_VAL]] 70; GCN: buffer_store_dword [[V_VAL]] 71define void @uniform_if_swap_br_targets_scc(i32 %cond, i32 addrspace(1)* %out) { 72entry: 73 %cmp0 = icmp eq i32 %cond, 0 74 br i1 %cmp0, label %else, label %if 75 76if: 77 br label %done 78 79else: 80 br label %done 81 82done: 83 %value = phi i32 [0, %if], [1, %else] 84 store i32 %value, i32 addrspace(1)* %out 85 ret void 86} 87 88; GCN-LABEL: {{^}}uniform_if_swap_br_targets_vcc: 89; GCN-DAG: v_cmp_neq_f32_e64 [[COND:vcc|s\[[0-9]+:[0-9]+\]]], s{{[0-9]+}}, 0{{$}} 90; GCN-DAG: s_mov_b32 [[S_VAL:s[0-9]+]], 0 91; GCN: s_cbranch_vccnz [[IF_LABEL:[0-9_A-Za-z]+]] 92 93; Fall-through to the else 94; GCN: s_mov_b32 [[S_VAL]], 1 95 96; GCN: [[IF_LABEL]]: 97; GCN: v_mov_b32_e32 [[V_VAL:v[0-9]+]], [[S_VAL]] 98; GCN: buffer_store_dword [[V_VAL]] 99define void @uniform_if_swap_br_targets_vcc(float %cond, i32 addrspace(1)* %out) { 100entry: 101 %cmp0 = fcmp oeq float %cond, 0.0 102 br i1 %cmp0, label %else, label %if 103 104if: 105 br label %done 106 107else: 108 br label %done 109 110done: 111 %value = phi i32 [0, %if], [1, %else] 112 store i32 %value, i32 addrspace(1)* %out 113 ret void 114} 115 116; GCN-LABEL: {{^}}uniform_if_move_valu: 117; GCN: v_add_f32_e32 [[CMP:v[0-9]+]] 118; Using a floating-point value in an integer compare will cause the compare to 119; be selected for the SALU and then later moved to the VALU. 120; GCN: v_cmp_ne_u32_e32 [[COND:vcc|s\[[0-9]+:[0-9]+\]]], 5, [[CMP]] 121; GCN: s_and_b64 vcc, exec, [[COND]] 122; GCN: s_cbranch_vccnz [[ENDIF_LABEL:[0-9_A-Za-z]+]] 123; GCN: buffer_store_dword 124; GCN: [[ENDIF_LABEL]]: 125; GCN: s_endpgm 126define void @uniform_if_move_valu(i32 addrspace(1)* %out, float %a) { 127entry: 128 %a.0 = fadd float %a, 10.0 129 %cond = bitcast float %a.0 to i32 130 %cmp = icmp eq i32 %cond, 5 131 br i1 %cmp, label %if, label %endif 132 133if: 134 store i32 0, i32 addrspace(1)* %out 135 br label %endif 136 137endif: 138 ret void 139} 140 141; GCN-LABEL: {{^}}uniform_if_move_valu_commute: 142; GCN: v_add_f32_e32 [[CMP:v[0-9]+]] 143; Using a floating-point value in an integer compare will cause the compare to 144; be selected for the SALU and then later moved to the VALU. 145; GCN: v_cmp_gt_u32_e32 [[COND:vcc|s\[[0-9]+:[0-9]+\]]], 6, [[CMP]] 146; GCN: s_and_b64 vcc, exec, [[COND]] 147; GCN: s_cbranch_vccnz [[ENDIF_LABEL:[0-9_A-Za-z]+]] 148; GCN: buffer_store_dword 149; GCN: [[ENDIF_LABEL]]: 150; GCN: s_endpgm 151define void @uniform_if_move_valu_commute(i32 addrspace(1)* %out, float %a) { 152entry: 153 %a.0 = fadd float %a, 10.0 154 %cond = bitcast float %a.0 to i32 155 %cmp = icmp ugt i32 %cond, 5 156 br i1 %cmp, label %if, label %endif 157 158if: 159 store i32 0, i32 addrspace(1)* %out 160 br label %endif 161 162endif: 163 ret void 164} 165 166 167; GCN-LABEL: {{^}}uniform_if_else_ret: 168; GCN: s_cmp_lg_u32 s{{[0-9]+}}, 0 169; GCN-NEXT: s_cbranch_scc0 [[IF_LABEL:[0-9_A-Za-z]+]] 170 171; GCN: v_mov_b32_e32 [[TWO:v[0-9]+]], 2 172; GCN: buffer_store_dword [[TWO]] 173; GCN: s_endpgm 174 175; GCN: {{^}}[[IF_LABEL]]: 176; GCN: v_mov_b32_e32 [[ONE:v[0-9]+]], 1 177; GCN: buffer_store_dword [[ONE]] 178; GCN: s_endpgm 179define void @uniform_if_else_ret(i32 addrspace(1)* nocapture %out, i32 %a) { 180entry: 181 %cmp = icmp eq i32 %a, 0 182 br i1 %cmp, label %if.then, label %if.else 183 184if.then: ; preds = %entry 185 store i32 1, i32 addrspace(1)* %out 186 br label %if.end 187 188if.else: ; preds = %entry 189 store i32 2, i32 addrspace(1)* %out 190 br label %if.end 191 192if.end: ; preds = %if.else, %if.then 193 ret void 194} 195 196; GCN-LABEL: {{^}}uniform_if_else: 197; GCN: s_cmp_lg_u32 s{{[0-9]+}}, 0 198; GCN-NEXT: s_cbranch_scc0 [[IF_LABEL:[0-9_A-Za-z]+]] 199 200; GCN: v_mov_b32_e32 [[IMM_REG:v[0-9]+]], 2 201; GCN: s_branch [[ENDIF_LABEL:[0-9_A-Za-z]+]] 202 203; GCN: [[IF_LABEL]]: 204; GCN-NEXT: v_mov_b32_e32 [[IMM_REG]], 1 205 206; GCN-NEXT: [[ENDIF_LABEL]]: 207; GCN: buffer_store_dword [[IMM_REG]] 208 209; GCN: v_mov_b32_e32 [[THREE:v[0-9]+]], 3 210; GCN: buffer_store_dword [[THREE]] 211; GCN: s_endpgm 212define void @uniform_if_else(i32 addrspace(1)* nocapture %out0, i32 addrspace(1)* nocapture %out1, i32 %a) { 213entry: 214 %cmp = icmp eq i32 %a, 0 215 br i1 %cmp, label %if.then, label %if.else 216 217if.then: ; preds = %entry 218 store i32 1, i32 addrspace(1)* %out0 219 br label %if.end 220 221if.else: ; preds = %entry 222 store i32 2, i32 addrspace(1)* %out0 223 br label %if.end 224 225if.end: ; preds = %if.else, %if.then 226 store i32 3, i32 addrspace(1)* %out1 227 ret void 228} 229 230; GCN-LABEL: {{^}}icmp_2_users: 231; GCN: s_cmp_lt_i32 s{{[0-9]+}}, 1 232; GCN: s_cbranch_scc1 [[LABEL:[a-zA-Z0-9_]+]] 233; GCN: buffer_store_dword 234; GCN: [[LABEL]]: 235; GCN: s_endpgm 236define void @icmp_2_users(i32 addrspace(1)* %out, i32 %cond) { 237main_body: 238 %0 = icmp sgt i32 %cond, 0 239 %1 = sext i1 %0 to i32 240 br i1 %0, label %IF, label %ENDIF 241 242IF: 243 store i32 %1, i32 addrspace(1)* %out 244 br label %ENDIF 245 246ENDIF: ; preds = %IF, %main_body 247 ret void 248} 249 250; GCN-LABEL: {{^}}icmp_users_different_blocks: 251; GCN: s_load_dword [[COND:s[0-9]+]] 252; GCN: s_cmp_lt_i32 [[COND]], 1 253; GCN: s_cbranch_scc1 [[EXIT:[A-Za-z0-9_]+]] 254; GCN: v_cmp_gt_i32_e64 vcc, [[COND]], 0{{$}} 255; GCN: s_cbranch_vccnz [[EXIT]] 256; GCN: buffer_store 257; GCN: {{^}}[[EXIT]]: 258; GCN: s_endpgm 259define void @icmp_users_different_blocks(i32 %cond0, i32 %cond1, i32 addrspace(1)* %out) { 260bb: 261 %tmp = tail call i32 @llvm.amdgcn.workitem.id.x() #0 262 %cmp0 = icmp sgt i32 %cond0, 0 263 %cmp1 = icmp sgt i32 %cond1, 0 264 br i1 %cmp0, label %bb2, label %bb9 265 266bb2: ; preds = %bb 267 %tmp2 = sext i1 %cmp1 to i32 268 %tmp3 = add i32 %tmp2, %tmp 269 br i1 %cmp1, label %bb9, label %bb7 270 271bb7: ; preds = %bb5 272 store i32 %tmp3, i32 addrspace(1)* %out 273 br label %bb9 274 275bb9: ; preds = %bb8, %bb4 276 ret void 277} 278 279; SI-LABEL: {{^}}uniform_loop: 280; SI: {{^}}[[LOOP_LABEL:[A-Z0-9_a-z]+]]: 281; SI: s_add_i32 [[I:s[0-9]+]], s{{[0-9]+}}, -1 282; SI: s_cmp_lg_u32 [[I]], 0 283; SI: s_cbranch_scc1 [[LOOP_LABEL]] 284; SI: s_endpgm 285define void @uniform_loop(i32 addrspace(1)* %out, i32 %a) { 286entry: 287 br label %loop 288 289loop: 290 %i = phi i32 [0, %entry], [%i.i, %loop] 291 %i.i = add i32 %i, 1 292 %cmp = icmp eq i32 %a, %i.i 293 br i1 %cmp, label %done, label %loop 294 295done: 296 ret void 297} 298 299; Test uniform and divergent. 300 301; GCN-LABEL: {{^}}uniform_inside_divergent: 302; GCN: v_cmp_gt_u32_e32 vcc, 16, v{{[0-9]+}} 303; GCN: s_and_saveexec_b64 [[MASK:s\[[0-9]+:[0-9]+\]]], vcc 304; GCN: s_xor_b64 [[MASK1:s\[[0-9]+:[0-9]+\]]], exec, [[MASK]] 305; GCN: s_cbranch_execz [[ENDIF_LABEL:[0-9_A-Za-z]+]] 306; GCN: s_cmp_lg_u32 {{s[0-9]+}}, 0 307; GCN: s_cbranch_scc1 [[ENDIF_LABEL]] 308; GCN: v_mov_b32_e32 [[ONE:v[0-9]+]], 1 309; GCN: buffer_store_dword [[ONE]] 310define void @uniform_inside_divergent(i32 addrspace(1)* %out, i32 %cond) { 311entry: 312 %tid = call i32 @llvm.amdgcn.workitem.id.x() #0 313 %d_cmp = icmp ult i32 %tid, 16 314 br i1 %d_cmp, label %if, label %endif 315 316if: 317 store i32 0, i32 addrspace(1)* %out 318 %u_cmp = icmp eq i32 %cond, 0 319 br i1 %u_cmp, label %if_uniform, label %endif 320 321if_uniform: 322 store i32 1, i32 addrspace(1)* %out 323 br label %endif 324 325endif: 326 ret void 327} 328 329; GCN-LABEL: {{^}}divergent_inside_uniform: 330; GCN: s_cmp_lg_u32 s{{[0-9]+}}, 0 331; GCN: s_cbranch_scc1 [[ENDIF_LABEL:[0-9_A-Za-z]+]] 332; GCN: v_cmp_gt_u32_e32 vcc, 16, v{{[0-9]+}} 333; GCN: s_and_saveexec_b64 [[MASK:s\[[0-9]+:[0-9]+\]]], vcc 334; GCN: s_xor_b64 [[MASK1:s\[[0-9]+:[0-9]+\]]], exec, [[MASK]] 335; GCN: v_mov_b32_e32 [[ONE:v[0-9]+]], 1 336; GCN: buffer_store_dword [[ONE]] 337; GCN: [[ENDIF_LABEL]]: 338; GCN: s_endpgm 339define void @divergent_inside_uniform(i32 addrspace(1)* %out, i32 %cond) { 340entry: 341 %u_cmp = icmp eq i32 %cond, 0 342 br i1 %u_cmp, label %if, label %endif 343 344if: 345 store i32 0, i32 addrspace(1)* %out 346 %tid = call i32 @llvm.amdgcn.workitem.id.x() #0 347 %d_cmp = icmp ult i32 %tid, 16 348 br i1 %d_cmp, label %if_uniform, label %endif 349 350if_uniform: 351 store i32 1, i32 addrspace(1)* %out 352 br label %endif 353 354endif: 355 ret void 356} 357 358; GCN-LABEL: {{^}}divergent_if_uniform_if: 359; GCN: v_cmp_eq_u32_e32 vcc, 0, v0 360; GCN: s_and_saveexec_b64 [[MASK:s\[[0-9]+:[0-9]+\]]], vcc 361; GCN: s_xor_b64 [[MASK:s\[[0-9]+:[0-9]+\]]], exec, [[MASK]] 362; GCN: v_mov_b32_e32 [[ONE:v[0-9]+]], 1 363; GCN: buffer_store_dword [[ONE]] 364; GCN: s_or_b64 exec, exec, [[MASK]] 365; GCN: s_cmp_lg_u32 s{{[0-9]+}}, 0 366; GCN: s_cbranch_scc1 [[EXIT:[A-Z0-9_]+]] 367; GCN: v_mov_b32_e32 [[TWO:v[0-9]+]], 2 368; GCN: buffer_store_dword [[TWO]] 369; GCN: [[EXIT]]: 370; GCN: s_endpgm 371define void @divergent_if_uniform_if(i32 addrspace(1)* %out, i32 %cond) { 372entry: 373 %tid = call i32 @llvm.amdgcn.workitem.id.x() #0 374 %d_cmp = icmp eq i32 %tid, 0 375 br i1 %d_cmp, label %if, label %endif 376 377if: 378 store i32 1, i32 addrspace(1)* %out 379 br label %endif 380 381endif: 382 %u_cmp = icmp eq i32 %cond, 0 383 br i1 %u_cmp, label %if_uniform, label %exit 384 385if_uniform: 386 store i32 2, i32 addrspace(1)* %out 387 br label %exit 388 389exit: 390 ret void 391} 392 393; The condition of the branches in the two blocks are 394; uniform. MachineCSE replaces the 2nd condition with the inverse of 395; the first, leaving an scc use in a different block than it was 396; defed. 397 398; GCN-LABEL: {{^}}cse_uniform_condition_different_blocks: 399; GCN: s_load_dword [[COND:s[0-9]+]] 400; GCN: s_cmp_lt_i32 [[COND]], 1 401; GCN: s_cbranch_scc1 BB[[FNNUM:[0-9]+]]_3 402 403; GCN: BB#1: 404; GCN-NOT: cmp 405; GCN: buffer_load_dword 406; GCN: buffer_store_dword 407; GCN: s_cbranch_scc1 BB[[FNNUM]]_3 408 409; GCN: BB[[FNNUM]]_3: 410; GCN: s_endpgm 411define void @cse_uniform_condition_different_blocks(i32 %cond, i32 addrspace(1)* %out) { 412bb: 413 %tmp = tail call i32 @llvm.amdgcn.workitem.id.x() #0 414 %tmp1 = icmp sgt i32 %cond, 0 415 br i1 %tmp1, label %bb2, label %bb9 416 417bb2: ; preds = %bb 418 %tmp3 = load volatile i32, i32 addrspace(1)* undef 419 store volatile i32 0, i32 addrspace(1)* undef 420 %tmp9 = icmp sle i32 %cond, 0 421 br i1 %tmp9, label %bb9, label %bb7 422 423bb7: ; preds = %bb5 424 store i32 %tmp3, i32 addrspace(1)* %out 425 br label %bb9 426 427bb9: ; preds = %bb8, %bb4 428 ret void 429} 430 431; GCN-LABEL: {{^}}uniform_if_scc_i64_eq: 432; VI-DAG: s_cmp_eq_u64 s{{\[[0-9]+:[0-9]+\]}}, 0 433; GCN-DAG: s_mov_b32 [[S_VAL:s[0-9]+]], 0 434 435; SI: v_cmp_eq_u64_e64 436; SI: s_cbranch_vccnz [[IF_LABEL:[0-9_A-Za-z]+]] 437 438; VI: s_cbranch_scc1 [[IF_LABEL:[0-9_A-Za-z]+]] 439 440; Fall-through to the else 441; GCN: s_mov_b32 [[S_VAL]], 1 442 443; GCN: [[IF_LABEL]]: 444; GCN: v_mov_b32_e32 [[V_VAL:v[0-9]+]], [[S_VAL]] 445; GCN: buffer_store_dword [[V_VAL]] 446define void @uniform_if_scc_i64_eq(i64 %cond, i32 addrspace(1)* %out) { 447entry: 448 %cmp0 = icmp eq i64 %cond, 0 449 br i1 %cmp0, label %if, label %else 450 451if: 452 br label %done 453 454else: 455 br label %done 456 457done: 458 %value = phi i32 [0, %if], [1, %else] 459 store i32 %value, i32 addrspace(1)* %out 460 ret void 461} 462 463; GCN-LABEL: {{^}}uniform_if_scc_i64_ne: 464; VI-DAG: s_cmp_lg_u64 s{{\[[0-9]+:[0-9]+\]}}, 0 465; GCN-DAG: s_mov_b32 [[S_VAL:s[0-9]+]], 0 466 467; SI: v_cmp_ne_u64_e64 468; SI: s_cbranch_vccnz [[IF_LABEL:[0-9_A-Za-z]+]] 469 470; VI: s_cbranch_scc1 [[IF_LABEL:[0-9_A-Za-z]+]] 471 472; Fall-through to the else 473; GCN: s_mov_b32 [[S_VAL]], 1 474 475; GCN: [[IF_LABEL]]: 476; GCN: v_mov_b32_e32 [[V_VAL:v[0-9]+]], [[S_VAL]] 477; GCN: buffer_store_dword [[V_VAL]] 478define void @uniform_if_scc_i64_ne(i64 %cond, i32 addrspace(1)* %out) { 479entry: 480 %cmp0 = icmp ne i64 %cond, 0 481 br i1 %cmp0, label %if, label %else 482 483if: 484 br label %done 485 486else: 487 br label %done 488 489done: 490 %value = phi i32 [0, %if], [1, %else] 491 store i32 %value, i32 addrspace(1)* %out 492 ret void 493} 494 495; GCN-LABEL: {{^}}uniform_if_scc_i64_sgt: 496; GCN: s_mov_b32 [[S_VAL:s[0-9]+]], 0 497; GCN: v_cmp_gt_i64_e64 498; GCN: s_cbranch_vccnz [[IF_LABEL:[0-9_A-Za-z]+]] 499 500; Fall-through to the else 501; GCN: s_mov_b32 [[S_VAL]], 1 502 503; GCN: [[IF_LABEL]]: 504; GCN: v_mov_b32_e32 [[V_VAL]], [[S_VAL]] 505; GCN: buffer_store_dword [[V_VAL]] 506define void @uniform_if_scc_i64_sgt(i64 %cond, i32 addrspace(1)* %out) { 507entry: 508 %cmp0 = icmp sgt i64 %cond, 0 509 br i1 %cmp0, label %if, label %else 510 511if: 512 br label %done 513 514else: 515 br label %done 516 517done: 518 %value = phi i32 [0, %if], [1, %else] 519 store i32 %value, i32 addrspace(1)* %out 520 ret void 521} 522 523; GCN-LABEL: {{^}}move_to_valu_i64_eq: 524; GCN: v_cmp_eq_u64_e32 525define void @move_to_valu_i64_eq(i32 addrspace(1)* %out) { 526 %cond = load volatile i64, i64 addrspace(3)* undef 527 %cmp0 = icmp eq i64 %cond, 0 528 br i1 %cmp0, label %if, label %else 529 530if: 531 br label %done 532 533else: 534 br label %done 535 536done: 537 %value = phi i32 [0, %if], [1, %else] 538 store i32 %value, i32 addrspace(1)* %out 539 ret void 540} 541 542; GCN-LABEL: {{^}}move_to_valu_i64_ne: 543; GCN: v_cmp_ne_u64_e32 544define void @move_to_valu_i64_ne(i32 addrspace(1)* %out) { 545 %cond = load volatile i64, i64 addrspace(3)* undef 546 %cmp0 = icmp ne i64 %cond, 0 547 br i1 %cmp0, label %if, label %else 548 549if: 550 br label %done 551 552else: 553 br label %done 554 555done: 556 %value = phi i32 [0, %if], [1, %else] 557 store i32 %value, i32 addrspace(1)* %out 558 ret void 559} 560 561declare i32 @llvm.amdgcn.workitem.id.x() #0 562 563attributes #0 = { nounwind readnone } 564