1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
2; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
3; RUN: opt -S -mtriple=amdgcn-- -structurizecfg -si-annotate-control-flow < %s | FileCheck -check-prefix=OPT %s
4; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
5
6; Ensure two if.break calls, for both the inner and outer loops
7; FIXME: duplicate comparison
8define amdgpu_vs void @multi_else_break(<4 x float> %vec, i32 %ub, i32 %cont) {
9; OPT-LABEL: @multi_else_break(
10; OPT-NEXT:  main_body:
11; OPT-NEXT:    br label [[LOOP_OUTER:%.*]]
12; OPT:       LOOP.outer:
13; OPT-NEXT:    [[PHI_BROKEN2:%.*]] = phi i64 [ [[TMP9:%.*]], [[FLOW1:%.*]] ], [ 0, [[MAIN_BODY:%.*]] ]
14; OPT-NEXT:    [[TMP43:%.*]] = phi i32 [ 0, [[MAIN_BODY]] ], [ [[TMP4:%.*]], [[FLOW1]] ]
15; OPT-NEXT:    br label [[LOOP:%.*]]
16; OPT:       LOOP:
17; OPT-NEXT:    [[PHI_BROKEN:%.*]] = phi i64 [ [[TMP7:%.*]], [[FLOW:%.*]] ], [ 0, [[LOOP_OUTER]] ]
18; OPT-NEXT:    [[TMP0:%.*]] = phi i32 [ undef, [[LOOP_OUTER]] ], [ [[TMP4]], [[FLOW]] ]
19; OPT-NEXT:    [[TMP45:%.*]] = phi i32 [ [[TMP43]], [[LOOP_OUTER]] ], [ [[TMP47:%.*]], [[FLOW]] ]
20; OPT-NEXT:    [[TMP47]] = add i32 [[TMP45]], 1
21; OPT-NEXT:    [[TMP48:%.*]] = icmp slt i32 [[TMP45]], [[UB:%.*]]
22; OPT-NEXT:    [[TMP1:%.*]] = call { i1, i64 } @llvm.amdgcn.if.i64(i1 [[TMP48]])
23; OPT-NEXT:    [[TMP2:%.*]] = extractvalue { i1, i64 } [[TMP1]], 0
24; OPT-NEXT:    [[TMP3:%.*]] = extractvalue { i1, i64 } [[TMP1]], 1
25; OPT-NEXT:    br i1 [[TMP2]], label [[ENDIF:%.*]], label [[FLOW]]
26; OPT:       Flow:
27; OPT-NEXT:    [[TMP4]] = phi i32 [ [[TMP47]], [[ENDIF]] ], [ [[TMP0]], [[LOOP]] ]
28; OPT-NEXT:    [[TMP5:%.*]] = phi i1 [ [[TMP51:%.*]], [[ENDIF]] ], [ true, [[LOOP]] ]
29; OPT-NEXT:    [[TMP6:%.*]] = phi i1 [ [[TMP11:%.*]], [[ENDIF]] ], [ true, [[LOOP]] ]
30; OPT-NEXT:    call void @llvm.amdgcn.end.cf.i64(i64 [[TMP3]])
31; OPT-NEXT:    [[TMP7]] = call i64 @llvm.amdgcn.if.break.i64(i1 [[TMP6]], i64 [[PHI_BROKEN]])
32; OPT-NEXT:    [[TMP8:%.*]] = call i1 @llvm.amdgcn.loop.i64(i64 [[TMP7]])
33; OPT-NEXT:    [[TMP9]] = call i64 @llvm.amdgcn.if.break.i64(i1 [[TMP5]], i64 [[PHI_BROKEN2]])
34; OPT-NEXT:    br i1 [[TMP8]], label [[FLOW1]], label [[LOOP]]
35; OPT:       Flow1:
36; OPT-NEXT:    call void @llvm.amdgcn.end.cf.i64(i64 [[TMP7]])
37; OPT-NEXT:    [[TMP10:%.*]] = call i1 @llvm.amdgcn.loop.i64(i64 [[TMP9]])
38; OPT-NEXT:    br i1 [[TMP10]], label [[IF:%.*]], label [[LOOP_OUTER]]
39; OPT:       IF:
40; OPT-NEXT:    call void @llvm.amdgcn.end.cf.i64(i64 [[TMP9]])
41; OPT-NEXT:    ret void
42; OPT:       ENDIF:
43; OPT-NEXT:    [[TMP51]] = icmp eq i32 [[TMP47]], [[CONT:%.*]]
44; OPT-NEXT:    [[TMP11]] = xor i1 [[TMP51]], true
45; OPT-NEXT:    br label [[FLOW]]
46;
47; GCN-LABEL: multi_else_break:
48; GCN:       ; %bb.0: ; %main_body
49; GCN-NEXT:    s_mov_b64 s[2:3], 0
50; GCN-NEXT:    v_mov_b32_e32 v0, 0
51; GCN-NEXT:    s_branch BB0_2
52; GCN-NEXT:  BB0_1: ; %Flow2
53; GCN-NEXT:    ; in Loop: Header=BB0_2 Depth=1
54; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
55; GCN-NEXT:    s_and_b64 s[0:1], exec, s[8:9]
56; GCN-NEXT:    s_or_b64 s[2:3], s[0:1], s[2:3]
57; GCN-NEXT:    s_andn2_b64 exec, exec, s[2:3]
58; GCN-NEXT:    s_cbranch_execz BB0_6
59; GCN-NEXT:  BB0_2: ; %LOOP.outer
60; GCN-NEXT:    ; =>This Loop Header: Depth=1
61; GCN-NEXT:    ; Child Loop BB0_4 Depth 2
62; GCN-NEXT:    ; implicit-def: $sgpr6_sgpr7
63; GCN-NEXT:    ; implicit-def: $sgpr8_sgpr9
64; GCN-NEXT:    s_mov_b64 s[4:5], 0
65; GCN-NEXT:    v_mov_b32_e32 v1, v0
66; GCN-NEXT:    s_branch BB0_4
67; GCN-NEXT:  BB0_3: ; %Flow
68; GCN-NEXT:    ; in Loop: Header=BB0_4 Depth=2
69; GCN-NEXT:    s_or_b64 exec, exec, s[10:11]
70; GCN-NEXT:    s_and_b64 s[0:1], exec, s[6:7]
71; GCN-NEXT:    s_or_b64 s[4:5], s[0:1], s[4:5]
72; GCN-NEXT:    s_andn2_b64 exec, exec, s[4:5]
73; GCN-NEXT:    s_cbranch_execz BB0_1
74; GCN-NEXT:  BB0_4: ; %LOOP
75; GCN-NEXT:    ; Parent Loop BB0_2 Depth=1
76; GCN-NEXT:    ; => This Inner Loop Header: Depth=2
77; GCN-NEXT:    v_mov_b32_e32 v2, v1
78; GCN-NEXT:    v_add_i32_e32 v1, vcc, 1, v2
79; GCN-NEXT:    v_cmp_lt_i32_e32 vcc, v2, v4
80; GCN-NEXT:    s_or_b64 s[8:9], s[8:9], exec
81; GCN-NEXT:    s_or_b64 s[6:7], s[6:7], exec
82; GCN-NEXT:    s_and_saveexec_b64 s[10:11], vcc
83; GCN-NEXT:    s_cbranch_execz BB0_3
84; GCN-NEXT:  ; %bb.5: ; %ENDIF
85; GCN-NEXT:    ; in Loop: Header=BB0_4 Depth=2
86; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v1
87; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], v5, v1
88; GCN-NEXT:    s_andn2_b64 s[8:9], s[8:9], exec
89; GCN-NEXT:    s_andn2_b64 s[6:7], s[6:7], exec
90; GCN-NEXT:    s_and_b64 s[12:13], vcc, exec
91; GCN-NEXT:    s_and_b64 s[0:1], s[0:1], exec
92; GCN-NEXT:    s_or_b64 s[8:9], s[8:9], s[12:13]
93; GCN-NEXT:    s_or_b64 s[6:7], s[6:7], s[0:1]
94; GCN-NEXT:    v_mov_b32_e32 v0, v1
95; GCN-NEXT:    s_branch BB0_3
96; GCN-NEXT:  BB0_6: ; %IF
97; GCN-NEXT:    s_endpgm
98main_body:
99  br label %LOOP.outer
100
101LOOP.outer:                                       ; preds = %ENDIF, %main_body
102  %tmp43 = phi i32 [ 0, %main_body ], [ %tmp47, %ENDIF ]
103  br label %LOOP
104
105LOOP:                                             ; preds = %ENDIF, %LOOP.outer
106  %tmp45 = phi i32 [ %tmp43, %LOOP.outer ], [ %tmp47, %ENDIF ]
107  %tmp47 = add i32 %tmp45, 1
108  %tmp48 = icmp slt i32 %tmp45, %ub
109  br i1 %tmp48, label %ENDIF, label %IF
110
111IF:                                               ; preds = %LOOP
112  ret void
113
114ENDIF:                                            ; preds = %LOOP
115  %tmp51 = icmp eq i32 %tmp47, %cont
116  br i1 %tmp51, label %LOOP, label %LOOP.outer
117}
118
119define amdgpu_kernel void @multi_if_break_loop(i32 %arg) #0 {
120; OPT-LABEL: @multi_if_break_loop(
121; OPT-NEXT:  bb:
122; OPT-NEXT:    [[ID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
123; OPT-NEXT:    [[TMP:%.*]] = sub i32 [[ID]], [[ARG:%.*]]
124; OPT-NEXT:    br label [[BB1:%.*]]
125; OPT:       bb1:
126; OPT-NEXT:    [[PHI_BROKEN:%.*]] = phi i64 [ [[TMP5:%.*]], [[FLOW4:%.*]] ], [ 0, [[BB:%.*]] ]
127; OPT-NEXT:    [[LSR_IV:%.*]] = phi i32 [ undef, [[BB]] ], [ [[LSR_IV_NEXT:%.*]], [[FLOW4]] ]
128; OPT-NEXT:    [[LSR_IV_NEXT]] = add i32 [[LSR_IV]], 1
129; OPT-NEXT:    [[CMP0:%.*]] = icmp slt i32 [[LSR_IV_NEXT]], 0
130; OPT-NEXT:    [[LOAD0:%.*]] = load volatile i32, i32 addrspace(1)* undef, align 4
131; OPT-NEXT:    br label [[NODEBLOCK:%.*]]
132; OPT:       NodeBlock:
133; OPT-NEXT:    [[PIVOT:%.*]] = icmp slt i32 [[LOAD0]], 1
134; OPT-NEXT:    [[TMP0:%.*]] = xor i1 [[PIVOT]], true
135; OPT-NEXT:    br i1 [[TMP0]], label [[LEAFBLOCK1:%.*]], label [[FLOW:%.*]]
136; OPT:       LeafBlock1:
137; OPT-NEXT:    [[SWITCHLEAF2:%.*]] = icmp eq i32 [[LOAD0]], 1
138; OPT-NEXT:    br i1 [[SWITCHLEAF2]], label [[CASE1:%.*]], label [[FLOW3:%.*]]
139; OPT:       Flow3:
140; OPT-NEXT:    [[TMP1:%.*]] = phi i1 [ [[TMP11:%.*]], [[CASE1]] ], [ true, [[LEAFBLOCK1]] ]
141; OPT-NEXT:    [[TMP2:%.*]] = phi i1 [ false, [[CASE1]] ], [ true, [[LEAFBLOCK1]] ]
142; OPT-NEXT:    br label [[FLOW]]
143; OPT:       LeafBlock:
144; OPT-NEXT:    [[SWITCHLEAF:%.*]] = icmp eq i32 [[LOAD0]], 0
145; OPT-NEXT:    br i1 [[SWITCHLEAF]], label [[CASE0:%.*]], label [[FLOW5:%.*]]
146; OPT:       Flow4:
147; OPT-NEXT:    [[TMP3:%.*]] = phi i1 [ [[TMP12:%.*]], [[FLOW5]] ], [ [[TMP8:%.*]], [[FLOW]] ]
148; OPT-NEXT:    [[TMP4:%.*]] = phi i1 [ [[TMP13:%.*]], [[FLOW5]] ], [ [[TMP9:%.*]], [[FLOW]] ]
149; OPT-NEXT:    [[TMP5]] = call i64 @llvm.amdgcn.if.break.i64(i1 [[TMP3]], i64 [[PHI_BROKEN]])
150; OPT-NEXT:    [[TMP6:%.*]] = call i1 @llvm.amdgcn.loop.i64(i64 [[TMP5]])
151; OPT-NEXT:    br i1 [[TMP6]], label [[FLOW6:%.*]], label [[BB1]]
152; OPT:       case0:
153; OPT-NEXT:    [[LOAD1:%.*]] = load volatile i32, i32 addrspace(1)* undef, align 4
154; OPT-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP]], [[LOAD1]]
155; OPT-NEXT:    [[TMP7:%.*]] = xor i1 [[CMP1]], true
156; OPT-NEXT:    br label [[FLOW5]]
157; OPT:       Flow:
158; OPT-NEXT:    [[TMP8]] = phi i1 [ [[TMP1]], [[FLOW3]] ], [ true, [[NODEBLOCK]] ]
159; OPT-NEXT:    [[TMP9]] = phi i1 [ [[TMP2]], [[FLOW3]] ], [ false, [[NODEBLOCK]] ]
160; OPT-NEXT:    [[TMP10:%.*]] = phi i1 [ false, [[FLOW3]] ], [ true, [[NODEBLOCK]] ]
161; OPT-NEXT:    br i1 [[TMP10]], label [[LEAFBLOCK:%.*]], label [[FLOW4]]
162; OPT:       case1:
163; OPT-NEXT:    [[LOAD2:%.*]] = load volatile i32, i32 addrspace(1)* undef, align 4
164; OPT-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP]], [[LOAD2]]
165; OPT-NEXT:    [[TMP11]] = xor i1 [[CMP2]], true
166; OPT-NEXT:    br label [[FLOW3]]
167; OPT:       Flow5:
168; OPT-NEXT:    [[TMP12]] = phi i1 [ [[TMP7]], [[CASE0]] ], [ [[TMP8]], [[LEAFBLOCK]] ]
169; OPT-NEXT:    [[TMP13]] = phi i1 [ false, [[CASE0]] ], [ true, [[LEAFBLOCK]] ]
170; OPT-NEXT:    br label [[FLOW4]]
171; OPT:       Flow6:
172; OPT-NEXT:    call void @llvm.amdgcn.end.cf.i64(i64 [[TMP5]])
173; OPT-NEXT:    [[TMP14:%.*]] = call { i1, i64 } @llvm.amdgcn.if.i64(i1 [[TMP4]])
174; OPT-NEXT:    [[TMP15:%.*]] = extractvalue { i1, i64 } [[TMP14]], 0
175; OPT-NEXT:    [[TMP16:%.*]] = extractvalue { i1, i64 } [[TMP14]], 1
176; OPT-NEXT:    br i1 [[TMP15]], label [[NEWDEFAULT:%.*]], label [[BB9:%.*]]
177; OPT:       NewDefault:
178; OPT-NEXT:    br label [[BB9]]
179; OPT:       bb9:
180; OPT-NEXT:    call void @llvm.amdgcn.end.cf.i64(i64 [[TMP16]])
181; OPT-NEXT:    ret void
182;
183; GCN-LABEL: multi_if_break_loop:
184; GCN:       ; %bb.0: ; %bb
185; GCN-NEXT:    s_load_dword s2, s[0:1], 0x9
186; GCN-NEXT:    s_mov_b64 s[0:1], 0
187; GCN-NEXT:    s_mov_b32 s3, 0xf000
188; GCN-NEXT:    s_waitcnt lgkmcnt(0)
189; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s2, v0
190; GCN-NEXT:    s_mov_b32 s2, -1
191; GCN-NEXT:    ; implicit-def: $sgpr4_sgpr5
192; GCN-NEXT:    s_branch BB1_2
193; GCN-NEXT:  BB1_1: ; %Flow4
194; GCN-NEXT:    ; in Loop: Header=BB1_2 Depth=1
195; GCN-NEXT:    s_and_b64 s[6:7], exec, s[6:7]
196; GCN-NEXT:    s_or_b64 s[0:1], s[6:7], s[0:1]
197; GCN-NEXT:    s_andn2_b64 s[4:5], s[4:5], exec
198; GCN-NEXT:    s_and_b64 s[6:7], s[8:9], exec
199; GCN-NEXT:    s_or_b64 s[4:5], s[4:5], s[6:7]
200; GCN-NEXT:    s_andn2_b64 exec, exec, s[0:1]
201; GCN-NEXT:    s_cbranch_execz BB1_9
202; GCN-NEXT:  BB1_2: ; %bb1
203; GCN-NEXT:    ; =>This Inner Loop Header: Depth=1
204; GCN-NEXT:    buffer_load_dword v1, off, s[0:3], 0
205; GCN-NEXT:    s_waitcnt vmcnt(0)
206; GCN-NEXT:    v_cmp_gt_i32_e32 vcc, 1, v1
207; GCN-NEXT:    s_and_b64 vcc, exec, vcc
208; GCN-NEXT:    s_cbranch_vccnz BB1_6
209; GCN-NEXT:  ; %bb.3: ; %LeafBlock1
210; GCN-NEXT:    ; in Loop: Header=BB1_2 Depth=1
211; GCN-NEXT:    s_mov_b64 s[6:7], -1
212; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v1
213; GCN-NEXT:    s_and_b64 vcc, exec, vcc
214; GCN-NEXT:    s_mov_b64 s[8:9], -1
215; GCN-NEXT:    s_cbranch_vccz BB1_5
216; GCN-NEXT:  ; %bb.4: ; %case1
217; GCN-NEXT:    ; in Loop: Header=BB1_2 Depth=1
218; GCN-NEXT:    buffer_load_dword v2, off, s[0:3], 0
219; GCN-NEXT:    s_waitcnt vmcnt(0)
220; GCN-NEXT:    v_cmp_ge_i32_e32 vcc, v0, v2
221; GCN-NEXT:    s_mov_b64 s[8:9], 0
222; GCN-NEXT:    s_orn2_b64 s[6:7], vcc, exec
223; GCN-NEXT:  BB1_5: ; %Flow3
224; GCN-NEXT:    ; in Loop: Header=BB1_2 Depth=1
225; GCN-NEXT:    s_mov_b64 s[10:11], 0
226; GCN-NEXT:    s_and_b64 vcc, exec, s[10:11]
227; GCN-NEXT:    s_cbranch_vccz BB1_1
228; GCN-NEXT:    s_branch BB1_7
229; GCN-NEXT:  BB1_6: ; in Loop: Header=BB1_2 Depth=1
230; GCN-NEXT:    s_mov_b64 s[8:9], 0
231; GCN-NEXT:    s_mov_b64 s[6:7], -1
232; GCN-NEXT:    s_and_b64 vcc, exec, -1
233; GCN-NEXT:    s_cbranch_execz BB1_1
234; GCN-NEXT:  BB1_7: ; %LeafBlock
235; GCN-NEXT:    ; in Loop: Header=BB1_2 Depth=1
236; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
237; GCN-NEXT:    s_and_b64 vcc, exec, vcc
238; GCN-NEXT:    s_mov_b64 s[8:9], -1
239; GCN-NEXT:    s_cbranch_vccz BB1_1
240; GCN-NEXT:  ; %bb.8: ; %case0
241; GCN-NEXT:    ; in Loop: Header=BB1_2 Depth=1
242; GCN-NEXT:    buffer_load_dword v1, off, s[0:3], 0
243; GCN-NEXT:    s_mov_b64 s[8:9], 0
244; GCN-NEXT:    s_waitcnt vmcnt(0)
245; GCN-NEXT:    v_cmp_ge_i32_e32 vcc, v0, v1
246; GCN-NEXT:    s_andn2_b64 s[6:7], s[6:7], exec
247; GCN-NEXT:    s_and_b64 s[10:11], vcc, exec
248; GCN-NEXT:    s_or_b64 s[6:7], s[6:7], s[10:11]
249; GCN-NEXT:    s_branch BB1_1
250; GCN-NEXT:  BB1_9: ; %Flow6
251; GCN-NEXT:    s_or_b64 exec, exec, s[0:1]
252; GCN-NEXT:    s_and_saveexec_b64 s[0:1], s[4:5]
253; GCN-NEXT:    s_endpgm
254bb:
255  %id = call i32 @llvm.amdgcn.workitem.id.x()
256  %tmp = sub i32 %id, %arg
257  br label %bb1
258
259bb1:
260  %lsr.iv = phi i32 [ undef, %bb ], [ %lsr.iv.next, %case0 ], [ %lsr.iv.next, %case1 ]
261  %lsr.iv.next = add i32 %lsr.iv, 1
262  %cmp0 = icmp slt i32 %lsr.iv.next, 0
263  %load0 = load volatile i32, i32 addrspace(1)* undef, align 4
264  switch i32 %load0, label %bb9 [
265  i32 0, label %case0
266  i32 1, label %case1
267  ]
268
269case0:
270  %load1 = load volatile i32, i32 addrspace(1)* undef, align 4
271  %cmp1 = icmp slt i32 %tmp, %load1
272  br i1 %cmp1, label %bb1, label %bb9
273
274case1:
275  %load2 = load volatile i32, i32 addrspace(1)* undef, align 4
276  %cmp2 = icmp slt i32 %tmp, %load2
277  br i1 %cmp2, label %bb1, label %bb9
278
279bb9:
280  ret void
281}
282
283declare i32 @llvm.amdgcn.workitem.id.x() #1
284
285attributes #0 = { nounwind }
286attributes #1 = { nounwind readnone }
287