1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GFX9 %s
3; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GFX10 %s
4; RUN: llc -march=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GFX11 %s
5
6define amdgpu_kernel void @udiv32_invariant_denom(i32 addrspace(1)* nocapture %arg, i32 %arg1) {
7; GFX9-LABEL: udiv32_invariant_denom:
8; GFX9:       ; %bb.0: ; %bb
9; GFX9-NEXT:    s_load_dword s4, s[0:1], 0x2c
10; GFX9-NEXT:    s_mov_b64 s[2:3], 0
11; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
12; GFX9-NEXT:    v_mov_b32_e32 v1, 0
13; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
14; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s4
15; GFX9-NEXT:    s_sub_i32 s5, 0, s4
16; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
17; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
18; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
19; GFX9-NEXT:  .LBB0_1: ; %bb3
20; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
21; GFX9-NEXT:    v_readfirstlane_b32 s6, v0
22; GFX9-NEXT:    s_mul_i32 s7, s5, s6
23; GFX9-NEXT:    s_mul_hi_u32 s7, s6, s7
24; GFX9-NEXT:    s_add_i32 s6, s6, s7
25; GFX9-NEXT:    s_mul_i32 s7, s3, s6
26; GFX9-NEXT:    s_mul_hi_u32 s6, s2, s6
27; GFX9-NEXT:    s_add_i32 s6, s6, s7
28; GFX9-NEXT:    s_mul_i32 s7, s5, s6
29; GFX9-NEXT:    s_add_i32 s7, s2, s7
30; GFX9-NEXT:    s_cmp_ge_u32 s7, s4
31; GFX9-NEXT:    v_mov_b32_e32 v2, s6
32; GFX9-NEXT:    v_mov_b32_e32 v3, s7
33; GFX9-NEXT:    s_cselect_b64 vcc, -1, 0
34; GFX9-NEXT:    s_add_i32 s7, s6, 1
35; GFX9-NEXT:    s_not_b32 s6, s6
36; GFX9-NEXT:    s_mul_i32 s6, s4, s6
37; GFX9-NEXT:    v_mov_b32_e32 v4, s7
38; GFX9-NEXT:    s_add_i32 s6, s2, s6
39; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
40; GFX9-NEXT:    v_mov_b32_e32 v4, s6
41; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
42; GFX9-NEXT:    v_add_u32_e32 v5, 1, v2
43; GFX9-NEXT:    s_add_u32 s2, s2, 1
44; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, s4, v3
45; GFX9-NEXT:    s_addc_u32 s3, s3, 0
46; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
47; GFX9-NEXT:    global_store_dword v1, v2, s[0:1]
48; GFX9-NEXT:    s_add_u32 s0, s0, 4
49; GFX9-NEXT:    s_addc_u32 s1, s1, 0
50; GFX9-NEXT:    s_cmpk_eq_i32 s2, 0x400
51; GFX9-NEXT:    s_cbranch_scc0 .LBB0_1
52; GFX9-NEXT:  ; %bb.2: ; %bb2
53; GFX9-NEXT:    s_endpgm
54;
55; GFX10-LABEL: udiv32_invariant_denom:
56; GFX10:       ; %bb.0: ; %bb
57; GFX10-NEXT:    s_load_dword s4, s[0:1], 0x2c
58; GFX10-NEXT:    v_mov_b32_e32 v1, 0
59; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
60; GFX10-NEXT:    s_mov_b64 s[2:3], 0
61; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
62; GFX10-NEXT:    v_cvt_f32_u32_e32 v0, s4
63; GFX10-NEXT:    s_sub_i32 s5, 0, s4
64; GFX10-NEXT:    v_rcp_iflag_f32_e32 v0, v0
65; GFX10-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
66; GFX10-NEXT:    v_cvt_u32_f32_e32 v0, v0
67; GFX10-NEXT:  .LBB0_1: ; %bb3
68; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
69; GFX10-NEXT:    v_readfirstlane_b32 s6, v0
70; GFX10-NEXT:    s_mul_i32 s7, s5, s6
71; GFX10-NEXT:    s_mul_hi_u32 s7, s6, s7
72; GFX10-NEXT:    s_add_i32 s6, s6, s7
73; GFX10-NEXT:    s_mul_i32 s7, s3, s6
74; GFX10-NEXT:    s_mul_hi_u32 s6, s2, s6
75; GFX10-NEXT:    s_add_i32 s6, s6, s7
76; GFX10-NEXT:    s_mul_i32 s7, s5, s6
77; GFX10-NEXT:    s_add_i32 s7, s2, s7
78; GFX10-NEXT:    s_cmp_ge_u32 s7, s4
79; GFX10-NEXT:    s_cselect_b32 vcc_lo, -1, 0
80; GFX10-NEXT:    s_add_i32 s8, s6, 1
81; GFX10-NEXT:    s_not_b32 s9, s6
82; GFX10-NEXT:    v_mov_b32_e32 v2, s8
83; GFX10-NEXT:    s_mul_i32 s8, s4, s9
84; GFX10-NEXT:    s_add_i32 s8, s2, s8
85; GFX10-NEXT:    s_add_u32 s2, s2, 1
86; GFX10-NEXT:    v_mov_b32_e32 v3, s8
87; GFX10-NEXT:    v_cndmask_b32_e32 v2, s6, v2, vcc_lo
88; GFX10-NEXT:    s_addc_u32 s3, s3, 0
89; GFX10-NEXT:    v_cndmask_b32_e32 v3, s7, v3, vcc_lo
90; GFX10-NEXT:    v_add_nc_u32_e32 v4, 1, v2
91; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, s4, v3
92; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc_lo
93; GFX10-NEXT:    global_store_dword v1, v2, s[0:1]
94; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
95; GFX10-NEXT:    s_add_u32 s0, s0, 4
96; GFX10-NEXT:    s_addc_u32 s1, s1, 0
97; GFX10-NEXT:    s_cmpk_eq_i32 s2, 0x400
98; GFX10-NEXT:    s_cbranch_scc0 .LBB0_1
99; GFX10-NEXT:  ; %bb.2: ; %bb2
100; GFX10-NEXT:    s_endpgm
101;
102; GFX11-LABEL: udiv32_invariant_denom:
103; GFX11:       ; %bb.0: ; %bb
104; GFX11-NEXT:    s_clause 0x1
105; GFX11-NEXT:    s_load_b32 s4, s[0:1], 0x2c
106; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
107; GFX11-NEXT:    s_mov_b64 s[2:3], 0
108; GFX11-NEXT:    v_mov_b32_e32 v1, 0
109; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
110; GFX11-NEXT:    v_cvt_f32_u32_e32 v0, s4
111; GFX11-NEXT:    s_sub_i32 s5, 0, s4
112; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
113; GFX11-NEXT:    v_rcp_iflag_f32_e32 v0, v0
114; GFX11-NEXT:    s_waitcnt_depctr 0xfff
115; GFX11-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
116; GFX11-NEXT:    v_cvt_u32_f32_e32 v0, v0
117; GFX11-NEXT:    .p2align 6
118; GFX11-NEXT:  .LBB0_1: ; %bb3
119; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
120; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
121; GFX11-NEXT:    v_readfirstlane_b32 s6, v0
122; GFX11-NEXT:    s_mul_i32 s7, s5, s6
123; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
124; GFX11-NEXT:    s_mul_hi_u32 s7, s6, s7
125; GFX11-NEXT:    s_add_i32 s6, s6, s7
126; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
127; GFX11-NEXT:    s_mul_i32 s7, s3, s6
128; GFX11-NEXT:    s_mul_hi_u32 s6, s2, s6
129; GFX11-NEXT:    s_add_i32 s6, s6, s7
130; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
131; GFX11-NEXT:    s_mul_i32 s7, s5, s6
132; GFX11-NEXT:    s_add_i32 s7, s2, s7
133; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
134; GFX11-NEXT:    s_cmp_ge_u32 s7, s4
135; GFX11-NEXT:    s_cselect_b32 vcc_lo, -1, 0
136; GFX11-NEXT:    s_add_i32 s8, s6, 1
137; GFX11-NEXT:    s_not_b32 s9, s6
138; GFX11-NEXT:    v_mov_b32_e32 v2, s8
139; GFX11-NEXT:    s_mul_i32 s8, s4, s9
140; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
141; GFX11-NEXT:    s_add_i32 s8, s2, s8
142; GFX11-NEXT:    s_add_u32 s2, s2, 1
143; GFX11-NEXT:    v_mov_b32_e32 v3, s8
144; GFX11-NEXT:    v_cndmask_b32_e32 v2, s6, v2, vcc_lo
145; GFX11-NEXT:    s_addc_u32 s3, s3, 0
146; GFX11-NEXT:    v_dual_cndmask_b32 v3, s7, v3 :: v_dual_add_nc_u32 v4, 1, v2
147; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
148; GFX11-NEXT:    v_cmp_le_u32_e32 vcc_lo, s4, v3
149; GFX11-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc_lo
150; GFX11-NEXT:    global_store_b32 v1, v2, s[0:1]
151; GFX11-NEXT:    s_add_u32 s0, s0, 4
152; GFX11-NEXT:    s_addc_u32 s1, s1, 0
153; GFX11-NEXT:    s_cmpk_eq_i32 s2, 0x400
154; GFX11-NEXT:    s_cbranch_scc0 .LBB0_1
155; GFX11-NEXT:  ; %bb.2: ; %bb2
156; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
157; GFX11-NEXT:    s_endpgm
158bb:
159  br label %bb3
160
161bb2:                                              ; preds = %bb3
162  ret void
163
164bb3:                                              ; preds = %bb3, %bb
165  %tmp = phi i32 [ 0, %bb ], [ %tmp7, %bb3 ]
166  %tmp4 = udiv i32 %tmp, %arg1
167  %tmp5 = zext i32 %tmp to i64
168  %tmp6 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 %tmp5
169  store i32 %tmp4, i32 addrspace(1)* %tmp6, align 4
170  %tmp7 = add nuw nsw i32 %tmp, 1
171  %tmp8 = icmp eq i32 %tmp7, 1024
172  br i1 %tmp8, label %bb2, label %bb3
173}
174
175define amdgpu_kernel void @urem32_invariant_denom(i32 addrspace(1)* nocapture %arg, i32 %arg1) {
176; GFX9-LABEL: urem32_invariant_denom:
177; GFX9:       ; %bb.0: ; %bb
178; GFX9-NEXT:    s_load_dword s4, s[0:1], 0x2c
179; GFX9-NEXT:    s_mov_b64 s[2:3], 0
180; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
181; GFX9-NEXT:    v_mov_b32_e32 v1, 0
182; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
183; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s4
184; GFX9-NEXT:    s_sub_i32 s5, 0, s4
185; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
186; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
187; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
188; GFX9-NEXT:  .LBB1_1: ; %bb3
189; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
190; GFX9-NEXT:    v_readfirstlane_b32 s6, v0
191; GFX9-NEXT:    s_mul_i32 s7, s5, s6
192; GFX9-NEXT:    s_mul_hi_u32 s7, s6, s7
193; GFX9-NEXT:    s_add_i32 s6, s6, s7
194; GFX9-NEXT:    s_mul_i32 s7, s3, s6
195; GFX9-NEXT:    s_mul_hi_u32 s6, s2, s6
196; GFX9-NEXT:    s_add_i32 s6, s6, s7
197; GFX9-NEXT:    s_mul_i32 s7, s5, s6
198; GFX9-NEXT:    s_not_b32 s6, s6
199; GFX9-NEXT:    s_mul_i32 s6, s4, s6
200; GFX9-NEXT:    s_add_i32 s7, s2, s7
201; GFX9-NEXT:    s_add_i32 s6, s2, s6
202; GFX9-NEXT:    s_cmp_ge_u32 s7, s4
203; GFX9-NEXT:    s_cselect_b32 s6, s6, s7
204; GFX9-NEXT:    s_sub_i32 s7, s6, s4
205; GFX9-NEXT:    s_cmp_ge_u32 s6, s4
206; GFX9-NEXT:    s_cselect_b32 s6, s7, s6
207; GFX9-NEXT:    s_add_u32 s2, s2, 1
208; GFX9-NEXT:    v_mov_b32_e32 v2, s6
209; GFX9-NEXT:    s_addc_u32 s3, s3, 0
210; GFX9-NEXT:    global_store_dword v1, v2, s[0:1]
211; GFX9-NEXT:    s_add_u32 s0, s0, 4
212; GFX9-NEXT:    s_addc_u32 s1, s1, 0
213; GFX9-NEXT:    s_cmpk_eq_i32 s2, 0x400
214; GFX9-NEXT:    s_cbranch_scc0 .LBB1_1
215; GFX9-NEXT:  ; %bb.2: ; %bb2
216; GFX9-NEXT:    s_endpgm
217;
218; GFX10-LABEL: urem32_invariant_denom:
219; GFX10:       ; %bb.0: ; %bb
220; GFX10-NEXT:    s_load_dword s4, s[0:1], 0x2c
221; GFX10-NEXT:    v_mov_b32_e32 v1, 0
222; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
223; GFX10-NEXT:    s_mov_b64 s[2:3], 0
224; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
225; GFX10-NEXT:    v_cvt_f32_u32_e32 v0, s4
226; GFX10-NEXT:    s_sub_i32 s5, 0, s4
227; GFX10-NEXT:    v_rcp_iflag_f32_e32 v0, v0
228; GFX10-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
229; GFX10-NEXT:    v_cvt_u32_f32_e32 v0, v0
230; GFX10-NEXT:  .LBB1_1: ; %bb3
231; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
232; GFX10-NEXT:    v_readfirstlane_b32 s6, v0
233; GFX10-NEXT:    s_mul_i32 s7, s5, s6
234; GFX10-NEXT:    s_mul_hi_u32 s7, s6, s7
235; GFX10-NEXT:    s_add_i32 s6, s6, s7
236; GFX10-NEXT:    s_mul_i32 s7, s3, s6
237; GFX10-NEXT:    s_mul_hi_u32 s6, s2, s6
238; GFX10-NEXT:    s_add_i32 s6, s6, s7
239; GFX10-NEXT:    s_not_b32 s7, s6
240; GFX10-NEXT:    s_mul_i32 s6, s5, s6
241; GFX10-NEXT:    s_mul_i32 s7, s4, s7
242; GFX10-NEXT:    s_add_i32 s6, s2, s6
243; GFX10-NEXT:    s_add_i32 s7, s2, s7
244; GFX10-NEXT:    s_cmp_ge_u32 s6, s4
245; GFX10-NEXT:    s_cselect_b32 s6, s7, s6
246; GFX10-NEXT:    s_sub_i32 s7, s6, s4
247; GFX10-NEXT:    s_cmp_ge_u32 s6, s4
248; GFX10-NEXT:    s_cselect_b32 s6, s7, s6
249; GFX10-NEXT:    s_add_u32 s2, s2, 1
250; GFX10-NEXT:    v_mov_b32_e32 v2, s6
251; GFX10-NEXT:    s_addc_u32 s3, s3, 0
252; GFX10-NEXT:    global_store_dword v1, v2, s[0:1]
253; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
254; GFX10-NEXT:    s_add_u32 s0, s0, 4
255; GFX10-NEXT:    s_addc_u32 s1, s1, 0
256; GFX10-NEXT:    s_cmpk_eq_i32 s2, 0x400
257; GFX10-NEXT:    s_cbranch_scc0 .LBB1_1
258; GFX10-NEXT:  ; %bb.2: ; %bb2
259; GFX10-NEXT:    s_endpgm
260;
261; GFX11-LABEL: urem32_invariant_denom:
262; GFX11:       ; %bb.0: ; %bb
263; GFX11-NEXT:    s_clause 0x1
264; GFX11-NEXT:    s_load_b32 s4, s[0:1], 0x2c
265; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
266; GFX11-NEXT:    s_mov_b64 s[2:3], 0
267; GFX11-NEXT:    v_mov_b32_e32 v1, 0
268; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
269; GFX11-NEXT:    v_cvt_f32_u32_e32 v0, s4
270; GFX11-NEXT:    s_sub_i32 s5, 0, s4
271; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
272; GFX11-NEXT:    v_rcp_iflag_f32_e32 v0, v0
273; GFX11-NEXT:    s_waitcnt_depctr 0xfff
274; GFX11-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
275; GFX11-NEXT:    v_cvt_u32_f32_e32 v0, v0
276; GFX11-NEXT:    .p2align 6
277; GFX11-NEXT:  .LBB1_1: ; %bb3
278; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
279; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
280; GFX11-NEXT:    v_readfirstlane_b32 s6, v0
281; GFX11-NEXT:    s_mul_i32 s7, s5, s6
282; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
283; GFX11-NEXT:    s_mul_hi_u32 s7, s6, s7
284; GFX11-NEXT:    s_add_i32 s6, s6, s7
285; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
286; GFX11-NEXT:    s_mul_i32 s7, s3, s6
287; GFX11-NEXT:    s_mul_hi_u32 s6, s2, s6
288; GFX11-NEXT:    s_add_i32 s6, s6, s7
289; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
290; GFX11-NEXT:    s_not_b32 s7, s6
291; GFX11-NEXT:    s_mul_i32 s6, s5, s6
292; GFX11-NEXT:    s_mul_i32 s7, s4, s7
293; GFX11-NEXT:    s_add_i32 s6, s2, s6
294; GFX11-NEXT:    s_add_i32 s7, s2, s7
295; GFX11-NEXT:    s_cmp_ge_u32 s6, s4
296; GFX11-NEXT:    s_cselect_b32 s6, s7, s6
297; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
298; GFX11-NEXT:    s_sub_i32 s7, s6, s4
299; GFX11-NEXT:    s_cmp_ge_u32 s6, s4
300; GFX11-NEXT:    s_cselect_b32 s6, s7, s6
301; GFX11-NEXT:    s_add_u32 s2, s2, 1
302; GFX11-NEXT:    v_mov_b32_e32 v2, s6
303; GFX11-NEXT:    s_addc_u32 s3, s3, 0
304; GFX11-NEXT:    global_store_b32 v1, v2, s[0:1]
305; GFX11-NEXT:    s_add_u32 s0, s0, 4
306; GFX11-NEXT:    s_addc_u32 s1, s1, 0
307; GFX11-NEXT:    s_cmpk_eq_i32 s2, 0x400
308; GFX11-NEXT:    s_cbranch_scc0 .LBB1_1
309; GFX11-NEXT:  ; %bb.2: ; %bb2
310; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
311; GFX11-NEXT:    s_endpgm
312bb:
313  br label %bb3
314
315bb2:                                              ; preds = %bb3
316  ret void
317
318bb3:                                              ; preds = %bb3, %bb
319  %tmp = phi i32 [ 0, %bb ], [ %tmp7, %bb3 ]
320  %tmp4 = urem i32 %tmp, %arg1
321  %tmp5 = zext i32 %tmp to i64
322  %tmp6 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 %tmp5
323  store i32 %tmp4, i32 addrspace(1)* %tmp6, align 4
324  %tmp7 = add nuw nsw i32 %tmp, 1
325  %tmp8 = icmp eq i32 %tmp7, 1024
326  br i1 %tmp8, label %bb2, label %bb3
327}
328
329define amdgpu_kernel void @sdiv32_invariant_denom(i32 addrspace(1)* nocapture %arg, i32 %arg1) {
330; GFX9-LABEL: sdiv32_invariant_denom:
331; GFX9:       ; %bb.0: ; %bb
332; GFX9-NEXT:    s_load_dword s3, s[0:1], 0x2c
333; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
334; GFX9-NEXT:    s_ashr_i32 s2, s3, 31
335; GFX9-NEXT:    s_add_i32 s3, s3, s2
336; GFX9-NEXT:    s_xor_b32 s3, s3, s2
337; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s3
338; GFX9-NEXT:    s_sub_i32 s4, 0, s3
339; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
340; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
341; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
342; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
343; GFX9-NEXT:    v_mul_lo_u32 v1, s4, v0
344; GFX9-NEXT:    s_mov_b32 s4, 0
345; GFX9-NEXT:    v_mul_hi_u32 v1, v0, v1
346; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
347; GFX9-NEXT:    v_mov_b32_e32 v1, 0
348; GFX9-NEXT:  .LBB2_1: ; %bb3
349; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
350; GFX9-NEXT:    v_mul_hi_u32 v2, s4, v0
351; GFX9-NEXT:    v_mul_lo_u32 v3, v2, s3
352; GFX9-NEXT:    v_add_u32_e32 v4, 1, v2
353; GFX9-NEXT:    v_sub_u32_e32 v3, s4, v3
354; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, s3, v3
355; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
356; GFX9-NEXT:    v_subrev_u32_e32 v4, s3, v3
357; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
358; GFX9-NEXT:    v_add_u32_e32 v4, 1, v2
359; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, s3, v3
360; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
361; GFX9-NEXT:    v_xor_b32_e32 v2, s2, v2
362; GFX9-NEXT:    s_add_i32 s4, s4, 1
363; GFX9-NEXT:    v_subrev_u32_e32 v2, s2, v2
364; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
365; GFX9-NEXT:    global_store_dword v1, v2, s[0:1]
366; GFX9-NEXT:    s_add_u32 s0, s0, 4
367; GFX9-NEXT:    s_addc_u32 s1, s1, 0
368; GFX9-NEXT:    s_cmpk_eq_i32 s4, 0x400
369; GFX9-NEXT:    s_cbranch_scc0 .LBB2_1
370; GFX9-NEXT:  ; %bb.2: ; %bb2
371; GFX9-NEXT:    s_endpgm
372;
373; GFX10-LABEL: sdiv32_invariant_denom:
374; GFX10:       ; %bb.0: ; %bb
375; GFX10-NEXT:    s_load_dword s3, s[0:1], 0x2c
376; GFX10-NEXT:    v_mov_b32_e32 v1, 0
377; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
378; GFX10-NEXT:    s_mov_b32 s4, 0
379; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
380; GFX10-NEXT:    s_ashr_i32 s2, s3, 31
381; GFX10-NEXT:    s_add_i32 s3, s3, s2
382; GFX10-NEXT:    s_xor_b32 s3, s3, s2
383; GFX10-NEXT:    v_cvt_f32_u32_e32 v0, s3
384; GFX10-NEXT:    s_sub_i32 s5, 0, s3
385; GFX10-NEXT:    v_rcp_iflag_f32_e32 v0, v0
386; GFX10-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
387; GFX10-NEXT:    v_cvt_u32_f32_e32 v0, v0
388; GFX10-NEXT:  .LBB2_1: ; %bb3
389; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
390; GFX10-NEXT:    v_readfirstlane_b32 s6, v0
391; GFX10-NEXT:    s_mul_i32 s7, s5, s6
392; GFX10-NEXT:    s_mul_hi_u32 s7, s6, s7
393; GFX10-NEXT:    s_add_i32 s6, s6, s7
394; GFX10-NEXT:    s_mul_hi_u32 s6, s4, s6
395; GFX10-NEXT:    s_mul_i32 s7, s6, s3
396; GFX10-NEXT:    s_sub_i32 s7, s4, s7
397; GFX10-NEXT:    s_cmp_ge_u32 s7, s3
398; GFX10-NEXT:    s_cselect_b32 vcc_lo, -1, 0
399; GFX10-NEXT:    s_add_i32 s8, s6, 1
400; GFX10-NEXT:    s_add_i32 s4, s4, 1
401; GFX10-NEXT:    v_mov_b32_e32 v2, s8
402; GFX10-NEXT:    s_sub_i32 s8, s7, s3
403; GFX10-NEXT:    v_mov_b32_e32 v3, s8
404; GFX10-NEXT:    v_cndmask_b32_e32 v2, s6, v2, vcc_lo
405; GFX10-NEXT:    v_cndmask_b32_e32 v3, s7, v3, vcc_lo
406; GFX10-NEXT:    v_add_nc_u32_e32 v4, 1, v2
407; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, s3, v3
408; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc_lo
409; GFX10-NEXT:    v_xor_b32_e32 v2, s2, v2
410; GFX10-NEXT:    v_subrev_nc_u32_e32 v2, s2, v2
411; GFX10-NEXT:    global_store_dword v1, v2, s[0:1]
412; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
413; GFX10-NEXT:    s_add_u32 s0, s0, 4
414; GFX10-NEXT:    s_addc_u32 s1, s1, 0
415; GFX10-NEXT:    s_cmpk_eq_i32 s4, 0x400
416; GFX10-NEXT:    s_cbranch_scc0 .LBB2_1
417; GFX10-NEXT:  ; %bb.2: ; %bb2
418; GFX10-NEXT:    s_endpgm
419;
420; GFX11-LABEL: sdiv32_invariant_denom:
421; GFX11:       ; %bb.0: ; %bb
422; GFX11-NEXT:    s_clause 0x1
423; GFX11-NEXT:    s_load_b32 s3, s[0:1], 0x2c
424; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
425; GFX11-NEXT:    s_mov_b32 s4, 0
426; GFX11-NEXT:    v_mov_b32_e32 v1, 0
427; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
428; GFX11-NEXT:    s_ashr_i32 s2, s3, 31
429; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
430; GFX11-NEXT:    s_add_i32 s3, s3, s2
431; GFX11-NEXT:    s_xor_b32 s3, s3, s2
432; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
433; GFX11-NEXT:    v_cvt_f32_u32_e32 v0, s3
434; GFX11-NEXT:    s_sub_i32 s5, 0, s3
435; GFX11-NEXT:    v_rcp_iflag_f32_e32 v0, v0
436; GFX11-NEXT:    s_waitcnt_depctr 0xfff
437; GFX11-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
438; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
439; GFX11-NEXT:    v_cvt_u32_f32_e32 v0, v0
440; GFX11-NEXT:    .p2align 6
441; GFX11-NEXT:  .LBB2_1: ; %bb3
442; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
443; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
444; GFX11-NEXT:    v_readfirstlane_b32 s6, v0
445; GFX11-NEXT:    s_mul_i32 s7, s5, s6
446; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
447; GFX11-NEXT:    s_mul_hi_u32 s7, s6, s7
448; GFX11-NEXT:    s_add_i32 s6, s6, s7
449; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
450; GFX11-NEXT:    s_mul_hi_u32 s6, s4, s6
451; GFX11-NEXT:    s_mul_i32 s7, s6, s3
452; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
453; GFX11-NEXT:    s_sub_i32 s7, s4, s7
454; GFX11-NEXT:    s_cmp_ge_u32 s7, s3
455; GFX11-NEXT:    s_cselect_b32 vcc_lo, -1, 0
456; GFX11-NEXT:    s_add_i32 s8, s6, 1
457; GFX11-NEXT:    s_add_i32 s4, s4, 1
458; GFX11-NEXT:    v_mov_b32_e32 v2, s8
459; GFX11-NEXT:    s_sub_i32 s8, s7, s3
460; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
461; GFX11-NEXT:    v_mov_b32_e32 v3, s8
462; GFX11-NEXT:    v_cndmask_b32_e32 v2, s6, v2, vcc_lo
463; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
464; GFX11-NEXT:    v_dual_cndmask_b32 v3, s7, v3 :: v_dual_add_nc_u32 v4, 1, v2
465; GFX11-NEXT:    v_cmp_le_u32_e32 vcc_lo, s3, v3
466; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
467; GFX11-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc_lo
468; GFX11-NEXT:    v_xor_b32_e32 v2, s2, v2
469; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
470; GFX11-NEXT:    v_subrev_nc_u32_e32 v2, s2, v2
471; GFX11-NEXT:    global_store_b32 v1, v2, s[0:1]
472; GFX11-NEXT:    s_add_u32 s0, s0, 4
473; GFX11-NEXT:    s_addc_u32 s1, s1, 0
474; GFX11-NEXT:    s_cmpk_eq_i32 s4, 0x400
475; GFX11-NEXT:    s_cbranch_scc0 .LBB2_1
476; GFX11-NEXT:  ; %bb.2: ; %bb2
477; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
478; GFX11-NEXT:    s_endpgm
479bb:
480  br label %bb3
481
482bb2:                                              ; preds = %bb3
483  ret void
484
485bb3:                                              ; preds = %bb3, %bb
486  %tmp = phi i32 [ 0, %bb ], [ %tmp7, %bb3 ]
487  %tmp4 = sdiv i32 %tmp, %arg1
488  %tmp5 = zext i32 %tmp to i64
489  %tmp6 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 %tmp5
490  store i32 %tmp4, i32 addrspace(1)* %tmp6, align 4
491  %tmp7 = add nuw nsw i32 %tmp, 1
492  %tmp8 = icmp eq i32 %tmp7, 1024
493  br i1 %tmp8, label %bb2, label %bb3
494}
495
496define amdgpu_kernel void @srem32_invariant_denom(i32 addrspace(1)* nocapture %arg, i32 %arg1) {
497; GFX9-LABEL: srem32_invariant_denom:
498; GFX9:       ; %bb.0: ; %bb
499; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x2c
500; GFX9-NEXT:    v_mov_b32_e32 v1, 0
501; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
502; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
503; GFX9-NEXT:    s_ashr_i32 s3, s2, 31
504; GFX9-NEXT:    s_add_i32 s2, s2, s3
505; GFX9-NEXT:    s_xor_b32 s2, s2, s3
506; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s2
507; GFX9-NEXT:    s_mov_b32 s3, 0
508; GFX9-NEXT:    s_sub_i32 s4, 0, s2
509; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
510; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
511; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
512; GFX9-NEXT:  .LBB3_1: ; %bb3
513; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
514; GFX9-NEXT:    v_readfirstlane_b32 s5, v0
515; GFX9-NEXT:    s_mul_i32 s6, s4, s5
516; GFX9-NEXT:    s_mul_hi_u32 s6, s5, s6
517; GFX9-NEXT:    s_add_i32 s5, s5, s6
518; GFX9-NEXT:    s_mul_hi_u32 s5, s3, s5
519; GFX9-NEXT:    s_mul_i32 s5, s5, s2
520; GFX9-NEXT:    s_sub_i32 s5, s3, s5
521; GFX9-NEXT:    s_sub_i32 s6, s5, s2
522; GFX9-NEXT:    s_cmp_ge_u32 s5, s2
523; GFX9-NEXT:    s_cselect_b32 s5, s6, s5
524; GFX9-NEXT:    s_sub_i32 s6, s5, s2
525; GFX9-NEXT:    s_cmp_ge_u32 s5, s2
526; GFX9-NEXT:    s_cselect_b32 s5, s6, s5
527; GFX9-NEXT:    s_add_i32 s3, s3, 1
528; GFX9-NEXT:    v_mov_b32_e32 v2, s5
529; GFX9-NEXT:    global_store_dword v1, v2, s[0:1]
530; GFX9-NEXT:    s_add_u32 s0, s0, 4
531; GFX9-NEXT:    s_addc_u32 s1, s1, 0
532; GFX9-NEXT:    s_cmpk_eq_i32 s3, 0x400
533; GFX9-NEXT:    s_cbranch_scc0 .LBB3_1
534; GFX9-NEXT:  ; %bb.2: ; %bb2
535; GFX9-NEXT:    s_endpgm
536;
537; GFX10-LABEL: srem32_invariant_denom:
538; GFX10:       ; %bb.0: ; %bb
539; GFX10-NEXT:    s_load_dword s2, s[0:1], 0x2c
540; GFX10-NEXT:    v_mov_b32_e32 v1, 0
541; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
542; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
543; GFX10-NEXT:    s_ashr_i32 s3, s2, 31
544; GFX10-NEXT:    s_add_i32 s2, s2, s3
545; GFX10-NEXT:    s_xor_b32 s2, s2, s3
546; GFX10-NEXT:    s_mov_b32 s3, 0
547; GFX10-NEXT:    v_cvt_f32_u32_e32 v0, s2
548; GFX10-NEXT:    s_sub_i32 s4, 0, s2
549; GFX10-NEXT:    v_rcp_iflag_f32_e32 v0, v0
550; GFX10-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
551; GFX10-NEXT:    v_cvt_u32_f32_e32 v0, v0
552; GFX10-NEXT:  .LBB3_1: ; %bb3
553; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
554; GFX10-NEXT:    v_readfirstlane_b32 s5, v0
555; GFX10-NEXT:    s_mul_i32 s6, s4, s5
556; GFX10-NEXT:    s_mul_hi_u32 s6, s5, s6
557; GFX10-NEXT:    s_add_i32 s5, s5, s6
558; GFX10-NEXT:    s_mul_hi_u32 s5, s3, s5
559; GFX10-NEXT:    s_mul_i32 s5, s5, s2
560; GFX10-NEXT:    s_sub_i32 s5, s3, s5
561; GFX10-NEXT:    s_sub_i32 s6, s5, s2
562; GFX10-NEXT:    s_cmp_ge_u32 s5, s2
563; GFX10-NEXT:    s_cselect_b32 s5, s6, s5
564; GFX10-NEXT:    s_sub_i32 s6, s5, s2
565; GFX10-NEXT:    s_cmp_ge_u32 s5, s2
566; GFX10-NEXT:    s_cselect_b32 s5, s6, s5
567; GFX10-NEXT:    s_add_i32 s3, s3, 1
568; GFX10-NEXT:    v_mov_b32_e32 v2, s5
569; GFX10-NEXT:    global_store_dword v1, v2, s[0:1]
570; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
571; GFX10-NEXT:    s_add_u32 s0, s0, 4
572; GFX10-NEXT:    s_addc_u32 s1, s1, 0
573; GFX10-NEXT:    s_cmpk_eq_i32 s3, 0x400
574; GFX10-NEXT:    s_cbranch_scc0 .LBB3_1
575; GFX10-NEXT:  ; %bb.2: ; %bb2
576; GFX10-NEXT:    s_endpgm
577;
578; GFX11-LABEL: srem32_invariant_denom:
579; GFX11:       ; %bb.0: ; %bb
580; GFX11-NEXT:    s_clause 0x1
581; GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x2c
582; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
583; GFX11-NEXT:    v_mov_b32_e32 v1, 0
584; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
585; GFX11-NEXT:    s_ashr_i32 s3, s2, 31
586; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
587; GFX11-NEXT:    s_add_i32 s2, s2, s3
588; GFX11-NEXT:    s_xor_b32 s2, s2, s3
589; GFX11-NEXT:    s_mov_b32 s3, 0
590; GFX11-NEXT:    v_cvt_f32_u32_e32 v0, s2
591; GFX11-NEXT:    s_sub_i32 s4, 0, s2
592; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
593; GFX11-NEXT:    v_rcp_iflag_f32_e32 v0, v0
594; GFX11-NEXT:    s_waitcnt_depctr 0xfff
595; GFX11-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
596; GFX11-NEXT:    v_cvt_u32_f32_e32 v0, v0
597; GFX11-NEXT:    .p2align 6
598; GFX11-NEXT:  .LBB3_1: ; %bb3
599; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
600; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
601; GFX11-NEXT:    v_readfirstlane_b32 s5, v0
602; GFX11-NEXT:    s_mul_i32 s6, s4, s5
603; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
604; GFX11-NEXT:    s_mul_hi_u32 s6, s5, s6
605; GFX11-NEXT:    s_add_i32 s5, s5, s6
606; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
607; GFX11-NEXT:    s_mul_hi_u32 s5, s3, s5
608; GFX11-NEXT:    s_mul_i32 s5, s5, s2
609; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
610; GFX11-NEXT:    s_sub_i32 s5, s3, s5
611; GFX11-NEXT:    s_sub_i32 s6, s5, s2
612; GFX11-NEXT:    s_cmp_ge_u32 s5, s2
613; GFX11-NEXT:    s_cselect_b32 s5, s6, s5
614; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
615; GFX11-NEXT:    s_sub_i32 s6, s5, s2
616; GFX11-NEXT:    s_cmp_ge_u32 s5, s2
617; GFX11-NEXT:    s_cselect_b32 s5, s6, s5
618; GFX11-NEXT:    s_add_i32 s3, s3, 1
619; GFX11-NEXT:    v_mov_b32_e32 v2, s5
620; GFX11-NEXT:    global_store_b32 v1, v2, s[0:1]
621; GFX11-NEXT:    s_add_u32 s0, s0, 4
622; GFX11-NEXT:    s_addc_u32 s1, s1, 0
623; GFX11-NEXT:    s_cmpk_eq_i32 s3, 0x400
624; GFX11-NEXT:    s_cbranch_scc0 .LBB3_1
625; GFX11-NEXT:  ; %bb.2: ; %bb2
626; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
627; GFX11-NEXT:    s_endpgm
628bb:
629  br label %bb3
630
631bb2:                                              ; preds = %bb3
632  ret void
633
634bb3:                                              ; preds = %bb3, %bb
635  %tmp = phi i32 [ 0, %bb ], [ %tmp7, %bb3 ]
636  %tmp4 = srem i32 %tmp, %arg1
637  %tmp5 = zext i32 %tmp to i64
638  %tmp6 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 %tmp5
639  store i32 %tmp4, i32 addrspace(1)* %tmp6, align 4
640  %tmp7 = add nuw nsw i32 %tmp, 1
641  %tmp8 = icmp eq i32 %tmp7, 1024
642  br i1 %tmp8, label %bb2, label %bb3
643}
644
645define amdgpu_kernel void @udiv16_invariant_denom(i16 addrspace(1)* nocapture %arg, i16 %arg1) {
646; GFX9-LABEL: udiv16_invariant_denom:
647; GFX9:       ; %bb.0: ; %bb
648; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x2c
649; GFX9-NEXT:    s_mov_b32 s5, 0
650; GFX9-NEXT:    v_mov_b32_e32 v2, 0
651; GFX9-NEXT:    s_movk_i32 s6, 0x400
652; GFX9-NEXT:    s_mov_b32 s7, 0
653; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
654; GFX9-NEXT:    s_and_b32 s2, 0xffff, s2
655; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s2
656; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
657; GFX9-NEXT:    v_rcp_iflag_f32_e32 v1, v0
658; GFX9-NEXT:  .LBB4_1: ; %bb3
659; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
660; GFX9-NEXT:    s_and_b32 s4, 0xffff, s7
661; GFX9-NEXT:    v_cvt_f32_u32_e32 v4, s4
662; GFX9-NEXT:    v_add_u16_e64 v3, s7, 1
663; GFX9-NEXT:    v_readfirstlane_b32 s7, v3
664; GFX9-NEXT:    v_cmp_eq_u16_e32 vcc, s6, v3
665; GFX9-NEXT:    v_mul_f32_e32 v3, v4, v1
666; GFX9-NEXT:    v_trunc_f32_e32 v3, v3
667; GFX9-NEXT:    v_cvt_u32_f32_e32 v5, v3
668; GFX9-NEXT:    s_lshl_b64 s[0:1], s[4:5], 1
669; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
670; GFX9-NEXT:    s_add_u32 s8, s2, s0
671; GFX9-NEXT:    v_mad_f32 v3, -v3, v0, v4
672; GFX9-NEXT:    s_addc_u32 s9, s3, s1
673; GFX9-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v3|, v0
674; GFX9-NEXT:    v_addc_co_u32_e64 v3, s[0:1], 0, v5, s[0:1]
675; GFX9-NEXT:    global_store_short v2, v3, s[8:9]
676; GFX9-NEXT:    s_cbranch_vccz .LBB4_1
677; GFX9-NEXT:  ; %bb.2: ; %bb2
678; GFX9-NEXT:    s_endpgm
679;
680; GFX10-LABEL: udiv16_invariant_denom:
681; GFX10:       ; %bb.0: ; %bb
682; GFX10-NEXT:    s_clause 0x1
683; GFX10-NEXT:    s_load_dword s4, s[0:1], 0x2c
684; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
685; GFX10-NEXT:    v_mov_b32_e32 v2, 0
686; GFX10-NEXT:    s_mov_b32 s1, 0
687; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
688; GFX10-NEXT:    s_and_b32 s0, 0xffff, s4
689; GFX10-NEXT:    s_mov_b32 s4, 0
690; GFX10-NEXT:    v_cvt_f32_u32_e32 v0, s0
691; GFX10-NEXT:    v_rcp_iflag_f32_e32 v1, v0
692; GFX10-NEXT:  .LBB4_1: ; %bb3
693; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
694; GFX10-NEXT:    s_and_b32 s0, 0xffff, s4
695; GFX10-NEXT:    v_add_nc_u16 v3, s4, 1
696; GFX10-NEXT:    v_cvt_f32_u32_e32 v4, s0
697; GFX10-NEXT:    s_lshl_b64 s[4:5], s[0:1], 1
698; GFX10-NEXT:    s_add_u32 s6, s2, s4
699; GFX10-NEXT:    v_readfirstlane_b32 s4, v3
700; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x400, v3
701; GFX10-NEXT:    v_mul_f32_e32 v3, v4, v1
702; GFX10-NEXT:    s_addc_u32 s7, s3, s5
703; GFX10-NEXT:    s_and_b32 vcc_lo, exec_lo, vcc_lo
704; GFX10-NEXT:    v_trunc_f32_e32 v3, v3
705; GFX10-NEXT:    v_mad_f32 v4, -v3, v0, v4
706; GFX10-NEXT:    v_cvt_u32_f32_e32 v3, v3
707; GFX10-NEXT:    v_cmp_ge_f32_e64 s0, |v4|, v0
708; GFX10-NEXT:    v_add_co_ci_u32_e64 v3, s0, 0, v3, s0
709; GFX10-NEXT:    global_store_short v2, v3, s[6:7]
710; GFX10-NEXT:    s_cbranch_vccz .LBB4_1
711; GFX10-NEXT:  ; %bb.2: ; %bb2
712; GFX10-NEXT:    s_endpgm
713;
714; GFX11-LABEL: udiv16_invariant_denom:
715; GFX11:       ; %bb.0: ; %bb
716; GFX11-NEXT:    s_clause 0x1
717; GFX11-NEXT:    s_load_b32 s4, s[0:1], 0x2c
718; GFX11-NEXT:    s_load_b64 s[2:3], s[0:1], 0x24
719; GFX11-NEXT:    v_mov_b32_e32 v2, 0
720; GFX11-NEXT:    s_mov_b32 s1, 0
721; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
722; GFX11-NEXT:    s_and_b32 s0, 0xffff, s4
723; GFX11-NEXT:    s_mov_b32 s4, 0
724; GFX11-NEXT:    v_cvt_f32_u32_e32 v0, s0
725; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
726; GFX11-NEXT:    v_rcp_iflag_f32_e32 v1, v0
727; GFX11-NEXT:    .p2align 6
728; GFX11-NEXT:  .LBB4_1: ; %bb3
729; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
730; GFX11-NEXT:    s_and_b32 s0, 0xffff, s4
731; GFX11-NEXT:    v_add_nc_u16 v3, s4, 1
732; GFX11-NEXT:    v_cvt_f32_u32_e32 v4, s0
733; GFX11-NEXT:    s_lshl_b64 s[4:5], s[0:1], 1
734; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
735; GFX11-NEXT:    s_add_u32 s6, s2, s4
736; GFX11-NEXT:    v_readfirstlane_b32 s4, v3
737; GFX11-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x400, v3
738; GFX11-NEXT:    s_waitcnt_depctr 0xfff
739; GFX11-NEXT:    v_mul_f32_e32 v3, v4, v1
740; GFX11-NEXT:    s_addc_u32 s7, s3, s5
741; GFX11-NEXT:    s_and_b32 vcc_lo, exec_lo, vcc_lo
742; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
743; GFX11-NEXT:    v_trunc_f32_e32 v3, v3
744; GFX11-NEXT:    v_fma_f32 v4, -v3, v0, v4
745; GFX11-NEXT:    v_cvt_u32_f32_e32 v3, v3
746; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
747; GFX11-NEXT:    v_cmp_ge_f32_e64 s0, |v4|, v0
748; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, s0, 0, v3, s0
749; GFX11-NEXT:    global_store_b16 v2, v3, s[6:7]
750; GFX11-NEXT:    s_cbranch_vccz .LBB4_1
751; GFX11-NEXT:  ; %bb.2: ; %bb2
752; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
753; GFX11-NEXT:    s_endpgm
754bb:
755  br label %bb3
756
757bb2:                                              ; preds = %bb3
758  ret void
759
760bb3:                                              ; preds = %bb3, %bb
761  %tmp = phi i16 [ 0, %bb ], [ %tmp7, %bb3 ]
762  %tmp4 = udiv i16 %tmp, %arg1
763  %tmp5 = zext i16 %tmp to i64
764  %tmp6 = getelementptr inbounds i16, i16 addrspace(1)* %arg, i64 %tmp5
765  store i16 %tmp4, i16 addrspace(1)* %tmp6, align 2
766  %tmp7 = add nuw nsw i16 %tmp, 1
767  %tmp8 = icmp eq i16 %tmp7, 1024
768  br i1 %tmp8, label %bb2, label %bb3
769}
770
771define amdgpu_kernel void @urem16_invariant_denom(i16 addrspace(1)* nocapture %arg, i16 %arg1) {
772; GFX9-LABEL: urem16_invariant_denom:
773; GFX9:       ; %bb.0: ; %bb
774; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x2c
775; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
776; GFX9-NEXT:    v_mov_b32_e32 v1, 0
777; GFX9-NEXT:    s_movk_i32 s7, 0x400
778; GFX9-NEXT:    v_mov_b32_e32 v4, 0
779; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
780; GFX9-NEXT:    s_and_b32 s6, 0xffff, s2
781; GFX9-NEXT:    v_cvt_f32_u32_e32 v2, s6
782; GFX9-NEXT:    v_rcp_iflag_f32_e32 v3, v2
783; GFX9-NEXT:  .LBB5_1: ; %bb3
784; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
785; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v4
786; GFX9-NEXT:    v_cvt_f32_u32_e32 v8, v0
787; GFX9-NEXT:    v_lshlrev_b64 v[5:6], 1, v[0:1]
788; GFX9-NEXT:    v_add_u16_e32 v4, 1, v4
789; GFX9-NEXT:    v_mov_b32_e32 v7, s5
790; GFX9-NEXT:    v_mul_f32_e32 v9, v8, v3
791; GFX9-NEXT:    v_trunc_f32_e32 v9, v9
792; GFX9-NEXT:    v_cvt_u32_f32_e32 v10, v9
793; GFX9-NEXT:    v_mad_f32 v8, -v9, v2, v8
794; GFX9-NEXT:    v_cmp_ge_f32_e64 s[2:3], |v8|, v2
795; GFX9-NEXT:    v_cmp_eq_u16_e32 vcc, s7, v4
796; GFX9-NEXT:    v_addc_co_u32_e64 v8, s[2:3], 0, v10, s[2:3]
797; GFX9-NEXT:    v_mul_lo_u32 v8, v8, s6
798; GFX9-NEXT:    v_add_co_u32_e64 v5, s[0:1], s4, v5
799; GFX9-NEXT:    v_addc_co_u32_e64 v6, s[0:1], v7, v6, s[0:1]
800; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v8
801; GFX9-NEXT:    global_store_short v[5:6], v0, off
802; GFX9-NEXT:    s_cbranch_vccz .LBB5_1
803; GFX9-NEXT:  ; %bb.2: ; %bb2
804; GFX9-NEXT:    s_endpgm
805;
806; GFX10-LABEL: urem16_invariant_denom:
807; GFX10:       ; %bb.0: ; %bb
808; GFX10-NEXT:    s_clause 0x1
809; GFX10-NEXT:    s_load_dword s4, s[0:1], 0x2c
810; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
811; GFX10-NEXT:    v_mov_b32_e32 v1, 0
812; GFX10-NEXT:    v_mov_b32_e32 v4, 0
813; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
814; GFX10-NEXT:    s_and_b32 s1, 0xffff, s4
815; GFX10-NEXT:    v_cvt_f32_u32_e32 v2, s1
816; GFX10-NEXT:    v_rcp_iflag_f32_e32 v3, v2
817; GFX10-NEXT:  .LBB5_1: ; %bb3
818; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
819; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v4
820; GFX10-NEXT:    v_add_nc_u16 v4, v4, 1
821; GFX10-NEXT:    v_cvt_f32_u32_e32 v7, v0
822; GFX10-NEXT:    v_lshlrev_b64 v[5:6], 1, v[0:1]
823; GFX10-NEXT:    v_mul_f32_e32 v8, v7, v3
824; GFX10-NEXT:    v_add_co_u32 v5, s0, s2, v5
825; GFX10-NEXT:    v_add_co_ci_u32_e64 v6, s0, s3, v6, s0
826; GFX10-NEXT:    v_trunc_f32_e32 v8, v8
827; GFX10-NEXT:    v_mad_f32 v7, -v8, v2, v7
828; GFX10-NEXT:    v_cvt_u32_f32_e32 v8, v8
829; GFX10-NEXT:    v_cmp_ge_f32_e64 vcc_lo, |v7|, v2
830; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v8, vcc_lo
831; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x400, v4
832; GFX10-NEXT:    v_mul_lo_u32 v7, v7, s1
833; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v0, v7
834; GFX10-NEXT:    global_store_short v[5:6], v0, off
835; GFX10-NEXT:    s_cbranch_vccz .LBB5_1
836; GFX10-NEXT:  ; %bb.2: ; %bb2
837; GFX10-NEXT:    s_endpgm
838;
839; GFX11-LABEL: urem16_invariant_denom:
840; GFX11:       ; %bb.0: ; %bb
841; GFX11-NEXT:    s_clause 0x1
842; GFX11-NEXT:    s_load_b32 s4, s[0:1], 0x2c
843; GFX11-NEXT:    s_load_b64 s[2:3], s[0:1], 0x24
844; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v4, 0
845; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
846; GFX11-NEXT:    s_and_b32 s1, 0xffff, s4
847; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
848; GFX11-NEXT:    v_cvt_f32_u32_e32 v2, s1
849; GFX11-NEXT:    v_rcp_iflag_f32_e32 v3, v2
850; GFX11-NEXT:    s_set_inst_prefetch_distance 0x1
851; GFX11-NEXT:    .p2align 6
852; GFX11-NEXT:  .LBB5_1: ; %bb3
853; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
854; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v4
855; GFX11-NEXT:    v_add_nc_u16 v4, v4, 1
856; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_1)
857; GFX11-NEXT:    v_cvt_f32_u32_e32 v7, v0
858; GFX11-NEXT:    v_lshlrev_b64 v[5:6], 1, v[0:1]
859; GFX11-NEXT:    s_waitcnt_depctr 0xfff
860; GFX11-NEXT:    v_mul_f32_e32 v8, v7, v3
861; GFX11-NEXT:    v_add_co_u32 v5, s0, s2, v5
862; GFX11-NEXT:    v_add_co_ci_u32_e64 v6, s0, s3, v6, s0
863; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
864; GFX11-NEXT:    v_trunc_f32_e32 v8, v8
865; GFX11-NEXT:    v_fma_f32 v7, -v8, v2, v7
866; GFX11-NEXT:    v_cvt_u32_f32_e32 v8, v8
867; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
868; GFX11-NEXT:    v_cmp_ge_f32_e64 vcc_lo, |v7|, v2
869; GFX11-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v8, vcc_lo
870; GFX11-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x400, v4
871; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
872; GFX11-NEXT:    v_mul_lo_u32 v7, v7, s1
873; GFX11-NEXT:    v_sub_nc_u32_e32 v0, v0, v7
874; GFX11-NEXT:    global_store_b16 v[5:6], v0, off
875; GFX11-NEXT:    s_cbranch_vccz .LBB5_1
876; GFX11-NEXT:  ; %bb.2: ; %bb2
877; GFX11-NEXT:    s_set_inst_prefetch_distance 0x2
878; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
879; GFX11-NEXT:    s_endpgm
880bb:
881  br label %bb3
882
883bb2:                                              ; preds = %bb3
884  ret void
885
886bb3:                                              ; preds = %bb3, %bb
887  %tmp = phi i16 [ 0, %bb ], [ %tmp7, %bb3 ]
888  %tmp4 = urem i16 %tmp, %arg1
889  %tmp5 = zext i16 %tmp to i64
890  %tmp6 = getelementptr inbounds i16, i16 addrspace(1)* %arg, i64 %tmp5
891  store i16 %tmp4, i16 addrspace(1)* %tmp6, align 2
892  %tmp7 = add nuw nsw i16 %tmp, 1
893  %tmp8 = icmp eq i16 %tmp7, 1024
894  br i1 %tmp8, label %bb2, label %bb3
895}
896
897define amdgpu_kernel void @sdiv16_invariant_denom(i16 addrspace(1)* nocapture %arg, i16 %arg1) {
898; GFX9-LABEL: sdiv16_invariant_denom:
899; GFX9:       ; %bb.0: ; %bb
900; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x2c
901; GFX9-NEXT:    s_mov_b32 s3, 0
902; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
903; GFX9-NEXT:    v_mov_b32_e32 v2, 0
904; GFX9-NEXT:    s_movk_i32 s5, 0x400
905; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
906; GFX9-NEXT:    s_sext_i32_i16 s4, s2
907; GFX9-NEXT:    v_cvt_f32_i32_e32 v0, s4
908; GFX9-NEXT:    s_mov_b32 s6, 0
909; GFX9-NEXT:    v_rcp_iflag_f32_e32 v1, v0
910; GFX9-NEXT:  .LBB6_1: ; %bb3
911; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
912; GFX9-NEXT:    s_sext_i32_i16 s2, s6
913; GFX9-NEXT:    v_cvt_f32_i32_e32 v4, s2
914; GFX9-NEXT:    s_xor_b32 s7, s2, s4
915; GFX9-NEXT:    s_ashr_i32 s2, s7, 30
916; GFX9-NEXT:    s_or_b32 s2, s2, 1
917; GFX9-NEXT:    v_mul_f32_e32 v5, v4, v1
918; GFX9-NEXT:    v_trunc_f32_e32 v5, v5
919; GFX9-NEXT:    v_mad_f32 v4, -v5, v0, v4
920; GFX9-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v4|, |v0|
921; GFX9-NEXT:    v_cvt_i32_f32_e32 v5, v5
922; GFX9-NEXT:    s_and_b64 s[8:9], s[8:9], exec
923; GFX9-NEXT:    s_cselect_b32 s7, s2, 0
924; GFX9-NEXT:    s_and_b32 s2, s6, 0xffff
925; GFX9-NEXT:    v_add_u16_e64 v3, s6, 1
926; GFX9-NEXT:    s_lshl_b64 s[8:9], s[2:3], 1
927; GFX9-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v3
928; GFX9-NEXT:    s_add_u32 s8, s0, s8
929; GFX9-NEXT:    v_readfirstlane_b32 s6, v3
930; GFX9-NEXT:    v_add_u32_e32 v3, s7, v5
931; GFX9-NEXT:    s_addc_u32 s9, s1, s9
932; GFX9-NEXT:    global_store_short v2, v3, s[8:9]
933; GFX9-NEXT:    s_cbranch_vccz .LBB6_1
934; GFX9-NEXT:  ; %bb.2: ; %bb2
935; GFX9-NEXT:    s_endpgm
936;
937; GFX10-LABEL: sdiv16_invariant_denom:
938; GFX10:       ; %bb.0: ; %bb
939; GFX10-NEXT:    s_clause 0x1
940; GFX10-NEXT:    s_load_dword s4, s[0:1], 0x2c
941; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
942; GFX10-NEXT:    v_mov_b32_e32 v2, 0
943; GFX10-NEXT:    s_mov_b32 s1, 0
944; GFX10-NEXT:    s_mov_b32 s5, 0
945; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
946; GFX10-NEXT:    s_sext_i32_i16 s4, s4
947; GFX10-NEXT:    v_cvt_f32_i32_e32 v0, s4
948; GFX10-NEXT:    v_rcp_iflag_f32_e32 v1, v0
949; GFX10-NEXT:  .LBB6_1: ; %bb3
950; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
951; GFX10-NEXT:    s_sext_i32_i16 s0, s5
952; GFX10-NEXT:    v_add_nc_u16 v3, s5, 1
953; GFX10-NEXT:    v_cvt_f32_i32_e32 v4, s0
954; GFX10-NEXT:    s_xor_b32 s0, s0, s4
955; GFX10-NEXT:    s_ashr_i32 s0, s0, 30
956; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x400, v3
957; GFX10-NEXT:    v_mul_f32_e32 v5, v4, v1
958; GFX10-NEXT:    s_or_b32 s0, s0, 1
959; GFX10-NEXT:    v_trunc_f32_e32 v5, v5
960; GFX10-NEXT:    v_mad_f32 v4, -v5, v0, v4
961; GFX10-NEXT:    v_cmp_ge_f32_e64 s6, |v4|, |v0|
962; GFX10-NEXT:    v_cvt_i32_f32_e32 v4, v5
963; GFX10-NEXT:    s_and_b32 s6, s6, exec_lo
964; GFX10-NEXT:    s_cselect_b32 s6, s0, 0
965; GFX10-NEXT:    s_and_b32 s0, s5, 0xffff
966; GFX10-NEXT:    v_readfirstlane_b32 s5, v3
967; GFX10-NEXT:    v_add_nc_u32_e32 v3, s6, v4
968; GFX10-NEXT:    s_lshl_b64 s[6:7], s[0:1], 1
969; GFX10-NEXT:    s_add_u32 s6, s2, s6
970; GFX10-NEXT:    s_addc_u32 s7, s3, s7
971; GFX10-NEXT:    global_store_short v2, v3, s[6:7]
972; GFX10-NEXT:    s_cbranch_vccz .LBB6_1
973; GFX10-NEXT:  ; %bb.2: ; %bb2
974; GFX10-NEXT:    s_endpgm
975;
976; GFX11-LABEL: sdiv16_invariant_denom:
977; GFX11:       ; %bb.0: ; %bb
978; GFX11-NEXT:    s_clause 0x1
979; GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x2c
980; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
981; GFX11-NEXT:    v_mov_b32_e32 v2, 0
982; GFX11-NEXT:    s_mov_b32 s3, 0
983; GFX11-NEXT:    s_mov_b32 s5, 0
984; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
985; GFX11-NEXT:    s_sext_i32_i16 s4, s2
986; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
987; GFX11-NEXT:    v_cvt_f32_i32_e32 v0, s4
988; GFX11-NEXT:    v_rcp_iflag_f32_e32 v1, v0
989; GFX11-NEXT:    .p2align 6
990; GFX11-NEXT:  .LBB6_1: ; %bb3
991; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
992; GFX11-NEXT:    s_sext_i32_i16 s2, s5
993; GFX11-NEXT:    v_add_nc_u16 v3, s5, 1
994; GFX11-NEXT:    v_cvt_f32_i32_e32 v4, s2
995; GFX11-NEXT:    s_xor_b32 s2, s2, s4
996; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
997; GFX11-NEXT:    s_ashr_i32 s2, s2, 30
998; GFX11-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x400, v3
999; GFX11-NEXT:    s_waitcnt_depctr 0xfff
1000; GFX11-NEXT:    v_mul_f32_e32 v5, v4, v1
1001; GFX11-NEXT:    s_or_b32 s2, s2, 1
1002; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
1003; GFX11-NEXT:    v_trunc_f32_e32 v5, v5
1004; GFX11-NEXT:    v_fma_f32 v4, -v5, v0, v4
1005; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
1006; GFX11-NEXT:    v_cmp_ge_f32_e64 s6, |v4|, |v0|
1007; GFX11-NEXT:    v_cvt_i32_f32_e32 v4, v5
1008; GFX11-NEXT:    s_and_b32 s6, s6, exec_lo
1009; GFX11-NEXT:    s_cselect_b32 s6, s2, 0
1010; GFX11-NEXT:    s_and_b32 s2, s5, 0xffff
1011; GFX11-NEXT:    v_readfirstlane_b32 s5, v3
1012; GFX11-NEXT:    v_add_nc_u32_e32 v3, s6, v4
1013; GFX11-NEXT:    s_lshl_b64 s[6:7], s[2:3], 1
1014; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
1015; GFX11-NEXT:    s_add_u32 s6, s0, s6
1016; GFX11-NEXT:    s_addc_u32 s7, s1, s7
1017; GFX11-NEXT:    global_store_b16 v2, v3, s[6:7]
1018; GFX11-NEXT:    s_cbranch_vccz .LBB6_1
1019; GFX11-NEXT:  ; %bb.2: ; %bb2
1020; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1021; GFX11-NEXT:    s_endpgm
1022bb:
1023  br label %bb3
1024
1025bb2:                                              ; preds = %bb3
1026  ret void
1027
1028bb3:                                              ; preds = %bb3, %bb
1029  %tmp = phi i16 [ 0, %bb ], [ %tmp7, %bb3 ]
1030  %tmp4 = sdiv i16 %tmp, %arg1
1031  %tmp5 = zext i16 %tmp to i64
1032  %tmp6 = getelementptr inbounds i16, i16 addrspace(1)* %arg, i64 %tmp5
1033  store i16 %tmp4, i16 addrspace(1)* %tmp6, align 2
1034  %tmp7 = add nuw nsw i16 %tmp, 1
1035  %tmp8 = icmp eq i16 %tmp7, 1024
1036  br i1 %tmp8, label %bb2, label %bb3
1037}
1038
1039define amdgpu_kernel void @srem16_invariant_denom(i16 addrspace(1)* nocapture %arg, i16 %arg1) {
1040; GFX9-LABEL: srem16_invariant_denom:
1041; GFX9:       ; %bb.0: ; %bb
1042; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x2c
1043; GFX9-NEXT:    s_mov_b32 s3, 0
1044; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
1045; GFX9-NEXT:    v_mov_b32_e32 v2, 0
1046; GFX9-NEXT:    s_movk_i32 s5, 0x400
1047; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
1048; GFX9-NEXT:    s_sext_i32_i16 s4, s2
1049; GFX9-NEXT:    v_cvt_f32_i32_e32 v0, s4
1050; GFX9-NEXT:    s_mov_b32 s6, 0
1051; GFX9-NEXT:    v_rcp_iflag_f32_e32 v1, v0
1052; GFX9-NEXT:  .LBB7_1: ; %bb3
1053; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
1054; GFX9-NEXT:    s_sext_i32_i16 s7, s6
1055; GFX9-NEXT:    v_cvt_f32_i32_e32 v4, s7
1056; GFX9-NEXT:    s_xor_b32 s2, s7, s4
1057; GFX9-NEXT:    s_ashr_i32 s2, s2, 30
1058; GFX9-NEXT:    s_or_b32 s2, s2, 1
1059; GFX9-NEXT:    v_mul_f32_e32 v5, v4, v1
1060; GFX9-NEXT:    v_trunc_f32_e32 v5, v5
1061; GFX9-NEXT:    v_mad_f32 v4, -v5, v0, v4
1062; GFX9-NEXT:    v_cvt_i32_f32_e32 v5, v5
1063; GFX9-NEXT:    v_cmp_ge_f32_e64 s[8:9], |v4|, |v0|
1064; GFX9-NEXT:    s_and_b64 s[8:9], s[8:9], exec
1065; GFX9-NEXT:    v_add_u16_e64 v3, s6, 1
1066; GFX9-NEXT:    s_cselect_b32 s8, s2, 0
1067; GFX9-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v3
1068; GFX9-NEXT:    s_and_b32 s2, s6, 0xffff
1069; GFX9-NEXT:    v_readfirstlane_b32 s6, v3
1070; GFX9-NEXT:    v_add_u32_e32 v3, s8, v5
1071; GFX9-NEXT:    v_mul_lo_u32 v3, v3, s4
1072; GFX9-NEXT:    s_lshl_b64 s[8:9], s[2:3], 1
1073; GFX9-NEXT:    s_add_u32 s8, s0, s8
1074; GFX9-NEXT:    s_addc_u32 s9, s1, s9
1075; GFX9-NEXT:    v_sub_u32_e32 v3, s7, v3
1076; GFX9-NEXT:    global_store_short v2, v3, s[8:9]
1077; GFX9-NEXT:    s_cbranch_vccz .LBB7_1
1078; GFX9-NEXT:  ; %bb.2: ; %bb2
1079; GFX9-NEXT:    s_endpgm
1080;
1081; GFX10-LABEL: srem16_invariant_denom:
1082; GFX10:       ; %bb.0: ; %bb
1083; GFX10-NEXT:    s_clause 0x1
1084; GFX10-NEXT:    s_load_dword s4, s[0:1], 0x2c
1085; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
1086; GFX10-NEXT:    v_mov_b32_e32 v2, 0
1087; GFX10-NEXT:    s_mov_b32 s1, 0
1088; GFX10-NEXT:    s_mov_b32 s5, 0
1089; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
1090; GFX10-NEXT:    s_sext_i32_i16 s4, s4
1091; GFX10-NEXT:    v_cvt_f32_i32_e32 v0, s4
1092; GFX10-NEXT:    v_rcp_iflag_f32_e32 v1, v0
1093; GFX10-NEXT:  .LBB7_1: ; %bb3
1094; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
1095; GFX10-NEXT:    s_sext_i32_i16 s8, s5
1096; GFX10-NEXT:    v_add_nc_u16 v3, s5, 1
1097; GFX10-NEXT:    v_cvt_f32_i32_e32 v4, s8
1098; GFX10-NEXT:    s_xor_b32 s0, s8, s4
1099; GFX10-NEXT:    s_ashr_i32 s0, s0, 30
1100; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x400, v3
1101; GFX10-NEXT:    v_mul_f32_e32 v5, v4, v1
1102; GFX10-NEXT:    s_or_b32 s0, s0, 1
1103; GFX10-NEXT:    v_trunc_f32_e32 v5, v5
1104; GFX10-NEXT:    v_mad_f32 v4, -v5, v0, v4
1105; GFX10-NEXT:    v_cmp_ge_f32_e64 s6, |v4|, |v0|
1106; GFX10-NEXT:    v_cvt_i32_f32_e32 v4, v5
1107; GFX10-NEXT:    s_and_b32 s6, s6, exec_lo
1108; GFX10-NEXT:    s_cselect_b32 s6, s0, 0
1109; GFX10-NEXT:    s_and_b32 s0, s5, 0xffff
1110; GFX10-NEXT:    v_add_nc_u32_e32 v4, s6, v4
1111; GFX10-NEXT:    v_readfirstlane_b32 s5, v3
1112; GFX10-NEXT:    s_lshl_b64 s[6:7], s[0:1], 1
1113; GFX10-NEXT:    s_add_u32 s6, s2, s6
1114; GFX10-NEXT:    v_mul_lo_u32 v3, v4, s4
1115; GFX10-NEXT:    s_addc_u32 s7, s3, s7
1116; GFX10-NEXT:    v_sub_nc_u32_e32 v3, s8, v3
1117; GFX10-NEXT:    global_store_short v2, v3, s[6:7]
1118; GFX10-NEXT:    s_cbranch_vccz .LBB7_1
1119; GFX10-NEXT:  ; %bb.2: ; %bb2
1120; GFX10-NEXT:    s_endpgm
1121;
1122; GFX11-LABEL: srem16_invariant_denom:
1123; GFX11:       ; %bb.0: ; %bb
1124; GFX11-NEXT:    s_clause 0x1
1125; GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x2c
1126; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
1127; GFX11-NEXT:    v_mov_b32_e32 v2, 0
1128; GFX11-NEXT:    s_mov_b32 s3, 0
1129; GFX11-NEXT:    s_mov_b32 s5, 0
1130; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
1131; GFX11-NEXT:    s_sext_i32_i16 s4, s2
1132; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
1133; GFX11-NEXT:    v_cvt_f32_i32_e32 v0, s4
1134; GFX11-NEXT:    v_rcp_iflag_f32_e32 v1, v0
1135; GFX11-NEXT:    s_set_inst_prefetch_distance 0x1
1136; GFX11-NEXT:    .p2align 6
1137; GFX11-NEXT:  .LBB7_1: ; %bb3
1138; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
1139; GFX11-NEXT:    s_sext_i32_i16 s8, s5
1140; GFX11-NEXT:    v_add_nc_u16 v3, s5, 1
1141; GFX11-NEXT:    v_cvt_f32_i32_e32 v4, s8
1142; GFX11-NEXT:    s_xor_b32 s2, s8, s4
1143; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
1144; GFX11-NEXT:    s_ashr_i32 s2, s2, 30
1145; GFX11-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x400, v3
1146; GFX11-NEXT:    s_waitcnt_depctr 0xfff
1147; GFX11-NEXT:    v_mul_f32_e32 v5, v4, v1
1148; GFX11-NEXT:    s_or_b32 s2, s2, 1
1149; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
1150; GFX11-NEXT:    v_trunc_f32_e32 v5, v5
1151; GFX11-NEXT:    v_fma_f32 v4, -v5, v0, v4
1152; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
1153; GFX11-NEXT:    v_cmp_ge_f32_e64 s6, |v4|, |v0|
1154; GFX11-NEXT:    v_cvt_i32_f32_e32 v4, v5
1155; GFX11-NEXT:    s_and_b32 s6, s6, exec_lo
1156; GFX11-NEXT:    s_cselect_b32 s6, s2, 0
1157; GFX11-NEXT:    s_and_b32 s2, s5, 0xffff
1158; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
1159; GFX11-NEXT:    v_add_nc_u32_e32 v4, s6, v4
1160; GFX11-NEXT:    v_readfirstlane_b32 s5, v3
1161; GFX11-NEXT:    s_lshl_b64 s[6:7], s[2:3], 1
1162; GFX11-NEXT:    s_add_u32 s6, s0, s6
1163; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
1164; GFX11-NEXT:    v_mul_lo_u32 v3, v4, s4
1165; GFX11-NEXT:    s_addc_u32 s7, s1, s7
1166; GFX11-NEXT:    v_sub_nc_u32_e32 v3, s8, v3
1167; GFX11-NEXT:    global_store_b16 v2, v3, s[6:7]
1168; GFX11-NEXT:    s_cbranch_vccz .LBB7_1
1169; GFX11-NEXT:  ; %bb.2: ; %bb2
1170; GFX11-NEXT:    s_set_inst_prefetch_distance 0x2
1171; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1172; GFX11-NEXT:    s_endpgm
1173bb:
1174  br label %bb3
1175
1176bb2:                                              ; preds = %bb3
1177  ret void
1178
1179bb3:                                              ; preds = %bb3, %bb
1180  %tmp = phi i16 [ 0, %bb ], [ %tmp7, %bb3 ]
1181  %tmp4 = srem i16 %tmp, %arg1
1182  %tmp5 = zext i16 %tmp to i64
1183  %tmp6 = getelementptr inbounds i16, i16 addrspace(1)* %arg, i64 %tmp5
1184  store i16 %tmp4, i16 addrspace(1)* %tmp6, align 2
1185  %tmp7 = add nuw nsw i16 %tmp, 1
1186  %tmp8 = icmp eq i16 %tmp7, 1024
1187  br i1 %tmp8, label %bb2, label %bb3
1188}
1189