1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GFX9 %s
3; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GFX10 %s
4; RUN: llc -march=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GFX11 %s
5
6define amdgpu_kernel void @udiv32_invariant_denom(i32 addrspace(1)* nocapture %arg, i32 %arg1) {
7; GFX9-LABEL: udiv32_invariant_denom:
8; GFX9:       ; %bb.0: ; %bb
9; GFX9-NEXT:    s_load_dword s4, s[0:1], 0x2c
10; GFX9-NEXT:    s_mov_b64 s[2:3], 0
11; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
12; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
13; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s4
14; GFX9-NEXT:    s_sub_i32 s5, 0, s4
15; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
16; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
17; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
18; GFX9-NEXT:    v_mul_lo_u32 v1, s5, v0
19; GFX9-NEXT:    v_mul_hi_u32 v1, v0, v1
20; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
21; GFX9-NEXT:    v_mov_b32_e32 v1, 0
22; GFX9-NEXT:  .LBB0_1: ; %bb3
23; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
24; GFX9-NEXT:    v_mul_lo_u32 v2, s3, v0
25; GFX9-NEXT:    v_mul_hi_u32 v3, s2, v0
26; GFX9-NEXT:    v_add_u32_e32 v2, v3, v2
27; GFX9-NEXT:    v_mul_lo_u32 v3, s5, v2
28; GFX9-NEXT:    v_not_b32_e32 v5, v2
29; GFX9-NEXT:    v_mul_lo_u32 v5, s4, v5
30; GFX9-NEXT:    v_add_u32_e32 v4, 1, v2
31; GFX9-NEXT:    v_add_u32_e32 v3, s2, v3
32; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, s4, v3
33; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
34; GFX9-NEXT:    v_add_u32_e32 v4, s2, v5
35; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
36; GFX9-NEXT:    s_add_u32 s2, s2, 1
37; GFX9-NEXT:    v_add_u32_e32 v4, 1, v2
38; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, s4, v3
39; GFX9-NEXT:    s_addc_u32 s3, s3, 0
40; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
41; GFX9-NEXT:    global_store_dword v1, v2, s[0:1]
42; GFX9-NEXT:    s_add_u32 s0, s0, 4
43; GFX9-NEXT:    s_addc_u32 s1, s1, 0
44; GFX9-NEXT:    s_cmpk_eq_i32 s2, 0x400
45; GFX9-NEXT:    s_cbranch_scc0 .LBB0_1
46; GFX9-NEXT:  ; %bb.2: ; %bb2
47; GFX9-NEXT:    s_endpgm
48;
49; GFX10-LABEL: udiv32_invariant_denom:
50; GFX10:       ; %bb.0: ; %bb
51; GFX10-NEXT:    s_load_dword s4, s[0:1], 0x2c
52; GFX10-NEXT:    s_mov_b64 s[2:3], 0
53; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
54; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
55; GFX10-NEXT:    v_cvt_f32_u32_e32 v0, s4
56; GFX10-NEXT:    s_sub_i32 s5, 0, s4
57; GFX10-NEXT:    v_rcp_iflag_f32_e32 v0, v0
58; GFX10-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
59; GFX10-NEXT:    v_cvt_u32_f32_e32 v0, v0
60; GFX10-NEXT:    v_mul_lo_u32 v1, s5, v0
61; GFX10-NEXT:    v_mul_hi_u32 v1, v0, v1
62; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v1
63; GFX10-NEXT:    v_mov_b32_e32 v1, 0
64; GFX10-NEXT:  .LBB0_1: ; %bb3
65; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
66; GFX10-NEXT:    v_mul_lo_u32 v2, s3, v0
67; GFX10-NEXT:    v_mul_hi_u32 v3, s2, v0
68; GFX10-NEXT:    v_add_nc_u32_e32 v2, v3, v2
69; GFX10-NEXT:    v_not_b32_e32 v3, v2
70; GFX10-NEXT:    v_mul_lo_u32 v4, s5, v2
71; GFX10-NEXT:    v_add_nc_u32_e32 v5, 1, v2
72; GFX10-NEXT:    v_mul_lo_u32 v3, s4, v3
73; GFX10-NEXT:    v_add_nc_u32_e32 v4, s2, v4
74; GFX10-NEXT:    v_add_nc_u32_e32 v3, s2, v3
75; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, s4, v4
76; GFX10-NEXT:    s_add_u32 s2, s2, 1
77; GFX10-NEXT:    s_addc_u32 s3, s3, 0
78; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc_lo
79; GFX10-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc_lo
80; GFX10-NEXT:    v_add_nc_u32_e32 v4, 1, v2
81; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, s4, v3
82; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc_lo
83; GFX10-NEXT:    global_store_dword v1, v2, s[0:1]
84; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
85; GFX10-NEXT:    s_add_u32 s0, s0, 4
86; GFX10-NEXT:    s_addc_u32 s1, s1, 0
87; GFX10-NEXT:    s_cmpk_eq_i32 s2, 0x400
88; GFX10-NEXT:    s_cbranch_scc0 .LBB0_1
89; GFX10-NEXT:  ; %bb.2: ; %bb2
90; GFX10-NEXT:    s_endpgm
91;
92; GFX11-LABEL: udiv32_invariant_denom:
93; GFX11:       ; %bb.0: ; %bb
94; GFX11-NEXT:    s_clause 0x1
95; GFX11-NEXT:    s_load_b32 s4, s[0:1], 0x2c
96; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
97; GFX11-NEXT:    s_mov_b64 s[2:3], 0
98; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
99; GFX11-NEXT:    v_cvt_f32_u32_e32 v0, s4
100; GFX11-NEXT:    s_sub_i32 s5, 0, s4
101; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
102; GFX11-NEXT:    v_rcp_iflag_f32_e32 v0, v0
103; GFX11-NEXT:    s_waitcnt_depctr 0xfff
104; GFX11-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
105; GFX11-NEXT:    v_cvt_u32_f32_e32 v0, v0
106; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
107; GFX11-NEXT:    v_mul_lo_u32 v1, s5, v0
108; GFX11-NEXT:    v_mul_hi_u32 v1, v0, v1
109; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
110; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, v0, v1
111; GFX11-NEXT:    .p2align 6
112; GFX11-NEXT:  .LBB0_1: ; %bb3
113; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
114; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
115; GFX11-NEXT:    v_mul_lo_u32 v2, s3, v0
116; GFX11-NEXT:    v_mul_hi_u32 v3, s2, v0
117; GFX11-NEXT:    v_add_nc_u32_e32 v2, v3, v2
118; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
119; GFX11-NEXT:    v_not_b32_e32 v3, v2
120; GFX11-NEXT:    v_mul_lo_u32 v4, s5, v2
121; GFX11-NEXT:    v_mul_lo_u32 v3, s4, v3
122; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
123; GFX11-NEXT:    v_add_nc_u32_e32 v4, s2, v4
124; GFX11-NEXT:    v_cmp_le_u32_e32 vcc_lo, s4, v4
125; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
126; GFX11-NEXT:    v_add_nc_u32_e32 v3, s2, v3
127; GFX11-NEXT:    s_add_u32 s2, s2, 1
128; GFX11-NEXT:    s_addc_u32 s3, s3, 0
129; GFX11-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc_lo
130; GFX11-NEXT:    v_add_nc_u32_e32 v5, 1, v2
131; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
132; GFX11-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc_lo
133; GFX11-NEXT:    v_cmp_le_u32_e32 vcc_lo, s4, v3
134; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
135; GFX11-NEXT:    v_add_nc_u32_e32 v4, 1, v2
136; GFX11-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc_lo
137; GFX11-NEXT:    global_store_b32 v1, v2, s[0:1]
138; GFX11-NEXT:    s_add_u32 s0, s0, 4
139; GFX11-NEXT:    s_addc_u32 s1, s1, 0
140; GFX11-NEXT:    s_cmpk_eq_i32 s2, 0x400
141; GFX11-NEXT:    s_cbranch_scc0 .LBB0_1
142; GFX11-NEXT:  ; %bb.2: ; %bb2
143; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
144; GFX11-NEXT:    s_endpgm
145bb:
146  br label %bb3
147
148bb2:                                              ; preds = %bb3
149  ret void
150
151bb3:                                              ; preds = %bb3, %bb
152  %tmp = phi i32 [ 0, %bb ], [ %tmp7, %bb3 ]
153  %tmp4 = udiv i32 %tmp, %arg1
154  %tmp5 = zext i32 %tmp to i64
155  %tmp6 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 %tmp5
156  store i32 %tmp4, i32 addrspace(1)* %tmp6, align 4
157  %tmp7 = add nuw nsw i32 %tmp, 1
158  %tmp8 = icmp eq i32 %tmp7, 1024
159  br i1 %tmp8, label %bb2, label %bb3
160}
161
162define amdgpu_kernel void @urem32_invariant_denom(i32 addrspace(1)* nocapture %arg, i32 %arg1) {
163; GFX9-LABEL: urem32_invariant_denom:
164; GFX9:       ; %bb.0: ; %bb
165; GFX9-NEXT:    s_load_dword s4, s[0:1], 0x2c
166; GFX9-NEXT:    s_mov_b64 s[2:3], 0
167; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
168; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
169; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s4
170; GFX9-NEXT:    s_sub_i32 s5, 0, s4
171; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
172; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
173; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
174; GFX9-NEXT:    v_mul_lo_u32 v1, s5, v0
175; GFX9-NEXT:    v_mul_hi_u32 v1, v0, v1
176; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
177; GFX9-NEXT:    v_mov_b32_e32 v1, 0
178; GFX9-NEXT:  .LBB1_1: ; %bb3
179; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
180; GFX9-NEXT:    v_mul_lo_u32 v2, s3, v0
181; GFX9-NEXT:    v_mul_hi_u32 v3, s2, v0
182; GFX9-NEXT:    v_add_u32_e32 v2, v3, v2
183; GFX9-NEXT:    v_mul_lo_u32 v3, s5, v2
184; GFX9-NEXT:    v_not_b32_e32 v2, v2
185; GFX9-NEXT:    v_mul_lo_u32 v2, s4, v2
186; GFX9-NEXT:    v_add_u32_e32 v3, s2, v3
187; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, s4, v3
188; GFX9-NEXT:    v_add_u32_e32 v2, s2, v2
189; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
190; GFX9-NEXT:    s_add_u32 s2, s2, 1
191; GFX9-NEXT:    v_subrev_u32_e32 v3, s4, v2
192; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, s4, v2
193; GFX9-NEXT:    s_addc_u32 s3, s3, 0
194; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
195; GFX9-NEXT:    global_store_dword v1, v2, s[0:1]
196; GFX9-NEXT:    s_add_u32 s0, s0, 4
197; GFX9-NEXT:    s_addc_u32 s1, s1, 0
198; GFX9-NEXT:    s_cmpk_eq_i32 s2, 0x400
199; GFX9-NEXT:    s_cbranch_scc0 .LBB1_1
200; GFX9-NEXT:  ; %bb.2: ; %bb2
201; GFX9-NEXT:    s_endpgm
202;
203; GFX10-LABEL: urem32_invariant_denom:
204; GFX10:       ; %bb.0: ; %bb
205; GFX10-NEXT:    s_load_dword s4, s[0:1], 0x2c
206; GFX10-NEXT:    s_mov_b64 s[2:3], 0
207; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
208; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
209; GFX10-NEXT:    v_cvt_f32_u32_e32 v0, s4
210; GFX10-NEXT:    s_sub_i32 s5, 0, s4
211; GFX10-NEXT:    v_rcp_iflag_f32_e32 v0, v0
212; GFX10-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
213; GFX10-NEXT:    v_cvt_u32_f32_e32 v0, v0
214; GFX10-NEXT:    v_mul_lo_u32 v1, s5, v0
215; GFX10-NEXT:    v_mul_hi_u32 v1, v0, v1
216; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v1
217; GFX10-NEXT:    v_mov_b32_e32 v1, 0
218; GFX10-NEXT:  .LBB1_1: ; %bb3
219; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
220; GFX10-NEXT:    v_mul_lo_u32 v2, s3, v0
221; GFX10-NEXT:    v_mul_hi_u32 v3, s2, v0
222; GFX10-NEXT:    v_add_nc_u32_e32 v2, v3, v2
223; GFX10-NEXT:    v_not_b32_e32 v3, v2
224; GFX10-NEXT:    v_mul_lo_u32 v2, s5, v2
225; GFX10-NEXT:    v_mul_lo_u32 v3, s4, v3
226; GFX10-NEXT:    v_add_nc_u32_e32 v2, s2, v2
227; GFX10-NEXT:    v_add_nc_u32_e32 v3, s2, v3
228; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, s4, v2
229; GFX10-NEXT:    s_add_u32 s2, s2, 1
230; GFX10-NEXT:    s_addc_u32 s3, s3, 0
231; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc_lo
232; GFX10-NEXT:    v_subrev_nc_u32_e32 v3, s4, v2
233; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, s4, v2
234; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc_lo
235; GFX10-NEXT:    global_store_dword v1, v2, s[0:1]
236; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
237; GFX10-NEXT:    s_add_u32 s0, s0, 4
238; GFX10-NEXT:    s_addc_u32 s1, s1, 0
239; GFX10-NEXT:    s_cmpk_eq_i32 s2, 0x400
240; GFX10-NEXT:    s_cbranch_scc0 .LBB1_1
241; GFX10-NEXT:  ; %bb.2: ; %bb2
242; GFX10-NEXT:    s_endpgm
243;
244; GFX11-LABEL: urem32_invariant_denom:
245; GFX11:       ; %bb.0: ; %bb
246; GFX11-NEXT:    s_clause 0x1
247; GFX11-NEXT:    s_load_b32 s4, s[0:1], 0x2c
248; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
249; GFX11-NEXT:    s_mov_b64 s[2:3], 0
250; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
251; GFX11-NEXT:    v_cvt_f32_u32_e32 v0, s4
252; GFX11-NEXT:    s_sub_i32 s5, 0, s4
253; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
254; GFX11-NEXT:    v_rcp_iflag_f32_e32 v0, v0
255; GFX11-NEXT:    s_waitcnt_depctr 0xfff
256; GFX11-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
257; GFX11-NEXT:    v_cvt_u32_f32_e32 v0, v0
258; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
259; GFX11-NEXT:    v_mul_lo_u32 v1, s5, v0
260; GFX11-NEXT:    v_mul_hi_u32 v1, v0, v1
261; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
262; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, v0, v1
263; GFX11-NEXT:    .p2align 6
264; GFX11-NEXT:  .LBB1_1: ; %bb3
265; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
266; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
267; GFX11-NEXT:    v_mul_lo_u32 v2, s3, v0
268; GFX11-NEXT:    v_mul_hi_u32 v3, s2, v0
269; GFX11-NEXT:    v_add_nc_u32_e32 v2, v3, v2
270; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
271; GFX11-NEXT:    v_not_b32_e32 v3, v2
272; GFX11-NEXT:    v_mul_lo_u32 v2, s5, v2
273; GFX11-NEXT:    v_mul_lo_u32 v3, s4, v3
274; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
275; GFX11-NEXT:    v_add_nc_u32_e32 v2, s2, v2
276; GFX11-NEXT:    v_add_nc_u32_e32 v3, s2, v3
277; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
278; GFX11-NEXT:    v_cmp_le_u32_e32 vcc_lo, s4, v2
279; GFX11-NEXT:    s_add_u32 s2, s2, 1
280; GFX11-NEXT:    s_addc_u32 s3, s3, 0
281; GFX11-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc_lo
282; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
283; GFX11-NEXT:    v_subrev_nc_u32_e32 v3, s4, v2
284; GFX11-NEXT:    v_cmp_le_u32_e32 vcc_lo, s4, v2
285; GFX11-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc_lo
286; GFX11-NEXT:    global_store_b32 v1, v2, s[0:1]
287; GFX11-NEXT:    s_add_u32 s0, s0, 4
288; GFX11-NEXT:    s_addc_u32 s1, s1, 0
289; GFX11-NEXT:    s_cmpk_eq_i32 s2, 0x400
290; GFX11-NEXT:    s_cbranch_scc0 .LBB1_1
291; GFX11-NEXT:  ; %bb.2: ; %bb2
292; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
293; GFX11-NEXT:    s_endpgm
294bb:
295  br label %bb3
296
297bb2:                                              ; preds = %bb3
298  ret void
299
300bb3:                                              ; preds = %bb3, %bb
301  %tmp = phi i32 [ 0, %bb ], [ %tmp7, %bb3 ]
302  %tmp4 = urem i32 %tmp, %arg1
303  %tmp5 = zext i32 %tmp to i64
304  %tmp6 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 %tmp5
305  store i32 %tmp4, i32 addrspace(1)* %tmp6, align 4
306  %tmp7 = add nuw nsw i32 %tmp, 1
307  %tmp8 = icmp eq i32 %tmp7, 1024
308  br i1 %tmp8, label %bb2, label %bb3
309}
310
311define amdgpu_kernel void @sdiv32_invariant_denom(i32 addrspace(1)* nocapture %arg, i32 %arg1) {
312; GFX9-LABEL: sdiv32_invariant_denom:
313; GFX9:       ; %bb.0: ; %bb
314; GFX9-NEXT:    s_load_dword s3, s[0:1], 0x2c
315; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
316; GFX9-NEXT:    s_ashr_i32 s2, s3, 31
317; GFX9-NEXT:    s_add_i32 s3, s3, s2
318; GFX9-NEXT:    s_xor_b32 s3, s3, s2
319; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s3
320; GFX9-NEXT:    s_sub_i32 s4, 0, s3
321; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
322; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
323; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
324; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
325; GFX9-NEXT:    v_mul_lo_u32 v1, s4, v0
326; GFX9-NEXT:    s_mov_b32 s4, 0
327; GFX9-NEXT:    v_mul_hi_u32 v1, v0, v1
328; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
329; GFX9-NEXT:    v_mov_b32_e32 v1, 0
330; GFX9-NEXT:  .LBB2_1: ; %bb3
331; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
332; GFX9-NEXT:    v_mul_hi_u32 v2, s4, v0
333; GFX9-NEXT:    v_mul_lo_u32 v3, v2, s3
334; GFX9-NEXT:    v_add_u32_e32 v4, 1, v2
335; GFX9-NEXT:    v_sub_u32_e32 v3, s4, v3
336; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, s3, v3
337; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
338; GFX9-NEXT:    v_subrev_u32_e32 v4, s3, v3
339; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
340; GFX9-NEXT:    v_add_u32_e32 v4, 1, v2
341; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, s3, v3
342; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
343; GFX9-NEXT:    v_xor_b32_e32 v2, s2, v2
344; GFX9-NEXT:    s_add_i32 s4, s4, 1
345; GFX9-NEXT:    v_subrev_u32_e32 v2, s2, v2
346; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
347; GFX9-NEXT:    global_store_dword v1, v2, s[0:1]
348; GFX9-NEXT:    s_add_u32 s0, s0, 4
349; GFX9-NEXT:    s_addc_u32 s1, s1, 0
350; GFX9-NEXT:    s_cmpk_eq_i32 s4, 0x400
351; GFX9-NEXT:    s_cbranch_scc0 .LBB2_1
352; GFX9-NEXT:  ; %bb.2: ; %bb2
353; GFX9-NEXT:    s_endpgm
354;
355; GFX10-LABEL: sdiv32_invariant_denom:
356; GFX10:       ; %bb.0: ; %bb
357; GFX10-NEXT:    s_load_dword s3, s[0:1], 0x2c
358; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
359; GFX10-NEXT:    s_ashr_i32 s2, s3, 31
360; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
361; GFX10-NEXT:    s_add_i32 s3, s3, s2
362; GFX10-NEXT:    s_xor_b32 s3, s3, s2
363; GFX10-NEXT:    v_cvt_f32_u32_e32 v0, s3
364; GFX10-NEXT:    s_sub_i32 s4, 0, s3
365; GFX10-NEXT:    v_rcp_iflag_f32_e32 v0, v0
366; GFX10-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
367; GFX10-NEXT:    v_cvt_u32_f32_e32 v0, v0
368; GFX10-NEXT:    v_mul_lo_u32 v1, s4, v0
369; GFX10-NEXT:    s_mov_b32 s4, 0
370; GFX10-NEXT:    v_mul_hi_u32 v1, v0, v1
371; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v1
372; GFX10-NEXT:    v_mov_b32_e32 v1, 0
373; GFX10-NEXT:  .LBB2_1: ; %bb3
374; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
375; GFX10-NEXT:    v_mul_hi_u32 v2, s4, v0
376; GFX10-NEXT:    v_mul_lo_u32 v3, v2, s3
377; GFX10-NEXT:    v_add_nc_u32_e32 v4, 1, v2
378; GFX10-NEXT:    v_sub_nc_u32_e32 v3, s4, v3
379; GFX10-NEXT:    s_add_i32 s4, s4, 1
380; GFX10-NEXT:    v_subrev_nc_u32_e32 v5, s3, v3
381; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, s3, v3
382; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc_lo
383; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc_lo
384; GFX10-NEXT:    v_add_nc_u32_e32 v4, 1, v2
385; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, s3, v3
386; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc_lo
387; GFX10-NEXT:    v_xor_b32_e32 v2, s2, v2
388; GFX10-NEXT:    v_subrev_nc_u32_e32 v2, s2, v2
389; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
390; GFX10-NEXT:    global_store_dword v1, v2, s[0:1]
391; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
392; GFX10-NEXT:    s_add_u32 s0, s0, 4
393; GFX10-NEXT:    s_addc_u32 s1, s1, 0
394; GFX10-NEXT:    s_cmpk_eq_i32 s4, 0x400
395; GFX10-NEXT:    s_cbranch_scc0 .LBB2_1
396; GFX10-NEXT:  ; %bb.2: ; %bb2
397; GFX10-NEXT:    s_endpgm
398;
399; GFX11-LABEL: sdiv32_invariant_denom:
400; GFX11:       ; %bb.0: ; %bb
401; GFX11-NEXT:    s_clause 0x1
402; GFX11-NEXT:    s_load_b32 s3, s[0:1], 0x2c
403; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
404; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
405; GFX11-NEXT:    s_ashr_i32 s2, s3, 31
406; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
407; GFX11-NEXT:    s_add_i32 s3, s3, s2
408; GFX11-NEXT:    s_xor_b32 s3, s3, s2
409; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
410; GFX11-NEXT:    v_cvt_f32_u32_e32 v0, s3
411; GFX11-NEXT:    s_sub_i32 s4, 0, s3
412; GFX11-NEXT:    v_rcp_iflag_f32_e32 v0, v0
413; GFX11-NEXT:    s_waitcnt_depctr 0xfff
414; GFX11-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
415; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
416; GFX11-NEXT:    v_cvt_u32_f32_e32 v0, v0
417; GFX11-NEXT:    v_mul_lo_u32 v1, s4, v0
418; GFX11-NEXT:    s_mov_b32 s4, 0
419; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
420; GFX11-NEXT:    v_mul_hi_u32 v1, v0, v1
421; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, v0, v1
422; GFX11-NEXT:    .p2align 6
423; GFX11-NEXT:  .LBB2_1: ; %bb3
424; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
425; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
426; GFX11-NEXT:    v_mul_hi_u32 v2, s4, v0
427; GFX11-NEXT:    v_mul_lo_u32 v3, v2, s3
428; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
429; GFX11-NEXT:    v_sub_nc_u32_e32 v3, s4, v3
430; GFX11-NEXT:    s_add_i32 s4, s4, 1
431; GFX11-NEXT:    v_subrev_nc_u32_e32 v5, s3, v3
432; GFX11-NEXT:    v_cmp_le_u32_e32 vcc_lo, s3, v3
433; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
434; GFX11-NEXT:    v_dual_cndmask_b32 v3, v3, v5 :: v_dual_add_nc_u32 v4, 1, v2
435; GFX11-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc_lo
436; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
437; GFX11-NEXT:    v_cmp_le_u32_e32 vcc_lo, s3, v3
438; GFX11-NEXT:    v_add_nc_u32_e32 v4, 1, v2
439; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
440; GFX11-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc_lo
441; GFX11-NEXT:    v_xor_b32_e32 v2, s2, v2
442; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
443; GFX11-NEXT:    v_subrev_nc_u32_e32 v2, s2, v2
444; GFX11-NEXT:    global_store_b32 v1, v2, s[0:1]
445; GFX11-NEXT:    s_add_u32 s0, s0, 4
446; GFX11-NEXT:    s_addc_u32 s1, s1, 0
447; GFX11-NEXT:    s_cmpk_eq_i32 s4, 0x400
448; GFX11-NEXT:    s_cbranch_scc0 .LBB2_1
449; GFX11-NEXT:  ; %bb.2: ; %bb2
450; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
451; GFX11-NEXT:    s_endpgm
452bb:
453  br label %bb3
454
455bb2:                                              ; preds = %bb3
456  ret void
457
458bb3:                                              ; preds = %bb3, %bb
459  %tmp = phi i32 [ 0, %bb ], [ %tmp7, %bb3 ]
460  %tmp4 = sdiv i32 %tmp, %arg1
461  %tmp5 = zext i32 %tmp to i64
462  %tmp6 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 %tmp5
463  store i32 %tmp4, i32 addrspace(1)* %tmp6, align 4
464  %tmp7 = add nuw nsw i32 %tmp, 1
465  %tmp8 = icmp eq i32 %tmp7, 1024
466  br i1 %tmp8, label %bb2, label %bb3
467}
468
469define amdgpu_kernel void @srem32_invariant_denom(i32 addrspace(1)* nocapture %arg, i32 %arg1) {
470; GFX9-LABEL: srem32_invariant_denom:
471; GFX9:       ; %bb.0: ; %bb
472; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x2c
473; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
474; GFX9-NEXT:    s_ashr_i32 s3, s2, 31
475; GFX9-NEXT:    s_add_i32 s2, s2, s3
476; GFX9-NEXT:    s_xor_b32 s2, s2, s3
477; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s2
478; GFX9-NEXT:    s_sub_i32 s3, 0, s2
479; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
480; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
481; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
482; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
483; GFX9-NEXT:    v_mul_lo_u32 v1, s3, v0
484; GFX9-NEXT:    s_mov_b32 s3, 0
485; GFX9-NEXT:    v_mul_hi_u32 v1, v0, v1
486; GFX9-NEXT:    v_add_u32_e32 v0, v0, v1
487; GFX9-NEXT:    v_mov_b32_e32 v1, 0
488; GFX9-NEXT:  .LBB3_1: ; %bb3
489; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
490; GFX9-NEXT:    v_mul_hi_u32 v2, s3, v0
491; GFX9-NEXT:    v_mul_lo_u32 v2, v2, s2
492; GFX9-NEXT:    v_sub_u32_e32 v2, s3, v2
493; GFX9-NEXT:    v_subrev_u32_e32 v3, s2, v2
494; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, s2, v2
495; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
496; GFX9-NEXT:    v_subrev_u32_e32 v3, s2, v2
497; GFX9-NEXT:    v_cmp_le_u32_e32 vcc, s2, v2
498; GFX9-NEXT:    s_add_i32 s3, s3, 1
499; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
500; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
501; GFX9-NEXT:    global_store_dword v1, v2, s[0:1]
502; GFX9-NEXT:    s_add_u32 s0, s0, 4
503; GFX9-NEXT:    s_addc_u32 s1, s1, 0
504; GFX9-NEXT:    s_cmpk_eq_i32 s3, 0x400
505; GFX9-NEXT:    s_cbranch_scc0 .LBB3_1
506; GFX9-NEXT:  ; %bb.2: ; %bb2
507; GFX9-NEXT:    s_endpgm
508;
509; GFX10-LABEL: srem32_invariant_denom:
510; GFX10:       ; %bb.0: ; %bb
511; GFX10-NEXT:    s_load_dword s2, s[0:1], 0x2c
512; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
513; GFX10-NEXT:    s_ashr_i32 s3, s2, 31
514; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
515; GFX10-NEXT:    s_add_i32 s2, s2, s3
516; GFX10-NEXT:    s_xor_b32 s2, s2, s3
517; GFX10-NEXT:    v_cvt_f32_u32_e32 v0, s2
518; GFX10-NEXT:    s_sub_i32 s3, 0, s2
519; GFX10-NEXT:    v_rcp_iflag_f32_e32 v0, v0
520; GFX10-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
521; GFX10-NEXT:    v_cvt_u32_f32_e32 v0, v0
522; GFX10-NEXT:    v_mul_lo_u32 v1, s3, v0
523; GFX10-NEXT:    s_mov_b32 s3, 0
524; GFX10-NEXT:    v_mul_hi_u32 v1, v0, v1
525; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v1
526; GFX10-NEXT:    v_mov_b32_e32 v1, 0
527; GFX10-NEXT:  .LBB3_1: ; %bb3
528; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
529; GFX10-NEXT:    v_mul_hi_u32 v2, s3, v0
530; GFX10-NEXT:    v_mul_lo_u32 v2, v2, s2
531; GFX10-NEXT:    v_sub_nc_u32_e32 v2, s3, v2
532; GFX10-NEXT:    s_add_i32 s3, s3, 1
533; GFX10-NEXT:    v_subrev_nc_u32_e32 v3, s2, v2
534; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, s2, v2
535; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc_lo
536; GFX10-NEXT:    v_subrev_nc_u32_e32 v3, s2, v2
537; GFX10-NEXT:    v_cmp_le_u32_e32 vcc_lo, s2, v2
538; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc_lo
539; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
540; GFX10-NEXT:    global_store_dword v1, v2, s[0:1]
541; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
542; GFX10-NEXT:    s_add_u32 s0, s0, 4
543; GFX10-NEXT:    s_addc_u32 s1, s1, 0
544; GFX10-NEXT:    s_cmpk_eq_i32 s3, 0x400
545; GFX10-NEXT:    s_cbranch_scc0 .LBB3_1
546; GFX10-NEXT:  ; %bb.2: ; %bb2
547; GFX10-NEXT:    s_endpgm
548;
549; GFX11-LABEL: srem32_invariant_denom:
550; GFX11:       ; %bb.0: ; %bb
551; GFX11-NEXT:    s_clause 0x1
552; GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x2c
553; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
554; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
555; GFX11-NEXT:    s_ashr_i32 s3, s2, 31
556; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
557; GFX11-NEXT:    s_add_i32 s2, s2, s3
558; GFX11-NEXT:    s_xor_b32 s2, s2, s3
559; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
560; GFX11-NEXT:    v_cvt_f32_u32_e32 v0, s2
561; GFX11-NEXT:    s_sub_i32 s3, 0, s2
562; GFX11-NEXT:    v_rcp_iflag_f32_e32 v0, v0
563; GFX11-NEXT:    s_waitcnt_depctr 0xfff
564; GFX11-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
565; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
566; GFX11-NEXT:    v_cvt_u32_f32_e32 v0, v0
567; GFX11-NEXT:    v_mul_lo_u32 v1, s3, v0
568; GFX11-NEXT:    s_mov_b32 s3, 0
569; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
570; GFX11-NEXT:    v_mul_hi_u32 v1, v0, v1
571; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, v0, v1
572; GFX11-NEXT:    .p2align 6
573; GFX11-NEXT:  .LBB3_1: ; %bb3
574; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
575; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
576; GFX11-NEXT:    v_mul_hi_u32 v2, s3, v0
577; GFX11-NEXT:    v_mul_lo_u32 v2, v2, s2
578; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
579; GFX11-NEXT:    v_sub_nc_u32_e32 v2, s3, v2
580; GFX11-NEXT:    s_add_i32 s3, s3, 1
581; GFX11-NEXT:    v_subrev_nc_u32_e32 v3, s2, v2
582; GFX11-NEXT:    v_cmp_le_u32_e32 vcc_lo, s2, v2
583; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
584; GFX11-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc_lo
585; GFX11-NEXT:    v_subrev_nc_u32_e32 v3, s2, v2
586; GFX11-NEXT:    v_cmp_le_u32_e32 vcc_lo, s2, v2
587; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
588; GFX11-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc_lo
589; GFX11-NEXT:    global_store_b32 v1, v2, s[0:1]
590; GFX11-NEXT:    s_add_u32 s0, s0, 4
591; GFX11-NEXT:    s_addc_u32 s1, s1, 0
592; GFX11-NEXT:    s_cmpk_eq_i32 s3, 0x400
593; GFX11-NEXT:    s_cbranch_scc0 .LBB3_1
594; GFX11-NEXT:  ; %bb.2: ; %bb2
595; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
596; GFX11-NEXT:    s_endpgm
597bb:
598  br label %bb3
599
600bb2:                                              ; preds = %bb3
601  ret void
602
603bb3:                                              ; preds = %bb3, %bb
604  %tmp = phi i32 [ 0, %bb ], [ %tmp7, %bb3 ]
605  %tmp4 = srem i32 %tmp, %arg1
606  %tmp5 = zext i32 %tmp to i64
607  %tmp6 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 %tmp5
608  store i32 %tmp4, i32 addrspace(1)* %tmp6, align 4
609  %tmp7 = add nuw nsw i32 %tmp, 1
610  %tmp8 = icmp eq i32 %tmp7, 1024
611  br i1 %tmp8, label %bb2, label %bb3
612}
613
614define amdgpu_kernel void @udiv16_invariant_denom(i16 addrspace(1)* nocapture %arg, i16 %arg1) {
615; GFX9-LABEL: udiv16_invariant_denom:
616; GFX9:       ; %bb.0: ; %bb
617; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x2c
618; GFX9-NEXT:    v_mov_b32_e32 v1, 0
619; GFX9-NEXT:    s_movk_i32 s4, 0x400
620; GFX9-NEXT:    v_mov_b32_e32 v4, 0
621; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
622; GFX9-NEXT:    s_and_b32 s2, 0xffff, s2
623; GFX9-NEXT:    v_cvt_f32_u32_e32 v2, s2
624; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
625; GFX9-NEXT:    v_rcp_iflag_f32_e32 v3, v2
626; GFX9-NEXT:  .LBB4_1: ; %bb3
627; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
628; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v4
629; GFX9-NEXT:    v_cvt_f32_u32_e32 v8, v0
630; GFX9-NEXT:    v_lshlrev_b64 v[5:6], 1, v[0:1]
631; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
632; GFX9-NEXT:    v_mov_b32_e32 v7, s3
633; GFX9-NEXT:    v_add_co_u32_e64 v5, s[0:1], s2, v5
634; GFX9-NEXT:    v_mul_f32_e32 v0, v8, v3
635; GFX9-NEXT:    v_trunc_f32_e32 v0, v0
636; GFX9-NEXT:    v_addc_co_u32_e64 v6, s[0:1], v7, v6, s[0:1]
637; GFX9-NEXT:    v_cvt_u32_f32_e32 v7, v0
638; GFX9-NEXT:    v_add_u16_e32 v4, 1, v4
639; GFX9-NEXT:    v_mad_f32 v0, -v0, v2, v8
640; GFX9-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v4
641; GFX9-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v0|, v2
642; GFX9-NEXT:    v_addc_co_u32_e64 v0, s[0:1], 0, v7, s[0:1]
643; GFX9-NEXT:    global_store_short v[5:6], v0, off
644; GFX9-NEXT:    s_cbranch_vccz .LBB4_1
645; GFX9-NEXT:  ; %bb.2: ; %bb2
646; GFX9-NEXT:    s_endpgm
647;
648; GFX10-LABEL: udiv16_invariant_denom:
649; GFX10:       ; %bb.0: ; %bb
650; GFX10-NEXT:    s_clause 0x1
651; GFX10-NEXT:    s_load_dword s4, s[0:1], 0x2c
652; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
653; GFX10-NEXT:    v_mov_b32_e32 v1, 0
654; GFX10-NEXT:    v_mov_b32_e32 v4, 0
655; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
656; GFX10-NEXT:    s_and_b32 s0, 0xffff, s4
657; GFX10-NEXT:    v_cvt_f32_u32_e32 v2, s0
658; GFX10-NEXT:    v_rcp_iflag_f32_e32 v3, v2
659; GFX10-NEXT:  .LBB4_1: ; %bb3
660; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
661; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v4
662; GFX10-NEXT:    v_add_nc_u16 v4, v4, 1
663; GFX10-NEXT:    v_cvt_f32_u32_e32 v7, v0
664; GFX10-NEXT:    v_lshlrev_b64 v[5:6], 1, v[0:1]
665; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x400, v4
666; GFX10-NEXT:    v_mul_f32_e32 v0, v7, v3
667; GFX10-NEXT:    v_add_co_u32 v5, s0, s2, v5
668; GFX10-NEXT:    v_add_co_ci_u32_e64 v6, s0, s3, v6, s0
669; GFX10-NEXT:    v_trunc_f32_e32 v0, v0
670; GFX10-NEXT:    s_and_b32 vcc_lo, exec_lo, vcc_lo
671; GFX10-NEXT:    v_mad_f32 v7, -v0, v2, v7
672; GFX10-NEXT:    v_cvt_u32_f32_e32 v0, v0
673; GFX10-NEXT:    v_cmp_ge_f32_e64 s0, |v7|, v2
674; GFX10-NEXT:    v_add_co_ci_u32_e64 v0, s0, 0, v0, s0
675; GFX10-NEXT:    global_store_short v[5:6], v0, off
676; GFX10-NEXT:    s_cbranch_vccz .LBB4_1
677; GFX10-NEXT:  ; %bb.2: ; %bb2
678; GFX10-NEXT:    s_endpgm
679;
680; GFX11-LABEL: udiv16_invariant_denom:
681; GFX11:       ; %bb.0: ; %bb
682; GFX11-NEXT:    s_clause 0x1
683; GFX11-NEXT:    s_load_b32 s4, s[0:1], 0x2c
684; GFX11-NEXT:    s_load_b64 s[2:3], s[0:1], 0x24
685; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v4, 0
686; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
687; GFX11-NEXT:    s_and_b32 s0, 0xffff, s4
688; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
689; GFX11-NEXT:    v_cvt_f32_u32_e32 v2, s0
690; GFX11-NEXT:    v_rcp_iflag_f32_e32 v3, v2
691; GFX11-NEXT:    .p2align 6
692; GFX11-NEXT:  .LBB4_1: ; %bb3
693; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
694; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v4
695; GFX11-NEXT:    v_add_nc_u16 v4, v4, 1
696; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
697; GFX11-NEXT:    v_cvt_f32_u32_e32 v7, v0
698; GFX11-NEXT:    v_lshlrev_b64 v[5:6], 1, v[0:1]
699; GFX11-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x400, v4
700; GFX11-NEXT:    s_waitcnt_depctr 0xfff
701; GFX11-NEXT:    v_mul_f32_e32 v0, v7, v3
702; GFX11-NEXT:    v_add_co_u32 v5, s0, s2, v5
703; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
704; GFX11-NEXT:    v_add_co_ci_u32_e64 v6, s0, s3, v6, s0
705; GFX11-NEXT:    v_trunc_f32_e32 v0, v0
706; GFX11-NEXT:    s_and_b32 vcc_lo, exec_lo, vcc_lo
707; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
708; GFX11-NEXT:    v_fma_f32 v7, -v0, v2, v7
709; GFX11-NEXT:    v_cvt_u32_f32_e32 v0, v0
710; GFX11-NEXT:    v_cmp_ge_f32_e64 s0, |v7|, v2
711; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
712; GFX11-NEXT:    v_add_co_ci_u32_e64 v0, s0, 0, v0, s0
713; GFX11-NEXT:    global_store_b16 v[5:6], v0, off
714; GFX11-NEXT:    s_cbranch_vccz .LBB4_1
715; GFX11-NEXT:  ; %bb.2: ; %bb2
716; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
717; GFX11-NEXT:    s_endpgm
718bb:
719  br label %bb3
720
721bb2:                                              ; preds = %bb3
722  ret void
723
724bb3:                                              ; preds = %bb3, %bb
725  %tmp = phi i16 [ 0, %bb ], [ %tmp7, %bb3 ]
726  %tmp4 = udiv i16 %tmp, %arg1
727  %tmp5 = zext i16 %tmp to i64
728  %tmp6 = getelementptr inbounds i16, i16 addrspace(1)* %arg, i64 %tmp5
729  store i16 %tmp4, i16 addrspace(1)* %tmp6, align 2
730  %tmp7 = add nuw nsw i16 %tmp, 1
731  %tmp8 = icmp eq i16 %tmp7, 1024
732  br i1 %tmp8, label %bb2, label %bb3
733}
734
735define amdgpu_kernel void @urem16_invariant_denom(i16 addrspace(1)* nocapture %arg, i16 %arg1) {
736; GFX9-LABEL: urem16_invariant_denom:
737; GFX9:       ; %bb.0: ; %bb
738; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x2c
739; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
740; GFX9-NEXT:    v_mov_b32_e32 v1, 0
741; GFX9-NEXT:    s_movk_i32 s7, 0x400
742; GFX9-NEXT:    v_mov_b32_e32 v4, 0
743; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
744; GFX9-NEXT:    s_and_b32 s6, 0xffff, s2
745; GFX9-NEXT:    v_cvt_f32_u32_e32 v2, s6
746; GFX9-NEXT:    v_rcp_iflag_f32_e32 v3, v2
747; GFX9-NEXT:  .LBB5_1: ; %bb3
748; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
749; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v4
750; GFX9-NEXT:    v_cvt_f32_u32_e32 v8, v0
751; GFX9-NEXT:    v_lshlrev_b64 v[5:6], 1, v[0:1]
752; GFX9-NEXT:    v_add_u16_e32 v4, 1, v4
753; GFX9-NEXT:    v_mov_b32_e32 v7, s5
754; GFX9-NEXT:    v_mul_f32_e32 v9, v8, v3
755; GFX9-NEXT:    v_trunc_f32_e32 v9, v9
756; GFX9-NEXT:    v_cvt_u32_f32_e32 v10, v9
757; GFX9-NEXT:    v_mad_f32 v8, -v9, v2, v8
758; GFX9-NEXT:    v_cmp_ge_f32_e64 s[2:3], |v8|, v2
759; GFX9-NEXT:    v_cmp_eq_u16_e32 vcc, s7, v4
760; GFX9-NEXT:    v_addc_co_u32_e64 v8, s[2:3], 0, v10, s[2:3]
761; GFX9-NEXT:    v_mul_lo_u32 v8, v8, s6
762; GFX9-NEXT:    v_add_co_u32_e64 v5, s[0:1], s4, v5
763; GFX9-NEXT:    v_addc_co_u32_e64 v6, s[0:1], v7, v6, s[0:1]
764; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v8
765; GFX9-NEXT:    global_store_short v[5:6], v0, off
766; GFX9-NEXT:    s_cbranch_vccz .LBB5_1
767; GFX9-NEXT:  ; %bb.2: ; %bb2
768; GFX9-NEXT:    s_endpgm
769;
770; GFX10-LABEL: urem16_invariant_denom:
771; GFX10:       ; %bb.0: ; %bb
772; GFX10-NEXT:    s_clause 0x1
773; GFX10-NEXT:    s_load_dword s4, s[0:1], 0x2c
774; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
775; GFX10-NEXT:    v_mov_b32_e32 v1, 0
776; GFX10-NEXT:    v_mov_b32_e32 v4, 0
777; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
778; GFX10-NEXT:    s_and_b32 s1, 0xffff, s4
779; GFX10-NEXT:    v_cvt_f32_u32_e32 v2, s1
780; GFX10-NEXT:    v_rcp_iflag_f32_e32 v3, v2
781; GFX10-NEXT:  .LBB5_1: ; %bb3
782; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
783; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v4
784; GFX10-NEXT:    v_add_nc_u16 v4, v4, 1
785; GFX10-NEXT:    v_cvt_f32_u32_e32 v7, v0
786; GFX10-NEXT:    v_lshlrev_b64 v[5:6], 1, v[0:1]
787; GFX10-NEXT:    v_mul_f32_e32 v8, v7, v3
788; GFX10-NEXT:    v_add_co_u32 v5, s0, s2, v5
789; GFX10-NEXT:    v_add_co_ci_u32_e64 v6, s0, s3, v6, s0
790; GFX10-NEXT:    v_trunc_f32_e32 v8, v8
791; GFX10-NEXT:    v_mad_f32 v7, -v8, v2, v7
792; GFX10-NEXT:    v_cvt_u32_f32_e32 v8, v8
793; GFX10-NEXT:    v_cmp_ge_f32_e64 vcc_lo, |v7|, v2
794; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v8, vcc_lo
795; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x400, v4
796; GFX10-NEXT:    v_mul_lo_u32 v7, v7, s1
797; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v0, v7
798; GFX10-NEXT:    global_store_short v[5:6], v0, off
799; GFX10-NEXT:    s_cbranch_vccz .LBB5_1
800; GFX10-NEXT:  ; %bb.2: ; %bb2
801; GFX10-NEXT:    s_endpgm
802;
803; GFX11-LABEL: urem16_invariant_denom:
804; GFX11:       ; %bb.0: ; %bb
805; GFX11-NEXT:    s_clause 0x1
806; GFX11-NEXT:    s_load_b32 s4, s[0:1], 0x2c
807; GFX11-NEXT:    s_load_b64 s[2:3], s[0:1], 0x24
808; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v4, 0
809; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
810; GFX11-NEXT:    s_and_b32 s1, 0xffff, s4
811; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
812; GFX11-NEXT:    v_cvt_f32_u32_e32 v2, s1
813; GFX11-NEXT:    v_rcp_iflag_f32_e32 v3, v2
814; GFX11-NEXT:    s_set_inst_prefetch_distance 0x1
815; GFX11-NEXT:    .p2align 6
816; GFX11-NEXT:  .LBB5_1: ; %bb3
817; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
818; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v4
819; GFX11-NEXT:    v_add_nc_u16 v4, v4, 1
820; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_1)
821; GFX11-NEXT:    v_cvt_f32_u32_e32 v7, v0
822; GFX11-NEXT:    v_lshlrev_b64 v[5:6], 1, v[0:1]
823; GFX11-NEXT:    s_waitcnt_depctr 0xfff
824; GFX11-NEXT:    v_mul_f32_e32 v8, v7, v3
825; GFX11-NEXT:    v_add_co_u32 v5, s0, s2, v5
826; GFX11-NEXT:    v_add_co_ci_u32_e64 v6, s0, s3, v6, s0
827; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
828; GFX11-NEXT:    v_trunc_f32_e32 v8, v8
829; GFX11-NEXT:    v_fma_f32 v7, -v8, v2, v7
830; GFX11-NEXT:    v_cvt_u32_f32_e32 v8, v8
831; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
832; GFX11-NEXT:    v_cmp_ge_f32_e64 vcc_lo, |v7|, v2
833; GFX11-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v8, vcc_lo
834; GFX11-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x400, v4
835; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
836; GFX11-NEXT:    v_mul_lo_u32 v7, v7, s1
837; GFX11-NEXT:    v_sub_nc_u32_e32 v0, v0, v7
838; GFX11-NEXT:    global_store_b16 v[5:6], v0, off
839; GFX11-NEXT:    s_cbranch_vccz .LBB5_1
840; GFX11-NEXT:  ; %bb.2: ; %bb2
841; GFX11-NEXT:    s_set_inst_prefetch_distance 0x2
842; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
843; GFX11-NEXT:    s_endpgm
844bb:
845  br label %bb3
846
847bb2:                                              ; preds = %bb3
848  ret void
849
850bb3:                                              ; preds = %bb3, %bb
851  %tmp = phi i16 [ 0, %bb ], [ %tmp7, %bb3 ]
852  %tmp4 = urem i16 %tmp, %arg1
853  %tmp5 = zext i16 %tmp to i64
854  %tmp6 = getelementptr inbounds i16, i16 addrspace(1)* %arg, i64 %tmp5
855  store i16 %tmp4, i16 addrspace(1)* %tmp6, align 2
856  %tmp7 = add nuw nsw i16 %tmp, 1
857  %tmp8 = icmp eq i16 %tmp7, 1024
858  br i1 %tmp8, label %bb2, label %bb3
859}
860
861define amdgpu_kernel void @sdiv16_invariant_denom(i16 addrspace(1)* nocapture %arg, i16 %arg1) {
862; GFX9-LABEL: sdiv16_invariant_denom:
863; GFX9:       ; %bb.0: ; %bb
864; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x2c
865; GFX9-NEXT:    v_mov_b32_e32 v1, 0
866; GFX9-NEXT:    s_movk_i32 s5, 0x400
867; GFX9-NEXT:    v_mov_b32_e32 v4, 0
868; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
869; GFX9-NEXT:    s_sext_i32_i16 s4, s2
870; GFX9-NEXT:    v_cvt_f32_i32_e32 v2, s4
871; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
872; GFX9-NEXT:    v_rcp_iflag_f32_e32 v3, v2
873; GFX9-NEXT:  .LBB6_1: ; %bb3
874; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
875; GFX9-NEXT:    v_bfe_i32 v5, v4, 0, 16
876; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v4
877; GFX9-NEXT:    v_cvt_f32_i32_e32 v9, v5
878; GFX9-NEXT:    v_xor_b32_e32 v8, s4, v5
879; GFX9-NEXT:    v_lshlrev_b64 v[5:6], 1, v[0:1]
880; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
881; GFX9-NEXT:    v_mov_b32_e32 v7, s3
882; GFX9-NEXT:    v_add_co_u32_e64 v5, s[0:1], s2, v5
883; GFX9-NEXT:    v_addc_co_u32_e64 v6, s[0:1], v7, v6, s[0:1]
884; GFX9-NEXT:    v_mul_f32_e32 v7, v9, v3
885; GFX9-NEXT:    v_trunc_f32_e32 v7, v7
886; GFX9-NEXT:    v_ashrrev_i32_e32 v0, 30, v8
887; GFX9-NEXT:    v_cvt_i32_f32_e32 v8, v7
888; GFX9-NEXT:    v_mad_f32 v7, -v7, v2, v9
889; GFX9-NEXT:    v_add_u16_e32 v4, 1, v4
890; GFX9-NEXT:    v_or_b32_e32 v0, 1, v0
891; GFX9-NEXT:    v_cmp_ge_f32_e64 s[0:1], |v7|, |v2|
892; GFX9-NEXT:    v_cmp_eq_u16_e32 vcc, s5, v4
893; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, v0, s[0:1]
894; GFX9-NEXT:    v_add_u32_e32 v0, v8, v0
895; GFX9-NEXT:    global_store_short v[5:6], v0, off
896; GFX9-NEXT:    s_cbranch_vccz .LBB6_1
897; GFX9-NEXT:  ; %bb.2: ; %bb2
898; GFX9-NEXT:    s_endpgm
899;
900; GFX10-LABEL: sdiv16_invariant_denom:
901; GFX10:       ; %bb.0: ; %bb
902; GFX10-NEXT:    s_clause 0x1
903; GFX10-NEXT:    s_load_dword s4, s[0:1], 0x2c
904; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
905; GFX10-NEXT:    v_mov_b32_e32 v1, 0
906; GFX10-NEXT:    v_mov_b32_e32 v4, 0
907; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
908; GFX10-NEXT:    s_sext_i32_i16 s4, s4
909; GFX10-NEXT:    v_cvt_f32_i32_e32 v2, s4
910; GFX10-NEXT:    v_rcp_iflag_f32_e32 v3, v2
911; GFX10-NEXT:  .LBB6_1: ; %bb3
912; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
913; GFX10-NEXT:    v_bfe_i32 v5, v4, 0, 16
914; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v4
915; GFX10-NEXT:    v_add_nc_u16 v4, v4, 1
916; GFX10-NEXT:    v_cvt_f32_i32_e32 v7, v5
917; GFX10-NEXT:    v_xor_b32_e32 v8, s4, v5
918; GFX10-NEXT:    v_lshlrev_b64 v[5:6], 1, v[0:1]
919; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x400, v4
920; GFX10-NEXT:    v_mul_f32_e32 v0, v7, v3
921; GFX10-NEXT:    v_ashrrev_i32_e32 v8, 30, v8
922; GFX10-NEXT:    v_add_co_u32 v5, s0, s2, v5
923; GFX10-NEXT:    v_trunc_f32_e32 v0, v0
924; GFX10-NEXT:    v_or_b32_e32 v8, 1, v8
925; GFX10-NEXT:    v_add_co_ci_u32_e64 v6, s0, s3, v6, s0
926; GFX10-NEXT:    s_and_b32 vcc_lo, exec_lo, vcc_lo
927; GFX10-NEXT:    v_mad_f32 v7, -v0, v2, v7
928; GFX10-NEXT:    v_cvt_i32_f32_e32 v0, v0
929; GFX10-NEXT:    v_cmp_ge_f32_e64 s1, |v7|, |v2|
930; GFX10-NEXT:    v_cndmask_b32_e64 v7, 0, v8, s1
931; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v7
932; GFX10-NEXT:    global_store_short v[5:6], v0, off
933; GFX10-NEXT:    s_cbranch_vccz .LBB6_1
934; GFX10-NEXT:  ; %bb.2: ; %bb2
935; GFX10-NEXT:    s_endpgm
936;
937; GFX11-LABEL: sdiv16_invariant_denom:
938; GFX11:       ; %bb.0: ; %bb
939; GFX11-NEXT:    s_clause 0x1
940; GFX11-NEXT:    s_load_b32 s4, s[0:1], 0x2c
941; GFX11-NEXT:    s_load_b64 s[2:3], s[0:1], 0x24
942; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v4, 0
943; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
944; GFX11-NEXT:    s_sext_i32_i16 s4, s4
945; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
946; GFX11-NEXT:    v_cvt_f32_i32_e32 v2, s4
947; GFX11-NEXT:    v_rcp_iflag_f32_e32 v3, v2
948; GFX11-NEXT:    s_set_inst_prefetch_distance 0x1
949; GFX11-NEXT:    .p2align 6
950; GFX11-NEXT:  .LBB6_1: ; %bb3
951; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
952; GFX11-NEXT:    v_bfe_i32 v5, v4, 0, 16
953; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v4
954; GFX11-NEXT:    v_add_nc_u16 v4, v4, 1
955; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
956; GFX11-NEXT:    v_cvt_f32_i32_e32 v7, v5
957; GFX11-NEXT:    v_xor_b32_e32 v8, s4, v5
958; GFX11-NEXT:    v_lshlrev_b64 v[5:6], 1, v[0:1]
959; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
960; GFX11-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x400, v4
961; GFX11-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instid1(VALU_DEP_4)
962; GFX11-NEXT:    v_mul_f32_e32 v0, v7, v3
963; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
964; GFX11-NEXT:    v_ashrrev_i32_e32 v8, 30, v8
965; GFX11-NEXT:    v_add_co_u32 v5, s0, s2, v5
966; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
967; GFX11-NEXT:    v_trunc_f32_e32 v0, v0
968; GFX11-NEXT:    v_or_b32_e32 v8, 1, v8
969; GFX11-NEXT:    v_add_co_ci_u32_e64 v6, s0, s3, v6, s0
970; GFX11-NEXT:    s_and_b32 vcc_lo, exec_lo, vcc_lo
971; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
972; GFX11-NEXT:    v_fma_f32 v7, -v0, v2, v7
973; GFX11-NEXT:    v_cvt_i32_f32_e32 v0, v0
974; GFX11-NEXT:    v_cmp_ge_f32_e64 s1, |v7|, |v2|
975; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
976; GFX11-NEXT:    v_cndmask_b32_e64 v7, 0, v8, s1
977; GFX11-NEXT:    v_add_nc_u32_e32 v0, v0, v7
978; GFX11-NEXT:    global_store_b16 v[5:6], v0, off
979; GFX11-NEXT:    s_cbranch_vccz .LBB6_1
980; GFX11-NEXT:  ; %bb.2: ; %bb2
981; GFX11-NEXT:    s_set_inst_prefetch_distance 0x2
982; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
983; GFX11-NEXT:    s_endpgm
984bb:
985  br label %bb3
986
987bb2:                                              ; preds = %bb3
988  ret void
989
990bb3:                                              ; preds = %bb3, %bb
991  %tmp = phi i16 [ 0, %bb ], [ %tmp7, %bb3 ]
992  %tmp4 = sdiv i16 %tmp, %arg1
993  %tmp5 = zext i16 %tmp to i64
994  %tmp6 = getelementptr inbounds i16, i16 addrspace(1)* %arg, i64 %tmp5
995  store i16 %tmp4, i16 addrspace(1)* %tmp6, align 2
996  %tmp7 = add nuw nsw i16 %tmp, 1
997  %tmp8 = icmp eq i16 %tmp7, 1024
998  br i1 %tmp8, label %bb2, label %bb3
999}
1000
1001define amdgpu_kernel void @srem16_invariant_denom(i16 addrspace(1)* nocapture %arg, i16 %arg1) {
1002; GFX9-LABEL: srem16_invariant_denom:
1003; GFX9:       ; %bb.0: ; %bb
1004; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x2c
1005; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
1006; GFX9-NEXT:    v_mov_b32_e32 v1, 0
1007; GFX9-NEXT:    s_movk_i32 s7, 0x400
1008; GFX9-NEXT:    v_mov_b32_e32 v4, 0
1009; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
1010; GFX9-NEXT:    s_sext_i32_i16 s6, s2
1011; GFX9-NEXT:    v_cvt_f32_i32_e32 v2, s6
1012; GFX9-NEXT:    v_rcp_iflag_f32_e32 v3, v2
1013; GFX9-NEXT:  .LBB7_1: ; %bb3
1014; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
1015; GFX9-NEXT:    v_bfe_i32 v7, v4, 0, 16
1016; GFX9-NEXT:    v_cvt_f32_i32_e32 v10, v7
1017; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v4
1018; GFX9-NEXT:    v_xor_b32_e32 v9, s6, v7
1019; GFX9-NEXT:    v_lshlrev_b64 v[5:6], 1, v[0:1]
1020; GFX9-NEXT:    v_ashrrev_i32_e32 v0, 30, v9
1021; GFX9-NEXT:    v_mul_f32_e32 v9, v10, v3
1022; GFX9-NEXT:    v_trunc_f32_e32 v9, v9
1023; GFX9-NEXT:    v_cvt_i32_f32_e32 v11, v9
1024; GFX9-NEXT:    v_mad_f32 v9, -v9, v2, v10
1025; GFX9-NEXT:    v_or_b32_e32 v0, 1, v0
1026; GFX9-NEXT:    v_cmp_ge_f32_e64 s[2:3], |v9|, |v2|
1027; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, v0, s[2:3]
1028; GFX9-NEXT:    v_add_u32_e32 v0, v11, v0
1029; GFX9-NEXT:    v_mul_lo_u32 v0, v0, s6
1030; GFX9-NEXT:    v_add_u16_e32 v4, 1, v4
1031; GFX9-NEXT:    v_mov_b32_e32 v8, s5
1032; GFX9-NEXT:    v_cmp_eq_u16_e32 vcc, s7, v4
1033; GFX9-NEXT:    v_add_co_u32_e64 v5, s[0:1], s4, v5
1034; GFX9-NEXT:    v_addc_co_u32_e64 v6, s[0:1], v8, v6, s[0:1]
1035; GFX9-NEXT:    v_sub_u32_e32 v0, v7, v0
1036; GFX9-NEXT:    global_store_short v[5:6], v0, off
1037; GFX9-NEXT:    s_cbranch_vccz .LBB7_1
1038; GFX9-NEXT:  ; %bb.2: ; %bb2
1039; GFX9-NEXT:    s_endpgm
1040;
1041; GFX10-LABEL: srem16_invariant_denom:
1042; GFX10:       ; %bb.0: ; %bb
1043; GFX10-NEXT:    s_clause 0x1
1044; GFX10-NEXT:    s_load_dword s4, s[0:1], 0x2c
1045; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
1046; GFX10-NEXT:    v_mov_b32_e32 v1, 0
1047; GFX10-NEXT:    v_mov_b32_e32 v4, 0
1048; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
1049; GFX10-NEXT:    s_sext_i32_i16 s1, s4
1050; GFX10-NEXT:    v_cvt_f32_i32_e32 v2, s1
1051; GFX10-NEXT:    v_rcp_iflag_f32_e32 v3, v2
1052; GFX10-NEXT:  .LBB7_1: ; %bb3
1053; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
1054; GFX10-NEXT:    v_bfe_i32 v7, v4, 0, 16
1055; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v4
1056; GFX10-NEXT:    v_add_nc_u16 v4, v4, 1
1057; GFX10-NEXT:    v_cvt_f32_i32_e32 v5, v7
1058; GFX10-NEXT:    v_xor_b32_e32 v6, s1, v7
1059; GFX10-NEXT:    v_mul_f32_e32 v8, v5, v3
1060; GFX10-NEXT:    v_ashrrev_i32_e32 v6, 30, v6
1061; GFX10-NEXT:    v_trunc_f32_e32 v8, v8
1062; GFX10-NEXT:    v_or_b32_e32 v6, 1, v6
1063; GFX10-NEXT:    v_mad_f32 v5, -v8, v2, v5
1064; GFX10-NEXT:    v_cvt_i32_f32_e32 v8, v8
1065; GFX10-NEXT:    v_cmp_ge_f32_e64 vcc_lo, |v5|, |v2|
1066; GFX10-NEXT:    v_cndmask_b32_e32 v9, 0, v6, vcc_lo
1067; GFX10-NEXT:    v_lshlrev_b64 v[5:6], 1, v[0:1]
1068; GFX10-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x400, v4
1069; GFX10-NEXT:    v_add_nc_u32_e32 v0, v8, v9
1070; GFX10-NEXT:    v_add_co_u32 v5, s0, s2, v5
1071; GFX10-NEXT:    v_add_co_ci_u32_e64 v6, s0, s3, v6, s0
1072; GFX10-NEXT:    v_mul_lo_u32 v0, v0, s1
1073; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v7, v0
1074; GFX10-NEXT:    global_store_short v[5:6], v0, off
1075; GFX10-NEXT:    s_cbranch_vccz .LBB7_1
1076; GFX10-NEXT:  ; %bb.2: ; %bb2
1077; GFX10-NEXT:    s_endpgm
1078;
1079; GFX11-LABEL: srem16_invariant_denom:
1080; GFX11:       ; %bb.0: ; %bb
1081; GFX11-NEXT:    s_clause 0x1
1082; GFX11-NEXT:    s_load_b32 s4, s[0:1], 0x2c
1083; GFX11-NEXT:    s_load_b64 s[2:3], s[0:1], 0x24
1084; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v4, 0
1085; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
1086; GFX11-NEXT:    s_sext_i32_i16 s1, s4
1087; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
1088; GFX11-NEXT:    v_cvt_f32_i32_e32 v2, s1
1089; GFX11-NEXT:    v_rcp_iflag_f32_e32 v3, v2
1090; GFX11-NEXT:    s_set_inst_prefetch_distance 0x1
1091; GFX11-NEXT:    .p2align 6
1092; GFX11-NEXT:  .LBB7_1: ; %bb3
1093; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
1094; GFX11-NEXT:    v_bfe_i32 v7, v4, 0, 16
1095; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
1096; GFX11-NEXT:    v_cvt_f32_i32_e32 v5, v7
1097; GFX11-NEXT:    v_xor_b32_e32 v6, s1, v7
1098; GFX11-NEXT:    s_waitcnt_depctr 0xfff
1099; GFX11-NEXT:    v_mul_f32_e32 v8, v5, v3
1100; GFX11-NEXT:    v_ashrrev_i32_e32 v6, 30, v6
1101; GFX11-NEXT:    v_trunc_f32_e32 v8, v8
1102; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
1103; GFX11-NEXT:    v_or_b32_e32 v6, 1, v6
1104; GFX11-NEXT:    v_fma_f32 v5, -v8, v2, v5
1105; GFX11-NEXT:    v_cvt_i32_f32_e32 v8, v8
1106; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
1107; GFX11-NEXT:    v_cmp_ge_f32_e64 vcc_lo, |v5|, |v2|
1108; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v4
1109; GFX11-NEXT:    v_add_nc_u16 v4, v4, 1
1110; GFX11-NEXT:    v_cndmask_b32_e32 v9, 0, v6, vcc_lo
1111; GFX11-NEXT:    v_lshlrev_b64 v[5:6], 1, v[0:1]
1112; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
1113; GFX11-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x400, v4
1114; GFX11-NEXT:    v_add_nc_u32_e32 v0, v8, v9
1115; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
1116; GFX11-NEXT:    v_add_co_u32 v5, s0, s2, v5
1117; GFX11-NEXT:    v_add_co_ci_u32_e64 v6, s0, s3, v6, s0
1118; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
1119; GFX11-NEXT:    v_mul_lo_u32 v0, v0, s1
1120; GFX11-NEXT:    v_sub_nc_u32_e32 v0, v7, v0
1121; GFX11-NEXT:    global_store_b16 v[5:6], v0, off
1122; GFX11-NEXT:    s_cbranch_vccz .LBB7_1
1123; GFX11-NEXT:  ; %bb.2: ; %bb2
1124; GFX11-NEXT:    s_set_inst_prefetch_distance 0x2
1125; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1126; GFX11-NEXT:    s_endpgm
1127bb:
1128  br label %bb3
1129
1130bb2:                                              ; preds = %bb3
1131  ret void
1132
1133bb3:                                              ; preds = %bb3, %bb
1134  %tmp = phi i16 [ 0, %bb ], [ %tmp7, %bb3 ]
1135  %tmp4 = srem i16 %tmp, %arg1
1136  %tmp5 = zext i16 %tmp to i64
1137  %tmp6 = getelementptr inbounds i16, i16 addrspace(1)* %arg, i64 %tmp5
1138  store i16 %tmp4, i16 addrspace(1)* %tmp6, align 2
1139  %tmp7 = add nuw nsw i16 %tmp, 1
1140  %tmp8 = icmp eq i16 %tmp7, 1024
1141  br i1 %tmp8, label %bb2, label %bb3
1142}
1143