1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -global-isel -march=amdgcn -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX7 %s
3; RUN: llc -global-isel -march=amdgcn -mcpu=gfx801 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX8 %s
4; RUN: llc -global-isel -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9 %s
5; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10 %s
6
7define amdgpu_ps i16 @s_mul_i16(i16 inreg %num, i16 inreg %den) {
8; GFX7-LABEL: s_mul_i16:
9; GFX7:       ; %bb.0:
10; GFX7-NEXT:    s_mul_i32 s0, s0, s1
11; GFX7-NEXT:    ; return to shader part epilog
12;
13; GFX8-LABEL: s_mul_i16:
14; GFX8:       ; %bb.0:
15; GFX8-NEXT:    s_and_b32 s0, s0, 0xffff
16; GFX8-NEXT:    s_and_b32 s1, s1, 0xffff
17; GFX8-NEXT:    s_mul_i32 s0, s0, s1
18; GFX8-NEXT:    ; return to shader part epilog
19;
20; GFX9-LABEL: s_mul_i16:
21; GFX9:       ; %bb.0:
22; GFX9-NEXT:    s_and_b32 s0, s0, 0xffff
23; GFX9-NEXT:    s_and_b32 s1, s1, 0xffff
24; GFX9-NEXT:    s_mul_i32 s0, s0, s1
25; GFX9-NEXT:    ; return to shader part epilog
26;
27; GFX10-LABEL: s_mul_i16:
28; GFX10:       ; %bb.0:
29; GFX10-NEXT:    s_and_b32 s0, s0, 0xffff
30; GFX10-NEXT:    s_and_b32 s1, s1, 0xffff
31; GFX10-NEXT:    s_mul_i32 s0, s0, s1
32; GFX10-NEXT:    ; return to shader part epilog
33  %result = mul i16 %num, %den
34  ret i16 %result
35}
36
37define i16 @v_mul_i16(i16 %num, i16 %den) {
38; GFX7-LABEL: v_mul_i16:
39; GFX7:       ; %bb.0:
40; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
41; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff, v0
42; GFX7-NEXT:    v_and_b32_e32 v1, 0xffff, v1
43; GFX7-NEXT:    v_mul_u32_u24_e32 v0, v0, v1
44; GFX7-NEXT:    s_setpc_b64 s[30:31]
45;
46; GFX8-LABEL: v_mul_i16:
47; GFX8:       ; %bb.0:
48; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
49; GFX8-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
50; GFX8-NEXT:    s_setpc_b64 s[30:31]
51;
52; GFX9-LABEL: v_mul_i16:
53; GFX9:       ; %bb.0:
54; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
55; GFX9-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
56; GFX9-NEXT:    s_setpc_b64 s[30:31]
57;
58; GFX10-LABEL: v_mul_i16:
59; GFX10:       ; %bb.0:
60; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
61; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
62; GFX10-NEXT:    v_mul_lo_u16 v0, v0, v1
63; GFX10-NEXT:    s_setpc_b64 s[30:31]
64  %result = mul i16 %num, %den
65  ret i16 %result
66}
67
68define amdgpu_ps zeroext i16 @s_mul_i16_zeroext(i16 inreg zeroext %num, i16 inreg zeroext %den) {
69; GFX7-LABEL: s_mul_i16_zeroext:
70; GFX7:       ; %bb.0:
71; GFX7-NEXT:    s_mul_i32 s0, s0, s1
72; GFX7-NEXT:    s_and_b32 s0, s0, 0xffff
73; GFX7-NEXT:    ; return to shader part epilog
74;
75; GFX8-LABEL: s_mul_i16_zeroext:
76; GFX8:       ; %bb.0:
77; GFX8-NEXT:    s_and_b32 s0, s0, 0xffff
78; GFX8-NEXT:    s_and_b32 s1, s1, 0xffff
79; GFX8-NEXT:    s_mul_i32 s0, s0, s1
80; GFX8-NEXT:    s_and_b32 s0, s0, 0xffff
81; GFX8-NEXT:    ; return to shader part epilog
82;
83; GFX9-LABEL: s_mul_i16_zeroext:
84; GFX9:       ; %bb.0:
85; GFX9-NEXT:    s_and_b32 s0, s0, 0xffff
86; GFX9-NEXT:    s_and_b32 s1, s1, 0xffff
87; GFX9-NEXT:    s_mul_i32 s0, s0, s1
88; GFX9-NEXT:    s_and_b32 s0, s0, 0xffff
89; GFX9-NEXT:    ; return to shader part epilog
90;
91; GFX10-LABEL: s_mul_i16_zeroext:
92; GFX10:       ; %bb.0:
93; GFX10-NEXT:    s_and_b32 s0, s0, 0xffff
94; GFX10-NEXT:    s_and_b32 s1, s1, 0xffff
95; GFX10-NEXT:    s_mul_i32 s0, s0, s1
96; GFX10-NEXT:    s_and_b32 s0, s0, 0xffff
97; GFX10-NEXT:    ; return to shader part epilog
98  %result = mul i16 %num, %den
99  ret i16 %result
100}
101
102define zeroext i16 @v_mul_i16_zeroext(i16 zeroext %num, i16 zeroext %den) {
103; GFX7-LABEL: v_mul_i16_zeroext:
104; GFX7:       ; %bb.0:
105; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
106; GFX7-NEXT:    v_mul_u32_u24_e32 v0, v0, v1
107; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff, v0
108; GFX7-NEXT:    s_setpc_b64 s[30:31]
109;
110; GFX8-LABEL: v_mul_i16_zeroext:
111; GFX8:       ; %bb.0:
112; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
113; GFX8-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
114; GFX8-NEXT:    s_setpc_b64 s[30:31]
115;
116; GFX9-LABEL: v_mul_i16_zeroext:
117; GFX9:       ; %bb.0:
118; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
119; GFX9-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
120; GFX9-NEXT:    s_setpc_b64 s[30:31]
121;
122; GFX10-LABEL: v_mul_i16_zeroext:
123; GFX10:       ; %bb.0:
124; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
125; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
126; GFX10-NEXT:    v_mul_lo_u16 v0, v0, v1
127; GFX10-NEXT:    v_bfe_u32 v0, v0, 0, 16
128; GFX10-NEXT:    s_setpc_b64 s[30:31]
129  %result = mul i16 %num, %den
130  ret i16 %result
131}
132
133define amdgpu_ps signext i16 @s_mul_i16_signext(i16 inreg signext %num, i16 inreg signext %den) {
134; GFX7-LABEL: s_mul_i16_signext:
135; GFX7:       ; %bb.0:
136; GFX7-NEXT:    s_mul_i32 s0, s0, s1
137; GFX7-NEXT:    s_sext_i32_i16 s0, s0
138; GFX7-NEXT:    ; return to shader part epilog
139;
140; GFX8-LABEL: s_mul_i16_signext:
141; GFX8:       ; %bb.0:
142; GFX8-NEXT:    s_and_b32 s0, s0, 0xffff
143; GFX8-NEXT:    s_and_b32 s1, s1, 0xffff
144; GFX8-NEXT:    s_mul_i32 s0, s0, s1
145; GFX8-NEXT:    s_sext_i32_i16 s0, s0
146; GFX8-NEXT:    ; return to shader part epilog
147;
148; GFX9-LABEL: s_mul_i16_signext:
149; GFX9:       ; %bb.0:
150; GFX9-NEXT:    s_and_b32 s0, s0, 0xffff
151; GFX9-NEXT:    s_and_b32 s1, s1, 0xffff
152; GFX9-NEXT:    s_mul_i32 s0, s0, s1
153; GFX9-NEXT:    s_sext_i32_i16 s0, s0
154; GFX9-NEXT:    ; return to shader part epilog
155;
156; GFX10-LABEL: s_mul_i16_signext:
157; GFX10:       ; %bb.0:
158; GFX10-NEXT:    s_and_b32 s0, s0, 0xffff
159; GFX10-NEXT:    s_and_b32 s1, s1, 0xffff
160; GFX10-NEXT:    s_mul_i32 s0, s0, s1
161; GFX10-NEXT:    s_sext_i32_i16 s0, s0
162; GFX10-NEXT:    ; return to shader part epilog
163  %result = mul i16 %num, %den
164  ret i16 %result
165}
166
167define signext i16 @v_mul_i16_signext(i16 signext %num, i16 signext %den) {
168; GFX7-LABEL: v_mul_i16_signext:
169; GFX7:       ; %bb.0:
170; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
171; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff, v0
172; GFX7-NEXT:    v_and_b32_e32 v1, 0xffff, v1
173; GFX7-NEXT:    v_mul_u32_u24_e32 v0, v0, v1
174; GFX7-NEXT:    v_bfe_i32 v0, v0, 0, 16
175; GFX7-NEXT:    s_setpc_b64 s[30:31]
176;
177; GFX8-LABEL: v_mul_i16_signext:
178; GFX8:       ; %bb.0:
179; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
180; GFX8-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
181; GFX8-NEXT:    v_bfe_i32 v0, v0, 0, 16
182; GFX8-NEXT:    s_setpc_b64 s[30:31]
183;
184; GFX9-LABEL: v_mul_i16_signext:
185; GFX9:       ; %bb.0:
186; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
187; GFX9-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
188; GFX9-NEXT:    v_bfe_i32 v0, v0, 0, 16
189; GFX9-NEXT:    s_setpc_b64 s[30:31]
190;
191; GFX10-LABEL: v_mul_i16_signext:
192; GFX10:       ; %bb.0:
193; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
194; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
195; GFX10-NEXT:    v_mul_lo_u16 v0, v0, v1
196; GFX10-NEXT:    v_bfe_i32 v0, v0, 0, 16
197; GFX10-NEXT:    s_setpc_b64 s[30:31]
198  %result = mul i16 %num, %den
199  ret i16 %result
200}
201
202define amdgpu_ps i32 @s_mul_i32(i32 inreg %num, i32 inreg %den) {
203; GCN-LABEL: s_mul_i32:
204; GCN:       ; %bb.0:
205; GCN-NEXT:    s_mul_i32 s0, s0, s1
206; GCN-NEXT:    ; return to shader part epilog
207;
208; GFX10-LABEL: s_mul_i32:
209; GFX10:       ; %bb.0:
210; GFX10-NEXT:    s_mul_i32 s0, s0, s1
211; GFX10-NEXT:    ; return to shader part epilog
212  %result = mul i32 %num, %den
213  ret i32 %result
214}
215
216define i32 @v_mul_i32(i32 %num, i32 %den) {
217; GCN-LABEL: v_mul_i32:
218; GCN:       ; %bb.0:
219; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
220; GCN-NEXT:    v_mul_lo_u32 v0, v0, v1
221; GCN-NEXT:    s_setpc_b64 s[30:31]
222;
223; GFX10-LABEL: v_mul_i32:
224; GFX10:       ; %bb.0:
225; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
226; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
227; GFX10-NEXT:    v_mul_lo_u32 v0, v0, v1
228; GFX10-NEXT:    s_setpc_b64 s[30:31]
229  %result = mul i32 %num, %den
230  ret i32 %result
231}
232
233define amdgpu_ps <2 x i32> @s_mul_v2i32(<2 x i32> inreg %num, <2 x i32> inreg %den) {
234; GCN-LABEL: s_mul_v2i32:
235; GCN:       ; %bb.0:
236; GCN-NEXT:    s_mul_i32 s0, s0, s2
237; GCN-NEXT:    s_mul_i32 s1, s1, s3
238; GCN-NEXT:    ; return to shader part epilog
239;
240; GFX10-LABEL: s_mul_v2i32:
241; GFX10:       ; %bb.0:
242; GFX10-NEXT:    s_mul_i32 s0, s0, s2
243; GFX10-NEXT:    s_mul_i32 s1, s1, s3
244; GFX10-NEXT:    ; return to shader part epilog
245  %result = mul <2 x i32> %num, %den
246  ret <2 x i32> %result
247}
248
249define <2 x i32> @v_mul_v2i32(<2 x i32> %num, <2 x i32> %den) {
250; GCN-LABEL: v_mul_v2i32:
251; GCN:       ; %bb.0:
252; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
253; GCN-NEXT:    v_mul_lo_u32 v0, v0, v2
254; GCN-NEXT:    v_mul_lo_u32 v1, v1, v3
255; GCN-NEXT:    s_setpc_b64 s[30:31]
256;
257; GFX10-LABEL: v_mul_v2i32:
258; GFX10:       ; %bb.0:
259; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
260; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
261; GFX10-NEXT:    v_mul_lo_u32 v0, v0, v2
262; GFX10-NEXT:    v_mul_lo_u32 v1, v1, v3
263; GFX10-NEXT:    s_setpc_b64 s[30:31]
264  %result = mul <2 x i32> %num, %den
265  ret <2 x i32> %result
266}
267
268define amdgpu_cs i33 @s_mul_i33(i33 inreg %num,  i33 inreg %den) {
269; GFX7-LABEL: s_mul_i33:
270; GFX7:       ; %bb.0:
271; GFX7-NEXT:    v_mov_b32_e32 v0, s2
272; GFX7-NEXT:    v_mul_hi_u32 v0, s0, v0
273; GFX7-NEXT:    s_mul_i32 s4, s0, s2
274; GFX7-NEXT:    s_mul_i32 s0, s0, s3
275; GFX7-NEXT:    s_mul_i32 s1, s1, s2
276; GFX7-NEXT:    v_readfirstlane_b32 s5, v0
277; GFX7-NEXT:    s_add_u32 s0, s0, s5
278; GFX7-NEXT:    s_add_u32 s1, s1, s0
279; GFX7-NEXT:    s_mov_b32 s0, s4
280; GFX7-NEXT:    ; return to shader part epilog
281;
282; GFX8-LABEL: s_mul_i33:
283; GFX8:       ; %bb.0:
284; GFX8-NEXT:    v_mov_b32_e32 v0, s2
285; GFX8-NEXT:    v_mul_hi_u32 v0, s0, v0
286; GFX8-NEXT:    s_mul_i32 s4, s0, s2
287; GFX8-NEXT:    s_mul_i32 s0, s0, s3
288; GFX8-NEXT:    s_mul_i32 s1, s1, s2
289; GFX8-NEXT:    v_readfirstlane_b32 s5, v0
290; GFX8-NEXT:    s_add_u32 s0, s0, s5
291; GFX8-NEXT:    s_add_u32 s1, s1, s0
292; GFX8-NEXT:    s_mov_b32 s0, s4
293; GFX8-NEXT:    ; return to shader part epilog
294;
295; GFX9-LABEL: s_mul_i33:
296; GFX9:       ; %bb.0:
297; GFX9-NEXT:    s_mul_i32 s4, s0, s2
298; GFX9-NEXT:    s_mul_hi_u32 s5, s0, s2
299; GFX9-NEXT:    s_mul_i32 s0, s0, s3
300; GFX9-NEXT:    s_add_u32 s0, s0, s5
301; GFX9-NEXT:    s_mul_i32 s1, s1, s2
302; GFX9-NEXT:    s_add_u32 s1, s1, s0
303; GFX9-NEXT:    s_mov_b32 s0, s4
304; GFX9-NEXT:    ; return to shader part epilog
305;
306; GFX10-LABEL: s_mul_i33:
307; GFX10:       ; %bb.0:
308; GFX10-NEXT:    s_mul_hi_u32 s4, s0, s2
309; GFX10-NEXT:    s_mul_i32 s3, s0, s3
310; GFX10-NEXT:    s_mul_i32 s1, s1, s2
311; GFX10-NEXT:    s_add_i32 s3, s4, s3
312; GFX10-NEXT:    s_mul_i32 s0, s0, s2
313; GFX10-NEXT:    s_add_i32 s1, s3, s1
314; GFX10-NEXT:    ; return to shader part epilog
315  %result = mul i33 %num, %den
316  ret i33 %result
317}
318
319define amdgpu_ps i64 @s_mul_i64(i64 inreg %num, i64 inreg %den) {
320; GFX7-LABEL: s_mul_i64:
321; GFX7:       ; %bb.0:
322; GFX7-NEXT:    v_mov_b32_e32 v0, s2
323; GFX7-NEXT:    v_mul_hi_u32 v0, s0, v0
324; GFX7-NEXT:    s_mul_i32 s4, s0, s2
325; GFX7-NEXT:    s_mul_i32 s0, s0, s3
326; GFX7-NEXT:    s_mul_i32 s1, s1, s2
327; GFX7-NEXT:    v_readfirstlane_b32 s5, v0
328; GFX7-NEXT:    s_add_u32 s0, s0, s5
329; GFX7-NEXT:    s_add_u32 s1, s1, s0
330; GFX7-NEXT:    s_mov_b32 s0, s4
331; GFX7-NEXT:    ; return to shader part epilog
332;
333; GFX8-LABEL: s_mul_i64:
334; GFX8:       ; %bb.0:
335; GFX8-NEXT:    v_mov_b32_e32 v0, s2
336; GFX8-NEXT:    v_mul_hi_u32 v0, s0, v0
337; GFX8-NEXT:    s_mul_i32 s4, s0, s2
338; GFX8-NEXT:    s_mul_i32 s0, s0, s3
339; GFX8-NEXT:    s_mul_i32 s1, s1, s2
340; GFX8-NEXT:    v_readfirstlane_b32 s5, v0
341; GFX8-NEXT:    s_add_u32 s0, s0, s5
342; GFX8-NEXT:    s_add_u32 s1, s1, s0
343; GFX8-NEXT:    s_mov_b32 s0, s4
344; GFX8-NEXT:    ; return to shader part epilog
345;
346; GFX9-LABEL: s_mul_i64:
347; GFX9:       ; %bb.0:
348; GFX9-NEXT:    s_mul_i32 s4, s0, s2
349; GFX9-NEXT:    s_mul_hi_u32 s5, s0, s2
350; GFX9-NEXT:    s_mul_i32 s0, s0, s3
351; GFX9-NEXT:    s_add_u32 s0, s0, s5
352; GFX9-NEXT:    s_mul_i32 s1, s1, s2
353; GFX9-NEXT:    s_add_u32 s1, s1, s0
354; GFX9-NEXT:    s_mov_b32 s0, s4
355; GFX9-NEXT:    ; return to shader part epilog
356;
357; GFX10-LABEL: s_mul_i64:
358; GFX10:       ; %bb.0:
359; GFX10-NEXT:    s_mul_hi_u32 s4, s0, s2
360; GFX10-NEXT:    s_mul_i32 s3, s0, s3
361; GFX10-NEXT:    s_mul_i32 s1, s1, s2
362; GFX10-NEXT:    s_add_i32 s3, s4, s3
363; GFX10-NEXT:    s_mul_i32 s0, s0, s2
364; GFX10-NEXT:    s_add_i32 s1, s3, s1
365; GFX10-NEXT:    ; return to shader part epilog
366  %result = mul i64 %num, %den
367  ret i64 %result
368}
369
370define i64 @v_mul_i64(i64 %num, i64 %den) {
371; GCN-LABEL: v_mul_i64:
372; GCN:       ; %bb.0:
373; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
374; GCN-NEXT:    v_mov_b32_e32 v4, v0
375; GCN-NEXT:    v_mov_b32_e32 v5, v1
376; GCN-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v4, v2, 0
377; GCN-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], v4, v3, v[1:2]
378; GCN-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v5, v2, v[3:4]
379; GCN-NEXT:    s_setpc_b64 s[30:31]
380;
381; GFX10-LABEL: v_mul_i64:
382; GFX10:       ; %bb.0:
383; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
384; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
385; GFX10-NEXT:    v_mov_b32_e32 v4, v0
386; GFX10-NEXT:    v_mov_b32_e32 v5, v1
387; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s4, v4, v2, 0
388; GFX10-NEXT:    v_mul_lo_u32 v3, v4, v3
389; GFX10-NEXT:    v_mul_lo_u32 v2, v5, v2
390; GFX10-NEXT:    v_add3_u32 v1, v1, v3, v2
391; GFX10-NEXT:    s_setpc_b64 s[30:31]
392  %result = mul i64 %num, %den
393  ret i64 %result
394}
395
396define amdgpu_ps <3 x i32> @s_mul_i96(i96 inreg %num, i96 inreg %den) {
397; GFX7-LABEL: s_mul_i96:
398; GFX7:       ; %bb.0:
399; GFX7-NEXT:    v_mov_b32_e32 v0, s3
400; GFX7-NEXT:    v_mul_hi_u32 v0, s0, v0
401; GFX7-NEXT:    v_mov_b32_e32 v1, s4
402; GFX7-NEXT:    v_mul_hi_u32 v1, s0, v1
403; GFX7-NEXT:    s_mul_i32 s5, s0, s5
404; GFX7-NEXT:    v_readfirstlane_b32 s7, v0
405; GFX7-NEXT:    s_mul_i32 s8, s1, s4
406; GFX7-NEXT:    v_mov_b32_e32 v0, s1
407; GFX7-NEXT:    s_add_u32 s5, s8, s5
408; GFX7-NEXT:    s_mul_i32 s2, s2, s3
409; GFX7-NEXT:    v_mul_hi_u32 v0, v0, s3
410; GFX7-NEXT:    s_mul_i32 s6, s0, s3
411; GFX7-NEXT:    s_add_u32 s2, s2, s5
412; GFX7-NEXT:    s_mul_i32 s0, s0, s4
413; GFX7-NEXT:    v_readfirstlane_b32 s4, v1
414; GFX7-NEXT:    s_add_u32 s0, s0, s7
415; GFX7-NEXT:    s_addc_u32 s2, s4, s2
416; GFX7-NEXT:    s_mul_i32 s1, s1, s3
417; GFX7-NEXT:    v_readfirstlane_b32 s3, v0
418; GFX7-NEXT:    s_add_u32 s1, s1, s0
419; GFX7-NEXT:    s_addc_u32 s2, s3, s2
420; GFX7-NEXT:    s_mov_b32 s0, s6
421; GFX7-NEXT:    ; return to shader part epilog
422;
423; GFX8-LABEL: s_mul_i96:
424; GFX8:       ; %bb.0:
425; GFX8-NEXT:    v_mov_b32_e32 v0, s3
426; GFX8-NEXT:    v_mul_hi_u32 v0, s0, v0
427; GFX8-NEXT:    v_mov_b32_e32 v1, s4
428; GFX8-NEXT:    v_mul_hi_u32 v1, s0, v1
429; GFX8-NEXT:    s_mul_i32 s5, s0, s5
430; GFX8-NEXT:    v_readfirstlane_b32 s7, v0
431; GFX8-NEXT:    s_mul_i32 s8, s1, s4
432; GFX8-NEXT:    v_mov_b32_e32 v0, s1
433; GFX8-NEXT:    s_add_u32 s5, s8, s5
434; GFX8-NEXT:    s_mul_i32 s2, s2, s3
435; GFX8-NEXT:    v_mul_hi_u32 v0, v0, s3
436; GFX8-NEXT:    s_mul_i32 s6, s0, s3
437; GFX8-NEXT:    s_add_u32 s2, s2, s5
438; GFX8-NEXT:    s_mul_i32 s0, s0, s4
439; GFX8-NEXT:    v_readfirstlane_b32 s4, v1
440; GFX8-NEXT:    s_add_u32 s0, s0, s7
441; GFX8-NEXT:    s_addc_u32 s2, s4, s2
442; GFX8-NEXT:    s_mul_i32 s1, s1, s3
443; GFX8-NEXT:    v_readfirstlane_b32 s3, v0
444; GFX8-NEXT:    s_add_u32 s1, s1, s0
445; GFX8-NEXT:    s_addc_u32 s2, s3, s2
446; GFX8-NEXT:    s_mov_b32 s0, s6
447; GFX8-NEXT:    ; return to shader part epilog
448;
449; GFX9-LABEL: s_mul_i96:
450; GFX9:       ; %bb.0:
451; GFX9-NEXT:    s_mul_i32 s5, s0, s5
452; GFX9-NEXT:    s_mul_i32 s8, s1, s4
453; GFX9-NEXT:    s_add_u32 s5, s8, s5
454; GFX9-NEXT:    s_mul_i32 s2, s2, s3
455; GFX9-NEXT:    s_mul_hi_u32 s7, s0, s3
456; GFX9-NEXT:    s_add_u32 s2, s2, s5
457; GFX9-NEXT:    s_mul_i32 s5, s0, s4
458; GFX9-NEXT:    s_mul_i32 s6, s0, s3
459; GFX9-NEXT:    s_mul_hi_u32 s0, s0, s4
460; GFX9-NEXT:    s_add_u32 s4, s5, s7
461; GFX9-NEXT:    s_addc_u32 s0, s0, s2
462; GFX9-NEXT:    s_mul_i32 s2, s1, s3
463; GFX9-NEXT:    s_mul_hi_u32 s3, s1, s3
464; GFX9-NEXT:    s_add_u32 s1, s2, s4
465; GFX9-NEXT:    s_addc_u32 s2, s3, s0
466; GFX9-NEXT:    s_mov_b32 s0, s6
467; GFX9-NEXT:    ; return to shader part epilog
468;
469; GFX10-LABEL: s_mul_i96:
470; GFX10:       ; %bb.0:
471; GFX10-NEXT:    s_mul_i32 s6, s0, s5
472; GFX10-NEXT:    s_mul_i32 s7, s1, s4
473; GFX10-NEXT:    s_mul_i32 s2, s2, s3
474; GFX10-NEXT:    s_add_i32 s6, s6, s7
475; GFX10-NEXT:    s_mul_hi_u32 s7, s0, s3
476; GFX10-NEXT:    s_add_i32 s6, s6, s2
477; GFX10-NEXT:    s_mul_i32 s2, s0, s4
478; GFX10-NEXT:    s_mul_i32 s5, s0, s3
479; GFX10-NEXT:    s_mul_hi_u32 s0, s0, s4
480; GFX10-NEXT:    s_add_u32 s2, s2, s7
481; GFX10-NEXT:    s_mul_i32 s4, s1, s3
482; GFX10-NEXT:    s_addc_u32 s0, s0, s6
483; GFX10-NEXT:    s_mul_hi_u32 s3, s1, s3
484; GFX10-NEXT:    s_add_u32 s1, s4, s2
485; GFX10-NEXT:    s_addc_u32 s2, s3, s0
486; GFX10-NEXT:    s_mov_b32 s0, s5
487; GFX10-NEXT:    ; return to shader part epilog
488  %result = mul i96 %num, %den
489  %cast = bitcast i96 %result to <3 x i32>
490  ret <3 x i32> %cast
491}
492
493define i96 @v_mul_i96(i96 %num, i96 %den) {
494; GCN-LABEL: v_mul_i96:
495; GCN:       ; %bb.0:
496; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
497; GCN-NEXT:    v_mov_b32_e32 v6, v0
498; GCN-NEXT:    v_mov_b32_e32 v7, v1
499; GCN-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v6, v5, 0
500; GCN-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v7, v4, v[0:1]
501; GCN-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v6, v3, 0
502; GCN-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v2, v3, v[8:9]
503; GCN-NEXT:    v_mov_b32_e32 v2, v8
504; GCN-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v6, v4, v[1:2]
505; GCN-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v7, v3, v[1:2]
506; GCN-NEXT:    s_setpc_b64 s[30:31]
507;
508; GFX10-LABEL: v_mul_i96:
509; GFX10:       ; %bb.0:
510; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
511; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
512; GFX10-NEXT:    v_mov_b32_e32 v6, v0
513; GFX10-NEXT:    v_mov_b32_e32 v7, v1
514; GFX10-NEXT:    v_mul_lo_u32 v2, v2, v3
515; GFX10-NEXT:    v_mul_lo_u32 v5, v6, v5
516; GFX10-NEXT:    v_mul_lo_u32 v8, v7, v4
517; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s4, v6, v3, 0
518; GFX10-NEXT:    v_add3_u32 v2, v5, v8, v2
519; GFX10-NEXT:    v_mad_u64_u32 v[1:2], s4, v6, v4, v[1:2]
520; GFX10-NEXT:    v_mad_u64_u32 v[1:2], s4, v7, v3, v[1:2]
521; GFX10-NEXT:    s_setpc_b64 s[30:31]
522  %result = mul i96 %num, %den
523  ret i96 %result
524}
525
526define amdgpu_ps <4 x i32> @s_mul_i128(i128 inreg %num, i128 inreg %den) {
527; GFX7-LABEL: s_mul_i128:
528; GFX7:       ; %bb.0:
529; GFX7-NEXT:    v_mov_b32_e32 v0, s4
530; GFX7-NEXT:    v_mul_hi_u32 v0, s0, v0
531; GFX7-NEXT:    v_mov_b32_e32 v1, s5
532; GFX7-NEXT:    v_mul_hi_u32 v2, s1, v1
533; GFX7-NEXT:    s_mul_i32 s10, s0, s6
534; GFX7-NEXT:    v_readfirstlane_b32 s9, v0
535; GFX7-NEXT:    v_mov_b32_e32 v0, s6
536; GFX7-NEXT:    v_mul_hi_u32 v0, s0, v0
537; GFX7-NEXT:    v_readfirstlane_b32 s13, v2
538; GFX7-NEXT:    v_mov_b32_e32 v2, s2
539; GFX7-NEXT:    v_mul_hi_u32 v2, v2, s4
540; GFX7-NEXT:    s_mul_i32 s12, s1, s5
541; GFX7-NEXT:    v_readfirstlane_b32 s11, v0
542; GFX7-NEXT:    s_add_u32 s10, s12, s10
543; GFX7-NEXT:    v_mul_hi_u32 v1, s0, v1
544; GFX7-NEXT:    v_mov_b32_e32 v0, s1
545; GFX7-NEXT:    s_addc_u32 s11, s13, s11
546; GFX7-NEXT:    s_mul_i32 s12, s2, s4
547; GFX7-NEXT:    v_readfirstlane_b32 s13, v2
548; GFX7-NEXT:    s_add_u32 s10, s12, s10
549; GFX7-NEXT:    v_mul_hi_u32 v0, v0, s4
550; GFX7-NEXT:    s_addc_u32 s11, s13, s11
551; GFX7-NEXT:    s_mul_i32 s12, s0, s5
552; GFX7-NEXT:    v_readfirstlane_b32 s13, v1
553; GFX7-NEXT:    s_add_u32 s9, s12, s9
554; GFX7-NEXT:    s_addc_u32 s10, s13, s10
555; GFX7-NEXT:    s_mul_i32 s13, s1, s4
556; GFX7-NEXT:    s_cselect_b32 s12, 1, 0
557; GFX7-NEXT:    v_readfirstlane_b32 s14, v0
558; GFX7-NEXT:    s_add_u32 s9, s13, s9
559; GFX7-NEXT:    s_mul_i32 s8, s0, s4
560; GFX7-NEXT:    s_addc_u32 s10, s14, s10
561; GFX7-NEXT:    s_mul_i32 s0, s0, s7
562; GFX7-NEXT:    s_addc_u32 s0, s11, s0
563; GFX7-NEXT:    s_mul_i32 s1, s1, s6
564; GFX7-NEXT:    s_cmp_lg_u32 s12, 0
565; GFX7-NEXT:    s_addc_u32 s0, s0, s1
566; GFX7-NEXT:    s_mul_i32 s2, s2, s5
567; GFX7-NEXT:    s_add_u32 s0, s2, s0
568; GFX7-NEXT:    s_mul_i32 s3, s3, s4
569; GFX7-NEXT:    s_add_u32 s3, s3, s0
570; GFX7-NEXT:    s_mov_b32 s0, s8
571; GFX7-NEXT:    s_mov_b32 s1, s9
572; GFX7-NEXT:    s_mov_b32 s2, s10
573; GFX7-NEXT:    ; return to shader part epilog
574;
575; GFX8-LABEL: s_mul_i128:
576; GFX8:       ; %bb.0:
577; GFX8-NEXT:    v_mov_b32_e32 v0, s4
578; GFX8-NEXT:    v_mul_hi_u32 v0, s0, v0
579; GFX8-NEXT:    v_mov_b32_e32 v1, s5
580; GFX8-NEXT:    v_mul_hi_u32 v2, s1, v1
581; GFX8-NEXT:    s_mul_i32 s10, s0, s6
582; GFX8-NEXT:    v_readfirstlane_b32 s9, v0
583; GFX8-NEXT:    v_mov_b32_e32 v0, s6
584; GFX8-NEXT:    v_mul_hi_u32 v0, s0, v0
585; GFX8-NEXT:    v_readfirstlane_b32 s13, v2
586; GFX8-NEXT:    v_mov_b32_e32 v2, s2
587; GFX8-NEXT:    v_mul_hi_u32 v2, v2, s4
588; GFX8-NEXT:    s_mul_i32 s12, s1, s5
589; GFX8-NEXT:    v_readfirstlane_b32 s11, v0
590; GFX8-NEXT:    s_add_u32 s10, s12, s10
591; GFX8-NEXT:    v_mul_hi_u32 v1, s0, v1
592; GFX8-NEXT:    v_mov_b32_e32 v0, s1
593; GFX8-NEXT:    s_addc_u32 s11, s13, s11
594; GFX8-NEXT:    s_mul_i32 s12, s2, s4
595; GFX8-NEXT:    v_readfirstlane_b32 s13, v2
596; GFX8-NEXT:    s_add_u32 s10, s12, s10
597; GFX8-NEXT:    v_mul_hi_u32 v0, v0, s4
598; GFX8-NEXT:    s_addc_u32 s11, s13, s11
599; GFX8-NEXT:    s_mul_i32 s12, s0, s5
600; GFX8-NEXT:    v_readfirstlane_b32 s13, v1
601; GFX8-NEXT:    s_add_u32 s9, s12, s9
602; GFX8-NEXT:    s_addc_u32 s10, s13, s10
603; GFX8-NEXT:    s_mul_i32 s13, s1, s4
604; GFX8-NEXT:    s_cselect_b32 s12, 1, 0
605; GFX8-NEXT:    v_readfirstlane_b32 s14, v0
606; GFX8-NEXT:    s_add_u32 s9, s13, s9
607; GFX8-NEXT:    s_mul_i32 s8, s0, s4
608; GFX8-NEXT:    s_addc_u32 s10, s14, s10
609; GFX8-NEXT:    s_mul_i32 s0, s0, s7
610; GFX8-NEXT:    s_addc_u32 s0, s11, s0
611; GFX8-NEXT:    s_mul_i32 s1, s1, s6
612; GFX8-NEXT:    s_cmp_lg_u32 s12, 0
613; GFX8-NEXT:    s_addc_u32 s0, s0, s1
614; GFX8-NEXT:    s_mul_i32 s2, s2, s5
615; GFX8-NEXT:    s_add_u32 s0, s2, s0
616; GFX8-NEXT:    s_mul_i32 s3, s3, s4
617; GFX8-NEXT:    s_add_u32 s3, s3, s0
618; GFX8-NEXT:    s_mov_b32 s0, s8
619; GFX8-NEXT:    s_mov_b32 s1, s9
620; GFX8-NEXT:    s_mov_b32 s2, s10
621; GFX8-NEXT:    ; return to shader part epilog
622;
623; GFX9-LABEL: s_mul_i128:
624; GFX9:       ; %bb.0:
625; GFX9-NEXT:    s_mul_i32 s10, s0, s6
626; GFX9-NEXT:    s_mul_i32 s12, s1, s5
627; GFX9-NEXT:    s_mul_hi_u32 s11, s0, s6
628; GFX9-NEXT:    s_mul_hi_u32 s13, s1, s5
629; GFX9-NEXT:    s_add_u32 s10, s12, s10
630; GFX9-NEXT:    s_addc_u32 s11, s13, s11
631; GFX9-NEXT:    s_mul_i32 s12, s2, s4
632; GFX9-NEXT:    s_mul_hi_u32 s13, s2, s4
633; GFX9-NEXT:    s_add_u32 s10, s12, s10
634; GFX9-NEXT:    s_mul_hi_u32 s9, s0, s4
635; GFX9-NEXT:    s_addc_u32 s11, s13, s11
636; GFX9-NEXT:    s_mul_i32 s12, s0, s5
637; GFX9-NEXT:    s_mul_hi_u32 s13, s0, s5
638; GFX9-NEXT:    s_add_u32 s9, s12, s9
639; GFX9-NEXT:    s_addc_u32 s10, s13, s10
640; GFX9-NEXT:    s_mul_i32 s13, s1, s4
641; GFX9-NEXT:    s_cselect_b32 s12, 1, 0
642; GFX9-NEXT:    s_mul_hi_u32 s14, s1, s4
643; GFX9-NEXT:    s_add_u32 s9, s13, s9
644; GFX9-NEXT:    s_mul_i32 s8, s0, s4
645; GFX9-NEXT:    s_addc_u32 s10, s14, s10
646; GFX9-NEXT:    s_mul_i32 s0, s0, s7
647; GFX9-NEXT:    s_addc_u32 s0, s11, s0
648; GFX9-NEXT:    s_mul_i32 s1, s1, s6
649; GFX9-NEXT:    s_cmp_lg_u32 s12, 0
650; GFX9-NEXT:    s_addc_u32 s0, s0, s1
651; GFX9-NEXT:    s_mul_i32 s2, s2, s5
652; GFX9-NEXT:    s_add_u32 s0, s2, s0
653; GFX9-NEXT:    s_mul_i32 s3, s3, s4
654; GFX9-NEXT:    s_add_u32 s3, s3, s0
655; GFX9-NEXT:    s_mov_b32 s0, s8
656; GFX9-NEXT:    s_mov_b32 s1, s9
657; GFX9-NEXT:    s_mov_b32 s2, s10
658; GFX9-NEXT:    ; return to shader part epilog
659;
660; GFX10-LABEL: s_mul_i128:
661; GFX10:       ; %bb.0:
662; GFX10-NEXT:    s_mul_i32 s9, s0, s6
663; GFX10-NEXT:    s_mul_i32 s11, s1, s5
664; GFX10-NEXT:    s_mul_hi_u32 s10, s0, s6
665; GFX10-NEXT:    s_mul_hi_u32 s12, s1, s5
666; GFX10-NEXT:    s_add_u32 s9, s11, s9
667; GFX10-NEXT:    s_mul_i32 s11, s2, s4
668; GFX10-NEXT:    s_addc_u32 s10, s12, s10
669; GFX10-NEXT:    s_mul_hi_u32 s12, s2, s4
670; GFX10-NEXT:    s_mul_hi_u32 s8, s0, s4
671; GFX10-NEXT:    s_add_u32 s9, s11, s9
672; GFX10-NEXT:    s_mul_i32 s11, s0, s5
673; GFX10-NEXT:    s_addc_u32 s10, s12, s10
674; GFX10-NEXT:    s_mul_hi_u32 s12, s0, s5
675; GFX10-NEXT:    s_add_u32 s8, s11, s8
676; GFX10-NEXT:    s_addc_u32 s9, s12, s9
677; GFX10-NEXT:    s_mul_i32 s12, s1, s4
678; GFX10-NEXT:    s_mul_hi_u32 s13, s1, s4
679; GFX10-NEXT:    s_cselect_b32 s11, 1, 0
680; GFX10-NEXT:    s_add_u32 s8, s12, s8
681; GFX10-NEXT:    s_mul_i32 s12, s0, s7
682; GFX10-NEXT:    s_addc_u32 s7, s13, s9
683; GFX10-NEXT:    s_addc_u32 s9, s10, s12
684; GFX10-NEXT:    s_mul_i32 s1, s1, s6
685; GFX10-NEXT:    s_cmp_lg_u32 s11, 0
686; GFX10-NEXT:    s_mul_i32 s2, s2, s5
687; GFX10-NEXT:    s_addc_u32 s1, s9, s1
688; GFX10-NEXT:    s_mul_i32 s3, s3, s4
689; GFX10-NEXT:    s_add_i32 s1, s1, s2
690; GFX10-NEXT:    s_mul_i32 s0, s0, s4
691; GFX10-NEXT:    s_add_i32 s3, s1, s3
692; GFX10-NEXT:    s_mov_b32 s1, s8
693; GFX10-NEXT:    s_mov_b32 s2, s7
694; GFX10-NEXT:    ; return to shader part epilog
695  %result = mul i128 %num, %den
696  %cast = bitcast i128 %result to <4 x i32>
697  ret <4 x i32> %cast
698}
699
700define i128 @v_mul_i128(i128 %num, i128 %den) {
701; GFX7-LABEL: v_mul_i128:
702; GFX7:       ; %bb.0:
703; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
704; GFX7-NEXT:    v_mov_b32_e32 v8, v0
705; GFX7-NEXT:    v_mov_b32_e32 v9, v1
706; GFX7-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v8, v6, 0
707; GFX7-NEXT:    v_mov_b32_e32 v10, v2
708; GFX7-NEXT:    v_mul_lo_u32 v7, v8, v7
709; GFX7-NEXT:    v_mad_u64_u32 v[11:12], s[4:5], v9, v5, v[0:1]
710; GFX7-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v8, v4, 0
711; GFX7-NEXT:    v_mad_u64_u32 v[11:12], s[4:5], v10, v4, v[11:12]
712; GFX7-NEXT:    v_mul_lo_u32 v6, v9, v6
713; GFX7-NEXT:    v_mov_b32_e32 v2, v11
714; GFX7-NEXT:    v_mad_u64_u32 v[1:2], vcc, v8, v5, v[1:2]
715; GFX7-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v9, v4, v[1:2]
716; GFX7-NEXT:    v_addc_u32_e64 v7, s[4:5], v12, v7, s[4:5]
717; GFX7-NEXT:    v_addc_u32_e32 v6, vcc, v7, v6, vcc
718; GFX7-NEXT:    v_mad_u64_u32 v[5:6], s[4:5], v10, v5, v[6:7]
719; GFX7-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], v3, v4, v[5:6]
720; GFX7-NEXT:    s_setpc_b64 s[30:31]
721;
722; GFX8-LABEL: v_mul_i128:
723; GFX8:       ; %bb.0:
724; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
725; GFX8-NEXT:    v_mov_b32_e32 v8, v0
726; GFX8-NEXT:    v_mov_b32_e32 v9, v1
727; GFX8-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v8, v6, 0
728; GFX8-NEXT:    v_mov_b32_e32 v10, v2
729; GFX8-NEXT:    v_mul_lo_u32 v7, v8, v7
730; GFX8-NEXT:    v_mad_u64_u32 v[11:12], s[4:5], v9, v5, v[0:1]
731; GFX8-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v8, v4, 0
732; GFX8-NEXT:    v_mad_u64_u32 v[11:12], s[4:5], v10, v4, v[11:12]
733; GFX8-NEXT:    v_mul_lo_u32 v6, v9, v6
734; GFX8-NEXT:    v_mov_b32_e32 v2, v11
735; GFX8-NEXT:    v_mad_u64_u32 v[1:2], vcc, v8, v5, v[1:2]
736; GFX8-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v9, v4, v[1:2]
737; GFX8-NEXT:    v_addc_u32_e64 v7, s[4:5], v12, v7, s[4:5]
738; GFX8-NEXT:    v_addc_u32_e32 v6, vcc, v7, v6, vcc
739; GFX8-NEXT:    v_mad_u64_u32 v[5:6], s[4:5], v10, v5, v[6:7]
740; GFX8-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], v3, v4, v[5:6]
741; GFX8-NEXT:    s_setpc_b64 s[30:31]
742;
743; GFX9-LABEL: v_mul_i128:
744; GFX9:       ; %bb.0:
745; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
746; GFX9-NEXT:    v_mov_b32_e32 v8, v0
747; GFX9-NEXT:    v_mov_b32_e32 v9, v1
748; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v8, v6, 0
749; GFX9-NEXT:    v_mov_b32_e32 v10, v2
750; GFX9-NEXT:    v_mul_lo_u32 v7, v8, v7
751; GFX9-NEXT:    v_mad_u64_u32 v[11:12], s[4:5], v9, v5, v[0:1]
752; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v8, v4, 0
753; GFX9-NEXT:    v_mad_u64_u32 v[11:12], s[4:5], v10, v4, v[11:12]
754; GFX9-NEXT:    v_mul_lo_u32 v6, v9, v6
755; GFX9-NEXT:    v_mov_b32_e32 v2, v11
756; GFX9-NEXT:    v_mad_u64_u32 v[1:2], vcc, v8, v5, v[1:2]
757; GFX9-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v9, v4, v[1:2]
758; GFX9-NEXT:    v_addc_co_u32_e64 v7, s[4:5], v12, v7, s[4:5]
759; GFX9-NEXT:    v_addc_co_u32_e32 v6, vcc, v7, v6, vcc
760; GFX9-NEXT:    v_mad_u64_u32 v[5:6], s[4:5], v10, v5, v[6:7]
761; GFX9-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], v3, v4, v[5:6]
762; GFX9-NEXT:    s_setpc_b64 s[30:31]
763;
764; GFX10-LABEL: v_mul_i128:
765; GFX10:       ; %bb.0:
766; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
767; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
768; GFX10-NEXT:    v_mov_b32_e32 v8, v0
769; GFX10-NEXT:    v_mov_b32_e32 v9, v1
770; GFX10-NEXT:    v_mov_b32_e32 v10, v2
771; GFX10-NEXT:    v_mul_lo_u32 v3, v3, v4
772; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s4, v8, v6, 0
773; GFX10-NEXT:    v_mul_lo_u32 v7, v8, v7
774; GFX10-NEXT:    v_mul_lo_u32 v6, v9, v6
775; GFX10-NEXT:    v_mad_u64_u32 v[11:12], s4, v9, v5, v[0:1]
776; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s4, v8, v4, 0
777; GFX10-NEXT:    v_mad_u64_u32 v[11:12], s4, v10, v4, v[11:12]
778; GFX10-NEXT:    v_mov_b32_e32 v2, v11
779; GFX10-NEXT:    v_mad_u64_u32 v[1:2], vcc_lo, v8, v5, v[1:2]
780; GFX10-NEXT:    v_mul_lo_u32 v5, v10, v5
781; GFX10-NEXT:    v_mad_u64_u32 v[1:2], s4, v9, v4, v[1:2]
782; GFX10-NEXT:    v_add_co_ci_u32_e64 v7, s4, v12, v7, s4
783; GFX10-NEXT:    v_add_co_ci_u32_e32 v4, vcc_lo, v7, v6, vcc_lo
784; GFX10-NEXT:    v_add3_u32 v3, v4, v5, v3
785; GFX10-NEXT:    s_setpc_b64 s[30:31]
786  %result = mul i128 %num, %den
787  ret i128 %result
788}
789
790define amdgpu_ps <8 x i32> @s_mul_i256(i256 inreg %num, i256 inreg %den) {
791; GFX7-LABEL: s_mul_i256:
792; GFX7:       ; %bb.0:
793; GFX7-NEXT:    s_mov_b32 s16, s0
794; GFX7-NEXT:    v_mov_b32_e32 v0, s8
795; GFX7-NEXT:    v_mul_hi_u32 v0, s16, v0
796; GFX7-NEXT:    v_mov_b32_e32 v1, s9
797; GFX7-NEXT:    v_mul_hi_u32 v2, s1, v1
798; GFX7-NEXT:    v_mul_hi_u32 v1, s16, v1
799; GFX7-NEXT:    v_readfirstlane_b32 s17, v0
800; GFX7-NEXT:    v_mov_b32_e32 v0, s10
801; GFX7-NEXT:    v_mul_hi_u32 v0, s16, v0
802; GFX7-NEXT:    v_readfirstlane_b32 s21, v2
803; GFX7-NEXT:    v_mov_b32_e32 v2, s2
804; GFX7-NEXT:    v_readfirstlane_b32 s23, v1
805; GFX7-NEXT:    v_readfirstlane_b32 s19, v0
806; GFX7-NEXT:    v_mov_b32_e32 v0, s1
807; GFX7-NEXT:    v_mul_hi_u32 v1, v0, s8
808; GFX7-NEXT:    v_mul_hi_u32 v3, v2, s8
809; GFX7-NEXT:    v_mov_b32_e32 v4, s11
810; GFX7-NEXT:    s_mul_i32 s18, s16, s10
811; GFX7-NEXT:    v_readfirstlane_b32 s24, v1
812; GFX7-NEXT:    v_mov_b32_e32 v1, s12
813; GFX7-NEXT:    v_readfirstlane_b32 s22, v3
814; GFX7-NEXT:    v_mul_hi_u32 v3, s16, v1
815; GFX7-NEXT:    s_mul_i32 s20, s1, s9
816; GFX7-NEXT:    v_mul_hi_u32 v5, s1, v4
817; GFX7-NEXT:    s_add_u32 s18, s20, s18
818; GFX7-NEXT:    v_readfirstlane_b32 s25, v3
819; GFX7-NEXT:    v_mul_hi_u32 v3, v2, s10
820; GFX7-NEXT:    s_addc_u32 s19, s21, s19
821; GFX7-NEXT:    s_mul_i32 s21, s2, s8
822; GFX7-NEXT:    s_cselect_b32 s20, 1, 0
823; GFX7-NEXT:    s_add_u32 s18, s21, s18
824; GFX7-NEXT:    v_readfirstlane_b32 s28, v3
825; GFX7-NEXT:    v_mov_b32_e32 v3, s3
826; GFX7-NEXT:    s_addc_u32 s19, s22, s19
827; GFX7-NEXT:    s_mul_i32 s22, s16, s9
828; GFX7-NEXT:    v_readfirstlane_b32 s27, v5
829; GFX7-NEXT:    v_mul_hi_u32 v5, v3, s9
830; GFX7-NEXT:    s_cselect_b32 s21, 1, 0
831; GFX7-NEXT:    s_add_u32 s17, s22, s17
832; GFX7-NEXT:    s_addc_u32 s18, s23, s18
833; GFX7-NEXT:    s_mul_i32 s23, s1, s8
834; GFX7-NEXT:    s_cselect_b32 s22, 1, 0
835; GFX7-NEXT:    s_add_u32 s17, s23, s17
836; GFX7-NEXT:    s_addc_u32 s18, s24, s18
837; GFX7-NEXT:    s_mul_i32 s24, s16, s12
838; GFX7-NEXT:    s_mul_i32 s26, s1, s11
839; GFX7-NEXT:    v_readfirstlane_b32 s29, v5
840; GFX7-NEXT:    v_mov_b32_e32 v5, s4
841; GFX7-NEXT:    s_cselect_b32 s23, 1, 0
842; GFX7-NEXT:    s_add_u32 s24, s26, s24
843; GFX7-NEXT:    v_mul_hi_u32 v6, v5, s8
844; GFX7-NEXT:    s_addc_u32 s25, s27, s25
845; GFX7-NEXT:    s_mul_i32 s27, s2, s10
846; GFX7-NEXT:    s_cselect_b32 s26, 1, 0
847; GFX7-NEXT:    s_add_u32 s24, s27, s24
848; GFX7-NEXT:    v_mul_hi_u32 v0, v0, s10
849; GFX7-NEXT:    s_addc_u32 s25, s28, s25
850; GFX7-NEXT:    s_mul_i32 s28, s3, s9
851; GFX7-NEXT:    s_cselect_b32 s27, 1, 0
852; GFX7-NEXT:    s_add_u32 s24, s28, s24
853; GFX7-NEXT:    v_readfirstlane_b32 s30, v6
854; GFX7-NEXT:    v_mul_hi_u32 v6, s16, v4
855; GFX7-NEXT:    s_addc_u32 s25, s29, s25
856; GFX7-NEXT:    s_mul_i32 s29, s4, s8
857; GFX7-NEXT:    s_cselect_b32 s28, 1, 0
858; GFX7-NEXT:    s_add_u32 s24, s29, s24
859; GFX7-NEXT:    v_readfirstlane_b32 s33, v0
860; GFX7-NEXT:    v_mul_hi_u32 v0, v2, s9
861; GFX7-NEXT:    s_addc_u32 s25, s30, s25
862; GFX7-NEXT:    s_mul_i32 s30, s16, s11
863; GFX7-NEXT:    s_cselect_b32 s29, 1, 0
864; GFX7-NEXT:    v_readfirstlane_b32 s31, v6
865; GFX7-NEXT:    s_add_u32 s19, s30, s19
866; GFX7-NEXT:    s_addc_u32 s24, s31, s24
867; GFX7-NEXT:    s_mul_i32 s31, s1, s10
868; GFX7-NEXT:    s_cselect_b32 s30, 1, 0
869; GFX7-NEXT:    s_add_u32 s19, s31, s19
870; GFX7-NEXT:    v_readfirstlane_b32 s34, v0
871; GFX7-NEXT:    v_mul_hi_u32 v0, v3, s8
872; GFX7-NEXT:    s_addc_u32 s24, s33, s24
873; GFX7-NEXT:    s_mul_i32 s33, s2, s9
874; GFX7-NEXT:    s_cselect_b32 s31, 1, 0
875; GFX7-NEXT:    s_add_u32 s19, s33, s19
876; GFX7-NEXT:    s_addc_u32 s24, s34, s24
877; GFX7-NEXT:    s_mul_i32 s34, s3, s8
878; GFX7-NEXT:    s_cselect_b32 s33, 1, 0
879; GFX7-NEXT:    v_readfirstlane_b32 s35, v0
880; GFX7-NEXT:    s_add_u32 s19, s34, s19
881; GFX7-NEXT:    v_mov_b32_e32 v0, s14
882; GFX7-NEXT:    s_addc_u32 s24, s35, s24
883; GFX7-NEXT:    v_mul_hi_u32 v0, s16, v0
884; GFX7-NEXT:    s_cselect_b32 s34, 1, 0
885; GFX7-NEXT:    s_cmp_lg_u32 s23, 0
886; GFX7-NEXT:    s_addc_u32 s19, s22, s19
887; GFX7-NEXT:    v_mov_b32_e32 v2, s13
888; GFX7-NEXT:    s_cselect_b32 s22, 1, 0
889; GFX7-NEXT:    s_cmp_lg_u32 s21, 0
890; GFX7-NEXT:    v_mul_hi_u32 v6, s1, v2
891; GFX7-NEXT:    s_addc_u32 s20, s20, 0
892; GFX7-NEXT:    v_readfirstlane_b32 s23, v0
893; GFX7-NEXT:    v_mul_hi_u32 v0, s2, v1
894; GFX7-NEXT:    s_cmp_lg_u32 s22, 0
895; GFX7-NEXT:    s_addc_u32 s20, s20, s24
896; GFX7-NEXT:    s_mul_i32 s22, s16, s14
897; GFX7-NEXT:    s_mul_i32 s24, s1, s13
898; GFX7-NEXT:    s_cselect_b32 s21, 1, 0
899; GFX7-NEXT:    v_readfirstlane_b32 s35, v6
900; GFX7-NEXT:    s_add_u32 s22, s24, s22
901; GFX7-NEXT:    s_addc_u32 s23, s35, s23
902; GFX7-NEXT:    v_readfirstlane_b32 s35, v0
903; GFX7-NEXT:    v_mul_hi_u32 v0, v3, s11
904; GFX7-NEXT:    s_mul_i32 s24, s2, s12
905; GFX7-NEXT:    s_add_u32 s22, s24, s22
906; GFX7-NEXT:    s_addc_u32 s23, s35, s23
907; GFX7-NEXT:    v_readfirstlane_b32 s35, v0
908; GFX7-NEXT:    v_mul_hi_u32 v0, v5, s10
909; GFX7-NEXT:    s_mul_i32 s24, s3, s11
910; GFX7-NEXT:    s_add_u32 s22, s24, s22
911; GFX7-NEXT:    s_addc_u32 s23, s35, s23
912; GFX7-NEXT:    v_readfirstlane_b32 s35, v0
913; GFX7-NEXT:    v_mov_b32_e32 v0, s5
914; GFX7-NEXT:    v_mul_hi_u32 v6, v0, s9
915; GFX7-NEXT:    s_mul_i32 s24, s4, s10
916; GFX7-NEXT:    s_add_u32 s22, s24, s22
917; GFX7-NEXT:    v_mul_hi_u32 v1, s1, v1
918; GFX7-NEXT:    s_addc_u32 s23, s35, s23
919; GFX7-NEXT:    v_readfirstlane_b32 s35, v6
920; GFX7-NEXT:    v_mov_b32_e32 v6, s6
921; GFX7-NEXT:    v_mul_hi_u32 v6, v6, s8
922; GFX7-NEXT:    s_mul_i32 s24, s5, s9
923; GFX7-NEXT:    s_add_u32 s22, s24, s22
924; GFX7-NEXT:    v_mul_hi_u32 v2, s16, v2
925; GFX7-NEXT:    v_readfirstlane_b32 s36, v1
926; GFX7-NEXT:    v_mul_hi_u32 v1, s2, v4
927; GFX7-NEXT:    s_addc_u32 s23, s35, s23
928; GFX7-NEXT:    s_mul_i32 s24, s6, s8
929; GFX7-NEXT:    v_readfirstlane_b32 s35, v6
930; GFX7-NEXT:    s_add_u32 s22, s24, s22
931; GFX7-NEXT:    s_addc_u32 s23, s35, s23
932; GFX7-NEXT:    s_mul_i32 s24, s16, s13
933; GFX7-NEXT:    v_readfirstlane_b32 s35, v2
934; GFX7-NEXT:    s_add_u32 s24, s24, s25
935; GFX7-NEXT:    v_readfirstlane_b32 s37, v1
936; GFX7-NEXT:    v_mul_hi_u32 v1, v3, s10
937; GFX7-NEXT:    s_addc_u32 s22, s35, s22
938; GFX7-NEXT:    s_mul_i32 s35, s1, s12
939; GFX7-NEXT:    s_cselect_b32 s25, 1, 0
940; GFX7-NEXT:    s_add_u32 s24, s35, s24
941; GFX7-NEXT:    s_addc_u32 s22, s36, s22
942; GFX7-NEXT:    s_mul_i32 s36, s2, s11
943; GFX7-NEXT:    s_cselect_b32 s35, 1, 0
944; GFX7-NEXT:    s_add_u32 s24, s36, s24
945; GFX7-NEXT:    v_readfirstlane_b32 s38, v1
946; GFX7-NEXT:    v_mul_hi_u32 v1, v5, s9
947; GFX7-NEXT:    s_addc_u32 s22, s37, s22
948; GFX7-NEXT:    s_mul_i32 s37, s3, s10
949; GFX7-NEXT:    s_cselect_b32 s36, 1, 0
950; GFX7-NEXT:    s_add_u32 s24, s37, s24
951; GFX7-NEXT:    v_mul_hi_u32 v0, v0, s8
952; GFX7-NEXT:    s_addc_u32 s22, s38, s22
953; GFX7-NEXT:    s_mul_i32 s38, s4, s9
954; GFX7-NEXT:    s_cselect_b32 s37, 1, 0
955; GFX7-NEXT:    v_readfirstlane_b32 s39, v1
956; GFX7-NEXT:    s_add_u32 s24, s38, s24
957; GFX7-NEXT:    s_addc_u32 s22, s39, s22
958; GFX7-NEXT:    s_mul_i32 s39, s5, s8
959; GFX7-NEXT:    s_cselect_b32 s38, 1, 0
960; GFX7-NEXT:    v_readfirstlane_b32 s40, v0
961; GFX7-NEXT:    s_add_u32 s24, s39, s24
962; GFX7-NEXT:    s_addc_u32 s22, s40, s22
963; GFX7-NEXT:    s_cselect_b32 s39, 1, 0
964; GFX7-NEXT:    s_cmp_lg_u32 s31, 0
965; GFX7-NEXT:    s_addc_u32 s30, s30, 0
966; GFX7-NEXT:    s_cmp_lg_u32 s33, 0
967; GFX7-NEXT:    s_addc_u32 s30, s30, 0
968; GFX7-NEXT:    s_cmp_lg_u32 s34, 0
969; GFX7-NEXT:    s_addc_u32 s30, s30, 0
970; GFX7-NEXT:    s_cmp_lg_u32 s21, 0
971; GFX7-NEXT:    s_addc_u32 s21, s30, s24
972; GFX7-NEXT:    s_cselect_b32 s24, 1, 0
973; GFX7-NEXT:    s_cmp_lg_u32 s27, 0
974; GFX7-NEXT:    s_addc_u32 s26, s26, 0
975; GFX7-NEXT:    s_cmp_lg_u32 s28, 0
976; GFX7-NEXT:    s_addc_u32 s26, s26, 0
977; GFX7-NEXT:    s_cmp_lg_u32 s29, 0
978; GFX7-NEXT:    s_addc_u32 s26, s26, 0
979; GFX7-NEXT:    s_cmp_lg_u32 s24, 0
980; GFX7-NEXT:    s_addc_u32 s22, s26, s22
981; GFX7-NEXT:    s_mul_i32 s16, s16, s15
982; GFX7-NEXT:    s_addc_u32 s15, s23, s16
983; GFX7-NEXT:    s_mul_i32 s1, s1, s14
984; GFX7-NEXT:    s_cmp_lg_u32 s39, 0
985; GFX7-NEXT:    s_addc_u32 s1, s15, s1
986; GFX7-NEXT:    s_mul_i32 s2, s2, s13
987; GFX7-NEXT:    s_cmp_lg_u32 s38, 0
988; GFX7-NEXT:    s_addc_u32 s1, s1, s2
989; GFX7-NEXT:    s_mul_i32 s3, s3, s12
990; GFX7-NEXT:    s_cmp_lg_u32 s37, 0
991; GFX7-NEXT:    s_addc_u32 s1, s1, s3
992; GFX7-NEXT:    s_mul_i32 s4, s4, s11
993; GFX7-NEXT:    s_cmp_lg_u32 s36, 0
994; GFX7-NEXT:    s_addc_u32 s1, s1, s4
995; GFX7-NEXT:    s_mul_i32 s5, s5, s10
996; GFX7-NEXT:    s_cmp_lg_u32 s35, 0
997; GFX7-NEXT:    s_addc_u32 s1, s1, s5
998; GFX7-NEXT:    s_mul_i32 s6, s6, s9
999; GFX7-NEXT:    s_cmp_lg_u32 s25, 0
1000; GFX7-NEXT:    s_addc_u32 s1, s1, s6
1001; GFX7-NEXT:    s_mul_i32 s7, s7, s8
1002; GFX7-NEXT:    s_mul_i32 s0, s0, s8
1003; GFX7-NEXT:    s_add_u32 s7, s7, s1
1004; GFX7-NEXT:    s_mov_b32 s1, s17
1005; GFX7-NEXT:    s_mov_b32 s2, s18
1006; GFX7-NEXT:    s_mov_b32 s3, s19
1007; GFX7-NEXT:    s_mov_b32 s4, s20
1008; GFX7-NEXT:    s_mov_b32 s5, s21
1009; GFX7-NEXT:    s_mov_b32 s6, s22
1010; GFX7-NEXT:    ; return to shader part epilog
1011;
1012; GFX8-LABEL: s_mul_i256:
1013; GFX8:       ; %bb.0:
1014; GFX8-NEXT:    s_mov_b32 s16, s0
1015; GFX8-NEXT:    v_mov_b32_e32 v0, s8
1016; GFX8-NEXT:    v_mul_hi_u32 v0, s16, v0
1017; GFX8-NEXT:    v_mov_b32_e32 v1, s9
1018; GFX8-NEXT:    v_mul_hi_u32 v2, s1, v1
1019; GFX8-NEXT:    v_mul_hi_u32 v1, s16, v1
1020; GFX8-NEXT:    v_readfirstlane_b32 s17, v0
1021; GFX8-NEXT:    v_mov_b32_e32 v0, s10
1022; GFX8-NEXT:    v_mul_hi_u32 v0, s16, v0
1023; GFX8-NEXT:    v_readfirstlane_b32 s21, v2
1024; GFX8-NEXT:    v_mov_b32_e32 v2, s2
1025; GFX8-NEXT:    v_readfirstlane_b32 s23, v1
1026; GFX8-NEXT:    v_readfirstlane_b32 s19, v0
1027; GFX8-NEXT:    v_mov_b32_e32 v0, s1
1028; GFX8-NEXT:    v_mul_hi_u32 v1, v0, s8
1029; GFX8-NEXT:    v_mul_hi_u32 v3, v2, s8
1030; GFX8-NEXT:    v_mov_b32_e32 v4, s11
1031; GFX8-NEXT:    s_mul_i32 s18, s16, s10
1032; GFX8-NEXT:    v_readfirstlane_b32 s24, v1
1033; GFX8-NEXT:    v_mov_b32_e32 v1, s12
1034; GFX8-NEXT:    v_readfirstlane_b32 s22, v3
1035; GFX8-NEXT:    v_mul_hi_u32 v3, s16, v1
1036; GFX8-NEXT:    s_mul_i32 s20, s1, s9
1037; GFX8-NEXT:    v_mul_hi_u32 v5, s1, v4
1038; GFX8-NEXT:    s_add_u32 s18, s20, s18
1039; GFX8-NEXT:    v_readfirstlane_b32 s25, v3
1040; GFX8-NEXT:    v_mul_hi_u32 v3, v2, s10
1041; GFX8-NEXT:    s_addc_u32 s19, s21, s19
1042; GFX8-NEXT:    s_mul_i32 s21, s2, s8
1043; GFX8-NEXT:    s_cselect_b32 s20, 1, 0
1044; GFX8-NEXT:    s_add_u32 s18, s21, s18
1045; GFX8-NEXT:    v_readfirstlane_b32 s28, v3
1046; GFX8-NEXT:    v_mov_b32_e32 v3, s3
1047; GFX8-NEXT:    s_addc_u32 s19, s22, s19
1048; GFX8-NEXT:    s_mul_i32 s22, s16, s9
1049; GFX8-NEXT:    v_readfirstlane_b32 s27, v5
1050; GFX8-NEXT:    v_mul_hi_u32 v5, v3, s9
1051; GFX8-NEXT:    s_cselect_b32 s21, 1, 0
1052; GFX8-NEXT:    s_add_u32 s17, s22, s17
1053; GFX8-NEXT:    s_addc_u32 s18, s23, s18
1054; GFX8-NEXT:    s_mul_i32 s23, s1, s8
1055; GFX8-NEXT:    s_cselect_b32 s22, 1, 0
1056; GFX8-NEXT:    s_add_u32 s17, s23, s17
1057; GFX8-NEXT:    s_addc_u32 s18, s24, s18
1058; GFX8-NEXT:    s_mul_i32 s24, s16, s12
1059; GFX8-NEXT:    s_mul_i32 s26, s1, s11
1060; GFX8-NEXT:    v_readfirstlane_b32 s29, v5
1061; GFX8-NEXT:    v_mov_b32_e32 v5, s4
1062; GFX8-NEXT:    s_cselect_b32 s23, 1, 0
1063; GFX8-NEXT:    s_add_u32 s24, s26, s24
1064; GFX8-NEXT:    v_mul_hi_u32 v6, v5, s8
1065; GFX8-NEXT:    s_addc_u32 s25, s27, s25
1066; GFX8-NEXT:    s_mul_i32 s27, s2, s10
1067; GFX8-NEXT:    s_cselect_b32 s26, 1, 0
1068; GFX8-NEXT:    s_add_u32 s24, s27, s24
1069; GFX8-NEXT:    v_mul_hi_u32 v0, v0, s10
1070; GFX8-NEXT:    s_addc_u32 s25, s28, s25
1071; GFX8-NEXT:    s_mul_i32 s28, s3, s9
1072; GFX8-NEXT:    s_cselect_b32 s27, 1, 0
1073; GFX8-NEXT:    s_add_u32 s24, s28, s24
1074; GFX8-NEXT:    v_readfirstlane_b32 s30, v6
1075; GFX8-NEXT:    v_mul_hi_u32 v6, s16, v4
1076; GFX8-NEXT:    s_addc_u32 s25, s29, s25
1077; GFX8-NEXT:    s_mul_i32 s29, s4, s8
1078; GFX8-NEXT:    s_cselect_b32 s28, 1, 0
1079; GFX8-NEXT:    s_add_u32 s24, s29, s24
1080; GFX8-NEXT:    v_readfirstlane_b32 s33, v0
1081; GFX8-NEXT:    v_mul_hi_u32 v0, v2, s9
1082; GFX8-NEXT:    s_addc_u32 s25, s30, s25
1083; GFX8-NEXT:    s_mul_i32 s30, s16, s11
1084; GFX8-NEXT:    s_cselect_b32 s29, 1, 0
1085; GFX8-NEXT:    v_readfirstlane_b32 s31, v6
1086; GFX8-NEXT:    s_add_u32 s19, s30, s19
1087; GFX8-NEXT:    s_addc_u32 s24, s31, s24
1088; GFX8-NEXT:    s_mul_i32 s31, s1, s10
1089; GFX8-NEXT:    s_cselect_b32 s30, 1, 0
1090; GFX8-NEXT:    s_add_u32 s19, s31, s19
1091; GFX8-NEXT:    v_readfirstlane_b32 s34, v0
1092; GFX8-NEXT:    v_mul_hi_u32 v0, v3, s8
1093; GFX8-NEXT:    s_addc_u32 s24, s33, s24
1094; GFX8-NEXT:    s_mul_i32 s33, s2, s9
1095; GFX8-NEXT:    s_cselect_b32 s31, 1, 0
1096; GFX8-NEXT:    s_add_u32 s19, s33, s19
1097; GFX8-NEXT:    s_addc_u32 s24, s34, s24
1098; GFX8-NEXT:    s_mul_i32 s34, s3, s8
1099; GFX8-NEXT:    s_cselect_b32 s33, 1, 0
1100; GFX8-NEXT:    v_readfirstlane_b32 s35, v0
1101; GFX8-NEXT:    s_add_u32 s19, s34, s19
1102; GFX8-NEXT:    v_mov_b32_e32 v0, s14
1103; GFX8-NEXT:    s_addc_u32 s24, s35, s24
1104; GFX8-NEXT:    v_mul_hi_u32 v0, s16, v0
1105; GFX8-NEXT:    s_cselect_b32 s34, 1, 0
1106; GFX8-NEXT:    s_cmp_lg_u32 s23, 0
1107; GFX8-NEXT:    s_addc_u32 s19, s22, s19
1108; GFX8-NEXT:    v_mov_b32_e32 v2, s13
1109; GFX8-NEXT:    s_cselect_b32 s22, 1, 0
1110; GFX8-NEXT:    s_cmp_lg_u32 s21, 0
1111; GFX8-NEXT:    v_mul_hi_u32 v6, s1, v2
1112; GFX8-NEXT:    s_addc_u32 s20, s20, 0
1113; GFX8-NEXT:    v_readfirstlane_b32 s23, v0
1114; GFX8-NEXT:    v_mul_hi_u32 v0, s2, v1
1115; GFX8-NEXT:    s_cmp_lg_u32 s22, 0
1116; GFX8-NEXT:    s_addc_u32 s20, s20, s24
1117; GFX8-NEXT:    s_mul_i32 s22, s16, s14
1118; GFX8-NEXT:    s_mul_i32 s24, s1, s13
1119; GFX8-NEXT:    s_cselect_b32 s21, 1, 0
1120; GFX8-NEXT:    v_readfirstlane_b32 s35, v6
1121; GFX8-NEXT:    s_add_u32 s22, s24, s22
1122; GFX8-NEXT:    s_addc_u32 s23, s35, s23
1123; GFX8-NEXT:    v_readfirstlane_b32 s35, v0
1124; GFX8-NEXT:    v_mul_hi_u32 v0, v3, s11
1125; GFX8-NEXT:    s_mul_i32 s24, s2, s12
1126; GFX8-NEXT:    s_add_u32 s22, s24, s22
1127; GFX8-NEXT:    s_addc_u32 s23, s35, s23
1128; GFX8-NEXT:    v_readfirstlane_b32 s35, v0
1129; GFX8-NEXT:    v_mul_hi_u32 v0, v5, s10
1130; GFX8-NEXT:    s_mul_i32 s24, s3, s11
1131; GFX8-NEXT:    s_add_u32 s22, s24, s22
1132; GFX8-NEXT:    s_addc_u32 s23, s35, s23
1133; GFX8-NEXT:    v_readfirstlane_b32 s35, v0
1134; GFX8-NEXT:    v_mov_b32_e32 v0, s5
1135; GFX8-NEXT:    v_mul_hi_u32 v6, v0, s9
1136; GFX8-NEXT:    s_mul_i32 s24, s4, s10
1137; GFX8-NEXT:    s_add_u32 s22, s24, s22
1138; GFX8-NEXT:    v_mul_hi_u32 v1, s1, v1
1139; GFX8-NEXT:    s_addc_u32 s23, s35, s23
1140; GFX8-NEXT:    v_readfirstlane_b32 s35, v6
1141; GFX8-NEXT:    v_mov_b32_e32 v6, s6
1142; GFX8-NEXT:    v_mul_hi_u32 v6, v6, s8
1143; GFX8-NEXT:    s_mul_i32 s24, s5, s9
1144; GFX8-NEXT:    s_add_u32 s22, s24, s22
1145; GFX8-NEXT:    v_mul_hi_u32 v2, s16, v2
1146; GFX8-NEXT:    v_readfirstlane_b32 s36, v1
1147; GFX8-NEXT:    v_mul_hi_u32 v1, s2, v4
1148; GFX8-NEXT:    s_addc_u32 s23, s35, s23
1149; GFX8-NEXT:    s_mul_i32 s24, s6, s8
1150; GFX8-NEXT:    v_readfirstlane_b32 s35, v6
1151; GFX8-NEXT:    s_add_u32 s22, s24, s22
1152; GFX8-NEXT:    s_addc_u32 s23, s35, s23
1153; GFX8-NEXT:    s_mul_i32 s24, s16, s13
1154; GFX8-NEXT:    v_readfirstlane_b32 s35, v2
1155; GFX8-NEXT:    s_add_u32 s24, s24, s25
1156; GFX8-NEXT:    v_readfirstlane_b32 s37, v1
1157; GFX8-NEXT:    v_mul_hi_u32 v1, v3, s10
1158; GFX8-NEXT:    s_addc_u32 s22, s35, s22
1159; GFX8-NEXT:    s_mul_i32 s35, s1, s12
1160; GFX8-NEXT:    s_cselect_b32 s25, 1, 0
1161; GFX8-NEXT:    s_add_u32 s24, s35, s24
1162; GFX8-NEXT:    s_addc_u32 s22, s36, s22
1163; GFX8-NEXT:    s_mul_i32 s36, s2, s11
1164; GFX8-NEXT:    s_cselect_b32 s35, 1, 0
1165; GFX8-NEXT:    s_add_u32 s24, s36, s24
1166; GFX8-NEXT:    v_readfirstlane_b32 s38, v1
1167; GFX8-NEXT:    v_mul_hi_u32 v1, v5, s9
1168; GFX8-NEXT:    s_addc_u32 s22, s37, s22
1169; GFX8-NEXT:    s_mul_i32 s37, s3, s10
1170; GFX8-NEXT:    s_cselect_b32 s36, 1, 0
1171; GFX8-NEXT:    s_add_u32 s24, s37, s24
1172; GFX8-NEXT:    v_mul_hi_u32 v0, v0, s8
1173; GFX8-NEXT:    s_addc_u32 s22, s38, s22
1174; GFX8-NEXT:    s_mul_i32 s38, s4, s9
1175; GFX8-NEXT:    s_cselect_b32 s37, 1, 0
1176; GFX8-NEXT:    v_readfirstlane_b32 s39, v1
1177; GFX8-NEXT:    s_add_u32 s24, s38, s24
1178; GFX8-NEXT:    s_addc_u32 s22, s39, s22
1179; GFX8-NEXT:    s_mul_i32 s39, s5, s8
1180; GFX8-NEXT:    s_cselect_b32 s38, 1, 0
1181; GFX8-NEXT:    v_readfirstlane_b32 s40, v0
1182; GFX8-NEXT:    s_add_u32 s24, s39, s24
1183; GFX8-NEXT:    s_addc_u32 s22, s40, s22
1184; GFX8-NEXT:    s_cselect_b32 s39, 1, 0
1185; GFX8-NEXT:    s_cmp_lg_u32 s31, 0
1186; GFX8-NEXT:    s_addc_u32 s30, s30, 0
1187; GFX8-NEXT:    s_cmp_lg_u32 s33, 0
1188; GFX8-NEXT:    s_addc_u32 s30, s30, 0
1189; GFX8-NEXT:    s_cmp_lg_u32 s34, 0
1190; GFX8-NEXT:    s_addc_u32 s30, s30, 0
1191; GFX8-NEXT:    s_cmp_lg_u32 s21, 0
1192; GFX8-NEXT:    s_addc_u32 s21, s30, s24
1193; GFX8-NEXT:    s_cselect_b32 s24, 1, 0
1194; GFX8-NEXT:    s_cmp_lg_u32 s27, 0
1195; GFX8-NEXT:    s_addc_u32 s26, s26, 0
1196; GFX8-NEXT:    s_cmp_lg_u32 s28, 0
1197; GFX8-NEXT:    s_addc_u32 s26, s26, 0
1198; GFX8-NEXT:    s_cmp_lg_u32 s29, 0
1199; GFX8-NEXT:    s_addc_u32 s26, s26, 0
1200; GFX8-NEXT:    s_cmp_lg_u32 s24, 0
1201; GFX8-NEXT:    s_addc_u32 s22, s26, s22
1202; GFX8-NEXT:    s_mul_i32 s16, s16, s15
1203; GFX8-NEXT:    s_addc_u32 s15, s23, s16
1204; GFX8-NEXT:    s_mul_i32 s1, s1, s14
1205; GFX8-NEXT:    s_cmp_lg_u32 s39, 0
1206; GFX8-NEXT:    s_addc_u32 s1, s15, s1
1207; GFX8-NEXT:    s_mul_i32 s2, s2, s13
1208; GFX8-NEXT:    s_cmp_lg_u32 s38, 0
1209; GFX8-NEXT:    s_addc_u32 s1, s1, s2
1210; GFX8-NEXT:    s_mul_i32 s3, s3, s12
1211; GFX8-NEXT:    s_cmp_lg_u32 s37, 0
1212; GFX8-NEXT:    s_addc_u32 s1, s1, s3
1213; GFX8-NEXT:    s_mul_i32 s4, s4, s11
1214; GFX8-NEXT:    s_cmp_lg_u32 s36, 0
1215; GFX8-NEXT:    s_addc_u32 s1, s1, s4
1216; GFX8-NEXT:    s_mul_i32 s5, s5, s10
1217; GFX8-NEXT:    s_cmp_lg_u32 s35, 0
1218; GFX8-NEXT:    s_addc_u32 s1, s1, s5
1219; GFX8-NEXT:    s_mul_i32 s6, s6, s9
1220; GFX8-NEXT:    s_cmp_lg_u32 s25, 0
1221; GFX8-NEXT:    s_addc_u32 s1, s1, s6
1222; GFX8-NEXT:    s_mul_i32 s7, s7, s8
1223; GFX8-NEXT:    s_mul_i32 s0, s0, s8
1224; GFX8-NEXT:    s_add_u32 s7, s7, s1
1225; GFX8-NEXT:    s_mov_b32 s1, s17
1226; GFX8-NEXT:    s_mov_b32 s2, s18
1227; GFX8-NEXT:    s_mov_b32 s3, s19
1228; GFX8-NEXT:    s_mov_b32 s4, s20
1229; GFX8-NEXT:    s_mov_b32 s5, s21
1230; GFX8-NEXT:    s_mov_b32 s6, s22
1231; GFX8-NEXT:    ; return to shader part epilog
1232;
1233; GFX9-LABEL: s_mul_i256:
1234; GFX9:       ; %bb.0:
1235; GFX9-NEXT:    s_mul_i32 s18, s0, s10
1236; GFX9-NEXT:    s_mul_i32 s20, s1, s9
1237; GFX9-NEXT:    s_mul_hi_u32 s19, s0, s10
1238; GFX9-NEXT:    s_mul_hi_u32 s21, s1, s9
1239; GFX9-NEXT:    s_add_u32 s18, s20, s18
1240; GFX9-NEXT:    s_addc_u32 s19, s21, s19
1241; GFX9-NEXT:    s_mul_i32 s21, s2, s8
1242; GFX9-NEXT:    s_cselect_b32 s20, 1, 0
1243; GFX9-NEXT:    s_mul_hi_u32 s22, s2, s8
1244; GFX9-NEXT:    s_add_u32 s18, s21, s18
1245; GFX9-NEXT:    s_mul_hi_u32 s17, s0, s8
1246; GFX9-NEXT:    s_addc_u32 s19, s22, s19
1247; GFX9-NEXT:    s_mul_i32 s22, s0, s9
1248; GFX9-NEXT:    s_cselect_b32 s21, 1, 0
1249; GFX9-NEXT:    s_mul_hi_u32 s23, s0, s9
1250; GFX9-NEXT:    s_add_u32 s17, s22, s17
1251; GFX9-NEXT:    s_addc_u32 s18, s23, s18
1252; GFX9-NEXT:    s_mul_i32 s23, s1, s8
1253; GFX9-NEXT:    s_cselect_b32 s22, 1, 0
1254; GFX9-NEXT:    s_mul_hi_u32 s24, s1, s8
1255; GFX9-NEXT:    s_add_u32 s17, s23, s17
1256; GFX9-NEXT:    s_addc_u32 s18, s24, s18
1257; GFX9-NEXT:    s_mul_i32 s24, s0, s12
1258; GFX9-NEXT:    s_mul_i32 s26, s1, s11
1259; GFX9-NEXT:    s_cselect_b32 s23, 1, 0
1260; GFX9-NEXT:    s_mul_hi_u32 s25, s0, s12
1261; GFX9-NEXT:    s_mul_hi_u32 s27, s1, s11
1262; GFX9-NEXT:    s_add_u32 s24, s26, s24
1263; GFX9-NEXT:    s_addc_u32 s25, s27, s25
1264; GFX9-NEXT:    s_mul_i32 s27, s2, s10
1265; GFX9-NEXT:    s_cselect_b32 s26, 1, 0
1266; GFX9-NEXT:    s_mul_hi_u32 s28, s2, s10
1267; GFX9-NEXT:    s_add_u32 s24, s27, s24
1268; GFX9-NEXT:    s_addc_u32 s25, s28, s25
1269; GFX9-NEXT:    s_mul_i32 s28, s3, s9
1270; GFX9-NEXT:    s_cselect_b32 s27, 1, 0
1271; GFX9-NEXT:    s_mul_hi_u32 s29, s3, s9
1272; GFX9-NEXT:    s_add_u32 s24, s28, s24
1273; GFX9-NEXT:    s_addc_u32 s25, s29, s25
1274; GFX9-NEXT:    s_mul_i32 s29, s4, s8
1275; GFX9-NEXT:    s_cselect_b32 s28, 1, 0
1276; GFX9-NEXT:    s_mul_hi_u32 s30, s4, s8
1277; GFX9-NEXT:    s_add_u32 s24, s29, s24
1278; GFX9-NEXT:    s_addc_u32 s25, s30, s25
1279; GFX9-NEXT:    s_mul_i32 s30, s0, s11
1280; GFX9-NEXT:    s_cselect_b32 s29, 1, 0
1281; GFX9-NEXT:    s_mul_hi_u32 s31, s0, s11
1282; GFX9-NEXT:    s_add_u32 s19, s30, s19
1283; GFX9-NEXT:    s_addc_u32 s24, s31, s24
1284; GFX9-NEXT:    s_mul_i32 s31, s1, s10
1285; GFX9-NEXT:    s_cselect_b32 s30, 1, 0
1286; GFX9-NEXT:    s_mul_hi_u32 s33, s1, s10
1287; GFX9-NEXT:    s_add_u32 s19, s31, s19
1288; GFX9-NEXT:    s_addc_u32 s24, s33, s24
1289; GFX9-NEXT:    s_mul_i32 s33, s2, s9
1290; GFX9-NEXT:    s_cselect_b32 s31, 1, 0
1291; GFX9-NEXT:    s_mul_hi_u32 s34, s2, s9
1292; GFX9-NEXT:    s_add_u32 s19, s33, s19
1293; GFX9-NEXT:    s_addc_u32 s24, s34, s24
1294; GFX9-NEXT:    s_mul_i32 s34, s3, s8
1295; GFX9-NEXT:    s_cselect_b32 s33, 1, 0
1296; GFX9-NEXT:    s_mul_hi_u32 s35, s3, s8
1297; GFX9-NEXT:    s_add_u32 s19, s34, s19
1298; GFX9-NEXT:    s_addc_u32 s24, s35, s24
1299; GFX9-NEXT:    s_cselect_b32 s34, 1, 0
1300; GFX9-NEXT:    s_cmp_lg_u32 s23, 0
1301; GFX9-NEXT:    s_addc_u32 s19, s22, s19
1302; GFX9-NEXT:    s_cselect_b32 s22, 1, 0
1303; GFX9-NEXT:    s_cmp_lg_u32 s21, 0
1304; GFX9-NEXT:    s_addc_u32 s20, s20, 0
1305; GFX9-NEXT:    s_cmp_lg_u32 s22, 0
1306; GFX9-NEXT:    s_addc_u32 s20, s20, s24
1307; GFX9-NEXT:    s_mul_i32 s22, s0, s14
1308; GFX9-NEXT:    s_mul_i32 s24, s1, s13
1309; GFX9-NEXT:    s_cselect_b32 s21, 1, 0
1310; GFX9-NEXT:    s_mul_hi_u32 s23, s0, s14
1311; GFX9-NEXT:    s_mul_hi_u32 s35, s1, s13
1312; GFX9-NEXT:    s_add_u32 s22, s24, s22
1313; GFX9-NEXT:    s_addc_u32 s23, s35, s23
1314; GFX9-NEXT:    s_mul_i32 s24, s2, s12
1315; GFX9-NEXT:    s_mul_hi_u32 s35, s2, s12
1316; GFX9-NEXT:    s_add_u32 s22, s24, s22
1317; GFX9-NEXT:    s_addc_u32 s23, s35, s23
1318; GFX9-NEXT:    s_mul_i32 s24, s3, s11
1319; GFX9-NEXT:    s_mul_hi_u32 s35, s3, s11
1320; GFX9-NEXT:    s_add_u32 s22, s24, s22
1321; GFX9-NEXT:    s_addc_u32 s23, s35, s23
1322; GFX9-NEXT:    s_mul_i32 s24, s4, s10
1323; GFX9-NEXT:    s_mul_hi_u32 s35, s4, s10
1324; GFX9-NEXT:    s_add_u32 s22, s24, s22
1325; GFX9-NEXT:    s_addc_u32 s23, s35, s23
1326; GFX9-NEXT:    s_mul_i32 s24, s5, s9
1327; GFX9-NEXT:    s_mul_hi_u32 s35, s5, s9
1328; GFX9-NEXT:    s_add_u32 s22, s24, s22
1329; GFX9-NEXT:    s_addc_u32 s23, s35, s23
1330; GFX9-NEXT:    s_mul_i32 s24, s6, s8
1331; GFX9-NEXT:    s_mul_hi_u32 s35, s6, s8
1332; GFX9-NEXT:    s_add_u32 s22, s24, s22
1333; GFX9-NEXT:    s_addc_u32 s23, s35, s23
1334; GFX9-NEXT:    s_mul_i32 s24, s0, s13
1335; GFX9-NEXT:    s_mul_hi_u32 s35, s0, s13
1336; GFX9-NEXT:    s_add_u32 s24, s24, s25
1337; GFX9-NEXT:    s_addc_u32 s22, s35, s22
1338; GFX9-NEXT:    s_mul_i32 s35, s1, s12
1339; GFX9-NEXT:    s_cselect_b32 s25, 1, 0
1340; GFX9-NEXT:    s_mul_hi_u32 s36, s1, s12
1341; GFX9-NEXT:    s_add_u32 s24, s35, s24
1342; GFX9-NEXT:    s_addc_u32 s22, s36, s22
1343; GFX9-NEXT:    s_mul_i32 s36, s2, s11
1344; GFX9-NEXT:    s_cselect_b32 s35, 1, 0
1345; GFX9-NEXT:    s_mul_hi_u32 s37, s2, s11
1346; GFX9-NEXT:    s_add_u32 s24, s36, s24
1347; GFX9-NEXT:    s_addc_u32 s22, s37, s22
1348; GFX9-NEXT:    s_mul_i32 s37, s3, s10
1349; GFX9-NEXT:    s_cselect_b32 s36, 1, 0
1350; GFX9-NEXT:    s_mul_hi_u32 s38, s3, s10
1351; GFX9-NEXT:    s_add_u32 s24, s37, s24
1352; GFX9-NEXT:    s_addc_u32 s22, s38, s22
1353; GFX9-NEXT:    s_mul_i32 s38, s4, s9
1354; GFX9-NEXT:    s_cselect_b32 s37, 1, 0
1355; GFX9-NEXT:    s_mul_hi_u32 s39, s4, s9
1356; GFX9-NEXT:    s_add_u32 s24, s38, s24
1357; GFX9-NEXT:    s_addc_u32 s22, s39, s22
1358; GFX9-NEXT:    s_mul_i32 s39, s5, s8
1359; GFX9-NEXT:    s_cselect_b32 s38, 1, 0
1360; GFX9-NEXT:    s_mul_hi_u32 s40, s5, s8
1361; GFX9-NEXT:    s_add_u32 s24, s39, s24
1362; GFX9-NEXT:    s_addc_u32 s22, s40, s22
1363; GFX9-NEXT:    s_cselect_b32 s39, 1, 0
1364; GFX9-NEXT:    s_cmp_lg_u32 s31, 0
1365; GFX9-NEXT:    s_addc_u32 s30, s30, 0
1366; GFX9-NEXT:    s_cmp_lg_u32 s33, 0
1367; GFX9-NEXT:    s_addc_u32 s30, s30, 0
1368; GFX9-NEXT:    s_cmp_lg_u32 s34, 0
1369; GFX9-NEXT:    s_addc_u32 s30, s30, 0
1370; GFX9-NEXT:    s_cmp_lg_u32 s21, 0
1371; GFX9-NEXT:    s_addc_u32 s21, s30, s24
1372; GFX9-NEXT:    s_cselect_b32 s24, 1, 0
1373; GFX9-NEXT:    s_cmp_lg_u32 s27, 0
1374; GFX9-NEXT:    s_addc_u32 s26, s26, 0
1375; GFX9-NEXT:    s_cmp_lg_u32 s28, 0
1376; GFX9-NEXT:    s_addc_u32 s26, s26, 0
1377; GFX9-NEXT:    s_cmp_lg_u32 s29, 0
1378; GFX9-NEXT:    s_addc_u32 s26, s26, 0
1379; GFX9-NEXT:    s_cmp_lg_u32 s24, 0
1380; GFX9-NEXT:    s_mul_i32 s16, s0, s8
1381; GFX9-NEXT:    s_addc_u32 s22, s26, s22
1382; GFX9-NEXT:    s_mul_i32 s0, s0, s15
1383; GFX9-NEXT:    s_addc_u32 s0, s23, s0
1384; GFX9-NEXT:    s_mul_i32 s1, s1, s14
1385; GFX9-NEXT:    s_cmp_lg_u32 s39, 0
1386; GFX9-NEXT:    s_addc_u32 s0, s0, s1
1387; GFX9-NEXT:    s_mul_i32 s2, s2, s13
1388; GFX9-NEXT:    s_cmp_lg_u32 s38, 0
1389; GFX9-NEXT:    s_addc_u32 s0, s0, s2
1390; GFX9-NEXT:    s_mul_i32 s3, s3, s12
1391; GFX9-NEXT:    s_cmp_lg_u32 s37, 0
1392; GFX9-NEXT:    s_addc_u32 s0, s0, s3
1393; GFX9-NEXT:    s_mul_i32 s4, s4, s11
1394; GFX9-NEXT:    s_cmp_lg_u32 s36, 0
1395; GFX9-NEXT:    s_addc_u32 s0, s0, s4
1396; GFX9-NEXT:    s_mul_i32 s5, s5, s10
1397; GFX9-NEXT:    s_cmp_lg_u32 s35, 0
1398; GFX9-NEXT:    s_addc_u32 s0, s0, s5
1399; GFX9-NEXT:    s_mul_i32 s6, s6, s9
1400; GFX9-NEXT:    s_cmp_lg_u32 s25, 0
1401; GFX9-NEXT:    s_addc_u32 s0, s0, s6
1402; GFX9-NEXT:    s_mul_i32 s7, s7, s8
1403; GFX9-NEXT:    s_add_u32 s7, s7, s0
1404; GFX9-NEXT:    s_mov_b32 s0, s16
1405; GFX9-NEXT:    s_mov_b32 s1, s17
1406; GFX9-NEXT:    s_mov_b32 s2, s18
1407; GFX9-NEXT:    s_mov_b32 s3, s19
1408; GFX9-NEXT:    s_mov_b32 s4, s20
1409; GFX9-NEXT:    s_mov_b32 s5, s21
1410; GFX9-NEXT:    s_mov_b32 s6, s22
1411; GFX9-NEXT:    ; return to shader part epilog
1412;
1413; GFX10-LABEL: s_mul_i256:
1414; GFX10:       ; %bb.0:
1415; GFX10-NEXT:    s_mul_i32 s17, s0, s10
1416; GFX10-NEXT:    s_mul_i32 s19, s1, s9
1417; GFX10-NEXT:    s_mul_hi_u32 s18, s0, s10
1418; GFX10-NEXT:    s_mul_hi_u32 s20, s1, s9
1419; GFX10-NEXT:    s_add_u32 s17, s19, s17
1420; GFX10-NEXT:    s_addc_u32 s18, s20, s18
1421; GFX10-NEXT:    s_mul_i32 s20, s2, s8
1422; GFX10-NEXT:    s_mul_hi_u32 s21, s2, s8
1423; GFX10-NEXT:    s_cselect_b32 s19, 1, 0
1424; GFX10-NEXT:    s_add_u32 s17, s20, s17
1425; GFX10-NEXT:    s_mul_hi_u32 s16, s0, s8
1426; GFX10-NEXT:    s_addc_u32 s18, s21, s18
1427; GFX10-NEXT:    s_mul_i32 s21, s0, s9
1428; GFX10-NEXT:    s_mul_hi_u32 s22, s0, s9
1429; GFX10-NEXT:    s_cselect_b32 s20, 1, 0
1430; GFX10-NEXT:    s_add_u32 s16, s21, s16
1431; GFX10-NEXT:    s_addc_u32 s17, s22, s17
1432; GFX10-NEXT:    s_mul_i32 s22, s1, s8
1433; GFX10-NEXT:    s_mul_hi_u32 s23, s1, s8
1434; GFX10-NEXT:    s_cselect_b32 s21, 1, 0
1435; GFX10-NEXT:    s_add_u32 s16, s22, s16
1436; GFX10-NEXT:    s_addc_u32 s17, s23, s17
1437; GFX10-NEXT:    s_mul_i32 s23, s0, s12
1438; GFX10-NEXT:    s_mul_i32 s25, s1, s11
1439; GFX10-NEXT:    s_mul_hi_u32 s24, s0, s12
1440; GFX10-NEXT:    s_mul_hi_u32 s26, s1, s11
1441; GFX10-NEXT:    s_cselect_b32 s22, 1, 0
1442; GFX10-NEXT:    s_add_u32 s23, s25, s23
1443; GFX10-NEXT:    s_addc_u32 s24, s26, s24
1444; GFX10-NEXT:    s_mul_i32 s26, s2, s10
1445; GFX10-NEXT:    s_mul_hi_u32 s27, s2, s10
1446; GFX10-NEXT:    s_cselect_b32 s25, 1, 0
1447; GFX10-NEXT:    s_add_u32 s23, s26, s23
1448; GFX10-NEXT:    s_addc_u32 s24, s27, s24
1449; GFX10-NEXT:    s_mul_i32 s27, s3, s9
1450; GFX10-NEXT:    s_mul_hi_u32 s28, s3, s9
1451; GFX10-NEXT:    s_cselect_b32 s26, 1, 0
1452; GFX10-NEXT:    s_add_u32 s23, s27, s23
1453; GFX10-NEXT:    s_addc_u32 s24, s28, s24
1454; GFX10-NEXT:    s_mul_i32 s28, s4, s8
1455; GFX10-NEXT:    s_mul_hi_u32 s29, s4, s8
1456; GFX10-NEXT:    s_cselect_b32 s27, 1, 0
1457; GFX10-NEXT:    s_add_u32 s23, s28, s23
1458; GFX10-NEXT:    s_addc_u32 s24, s29, s24
1459; GFX10-NEXT:    s_mul_i32 s29, s0, s11
1460; GFX10-NEXT:    s_mul_hi_u32 s30, s0, s11
1461; GFX10-NEXT:    s_cselect_b32 s28, 1, 0
1462; GFX10-NEXT:    s_add_u32 s18, s29, s18
1463; GFX10-NEXT:    s_addc_u32 s23, s30, s23
1464; GFX10-NEXT:    s_mul_i32 s30, s1, s10
1465; GFX10-NEXT:    s_mul_hi_u32 s31, s1, s10
1466; GFX10-NEXT:    s_cselect_b32 s29, 1, 0
1467; GFX10-NEXT:    s_add_u32 s18, s30, s18
1468; GFX10-NEXT:    s_addc_u32 s23, s31, s23
1469; GFX10-NEXT:    s_mul_i32 s31, s2, s9
1470; GFX10-NEXT:    s_mul_hi_u32 s33, s2, s9
1471; GFX10-NEXT:    s_cselect_b32 s30, 1, 0
1472; GFX10-NEXT:    s_add_u32 s18, s31, s18
1473; GFX10-NEXT:    s_addc_u32 s23, s33, s23
1474; GFX10-NEXT:    s_mul_i32 s33, s3, s8
1475; GFX10-NEXT:    s_mul_hi_u32 s34, s3, s8
1476; GFX10-NEXT:    s_cselect_b32 s31, 1, 0
1477; GFX10-NEXT:    s_add_u32 s18, s33, s18
1478; GFX10-NEXT:    s_addc_u32 s23, s34, s23
1479; GFX10-NEXT:    s_cselect_b32 s33, 1, 0
1480; GFX10-NEXT:    s_cmp_lg_u32 s22, 0
1481; GFX10-NEXT:    s_mul_hi_u32 s22, s0, s14
1482; GFX10-NEXT:    s_addc_u32 s18, s21, s18
1483; GFX10-NEXT:    s_cselect_b32 s21, 1, 0
1484; GFX10-NEXT:    s_cmp_lg_u32 s20, 0
1485; GFX10-NEXT:    s_mul_hi_u32 s34, s1, s13
1486; GFX10-NEXT:    s_addc_u32 s19, s19, 0
1487; GFX10-NEXT:    s_cmp_lg_u32 s21, 0
1488; GFX10-NEXT:    s_mul_i32 s21, s0, s14
1489; GFX10-NEXT:    s_addc_u32 s19, s19, s23
1490; GFX10-NEXT:    s_mul_i32 s23, s1, s13
1491; GFX10-NEXT:    s_cselect_b32 s20, 1, 0
1492; GFX10-NEXT:    s_add_u32 s21, s23, s21
1493; GFX10-NEXT:    s_mul_i32 s23, s2, s12
1494; GFX10-NEXT:    s_addc_u32 s22, s34, s22
1495; GFX10-NEXT:    s_mul_hi_u32 s34, s2, s12
1496; GFX10-NEXT:    s_add_u32 s21, s23, s21
1497; GFX10-NEXT:    s_mul_i32 s23, s3, s11
1498; GFX10-NEXT:    s_addc_u32 s22, s34, s22
1499; GFX10-NEXT:    s_mul_hi_u32 s34, s3, s11
1500; GFX10-NEXT:    s_add_u32 s21, s23, s21
1501; GFX10-NEXT:    s_mul_i32 s23, s4, s10
1502; GFX10-NEXT:    s_addc_u32 s22, s34, s22
1503; GFX10-NEXT:    s_mul_hi_u32 s34, s4, s10
1504; GFX10-NEXT:    s_add_u32 s21, s23, s21
1505; GFX10-NEXT:    s_mul_i32 s23, s5, s9
1506; GFX10-NEXT:    s_addc_u32 s22, s34, s22
1507; GFX10-NEXT:    s_mul_hi_u32 s34, s5, s9
1508; GFX10-NEXT:    s_add_u32 s21, s23, s21
1509; GFX10-NEXT:    s_mul_i32 s23, s6, s8
1510; GFX10-NEXT:    s_addc_u32 s22, s34, s22
1511; GFX10-NEXT:    s_mul_hi_u32 s34, s6, s8
1512; GFX10-NEXT:    s_add_u32 s21, s23, s21
1513; GFX10-NEXT:    s_mul_i32 s23, s0, s13
1514; GFX10-NEXT:    s_addc_u32 s22, s34, s22
1515; GFX10-NEXT:    s_mul_hi_u32 s34, s0, s13
1516; GFX10-NEXT:    s_add_u32 s23, s23, s24
1517; GFX10-NEXT:    s_addc_u32 s21, s34, s21
1518; GFX10-NEXT:    s_mul_i32 s34, s1, s12
1519; GFX10-NEXT:    s_mul_hi_u32 s35, s1, s12
1520; GFX10-NEXT:    s_cselect_b32 s24, 1, 0
1521; GFX10-NEXT:    s_add_u32 s23, s34, s23
1522; GFX10-NEXT:    s_addc_u32 s21, s35, s21
1523; GFX10-NEXT:    s_mul_i32 s35, s2, s11
1524; GFX10-NEXT:    s_mul_hi_u32 s36, s2, s11
1525; GFX10-NEXT:    s_cselect_b32 s34, 1, 0
1526; GFX10-NEXT:    s_add_u32 s23, s35, s23
1527; GFX10-NEXT:    s_addc_u32 s21, s36, s21
1528; GFX10-NEXT:    s_mul_i32 s36, s3, s10
1529; GFX10-NEXT:    s_mul_hi_u32 s37, s3, s10
1530; GFX10-NEXT:    s_cselect_b32 s35, 1, 0
1531; GFX10-NEXT:    s_add_u32 s23, s36, s23
1532; GFX10-NEXT:    s_addc_u32 s21, s37, s21
1533; GFX10-NEXT:    s_mul_i32 s37, s4, s9
1534; GFX10-NEXT:    s_mul_hi_u32 s38, s4, s9
1535; GFX10-NEXT:    s_cselect_b32 s36, 1, 0
1536; GFX10-NEXT:    s_add_u32 s23, s37, s23
1537; GFX10-NEXT:    s_addc_u32 s21, s38, s21
1538; GFX10-NEXT:    s_mul_i32 s38, s5, s8
1539; GFX10-NEXT:    s_mul_hi_u32 s39, s5, s8
1540; GFX10-NEXT:    s_cselect_b32 s37, 1, 0
1541; GFX10-NEXT:    s_add_u32 s23, s38, s23
1542; GFX10-NEXT:    s_addc_u32 s21, s39, s21
1543; GFX10-NEXT:    s_cselect_b32 s38, 1, 0
1544; GFX10-NEXT:    s_cmp_lg_u32 s30, 0
1545; GFX10-NEXT:    s_mul_i32 s1, s1, s14
1546; GFX10-NEXT:    s_addc_u32 s29, s29, 0
1547; GFX10-NEXT:    s_cmp_lg_u32 s31, 0
1548; GFX10-NEXT:    s_mul_i32 s2, s2, s13
1549; GFX10-NEXT:    s_addc_u32 s29, s29, 0
1550; GFX10-NEXT:    s_cmp_lg_u32 s33, 0
1551; GFX10-NEXT:    s_mul_i32 s3, s3, s12
1552; GFX10-NEXT:    s_addc_u32 s29, s29, 0
1553; GFX10-NEXT:    s_cmp_lg_u32 s20, 0
1554; GFX10-NEXT:    s_mul_i32 s4, s4, s11
1555; GFX10-NEXT:    s_addc_u32 s20, s29, s23
1556; GFX10-NEXT:    s_cselect_b32 s23, 1, 0
1557; GFX10-NEXT:    s_cmp_lg_u32 s26, 0
1558; GFX10-NEXT:    s_mul_i32 s26, s0, s15
1559; GFX10-NEXT:    s_addc_u32 s25, s25, 0
1560; GFX10-NEXT:    s_cmp_lg_u32 s27, 0
1561; GFX10-NEXT:    s_mul_i32 s5, s5, s10
1562; GFX10-NEXT:    s_addc_u32 s25, s25, 0
1563; GFX10-NEXT:    s_cmp_lg_u32 s28, 0
1564; GFX10-NEXT:    s_mul_i32 s6, s6, s9
1565; GFX10-NEXT:    s_addc_u32 s25, s25, 0
1566; GFX10-NEXT:    s_cmp_lg_u32 s23, 0
1567; GFX10-NEXT:    s_mul_i32 s7, s7, s8
1568; GFX10-NEXT:    s_addc_u32 s15, s25, s21
1569; GFX10-NEXT:    s_addc_u32 s21, s22, s26
1570; GFX10-NEXT:    s_cmp_lg_u32 s38, 0
1571; GFX10-NEXT:    s_mul_i32 s0, s0, s8
1572; GFX10-NEXT:    s_addc_u32 s1, s21, s1
1573; GFX10-NEXT:    s_cmp_lg_u32 s37, 0
1574; GFX10-NEXT:    s_addc_u32 s1, s1, s2
1575; GFX10-NEXT:    s_cmp_lg_u32 s36, 0
1576; GFX10-NEXT:    s_mov_b32 s2, s17
1577; GFX10-NEXT:    s_addc_u32 s1, s1, s3
1578; GFX10-NEXT:    s_cmp_lg_u32 s35, 0
1579; GFX10-NEXT:    s_mov_b32 s3, s18
1580; GFX10-NEXT:    s_addc_u32 s1, s1, s4
1581; GFX10-NEXT:    s_cmp_lg_u32 s34, 0
1582; GFX10-NEXT:    s_mov_b32 s4, s19
1583; GFX10-NEXT:    s_addc_u32 s1, s1, s5
1584; GFX10-NEXT:    s_cmp_lg_u32 s24, 0
1585; GFX10-NEXT:    s_mov_b32 s5, s20
1586; GFX10-NEXT:    s_addc_u32 s1, s1, s6
1587; GFX10-NEXT:    s_mov_b32 s6, s15
1588; GFX10-NEXT:    s_add_i32 s7, s1, s7
1589; GFX10-NEXT:    s_mov_b32 s1, s16
1590; GFX10-NEXT:    ; return to shader part epilog
1591  %result = mul i256 %num, %den
1592  %cast = bitcast i256 %result to <8 x i32>
1593  ret <8 x i32> %cast
1594}
1595
1596define i256 @v_mul_i256(i256 %num, i256 %den) {
1597; GFX7-LABEL: v_mul_i256:
1598; GFX7:       ; %bb.0:
1599; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1600; GFX7-NEXT:    v_mov_b32_e32 v16, v0
1601; GFX7-NEXT:    v_mov_b32_e32 v17, v1
1602; GFX7-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v16, v14, 0
1603; GFX7-NEXT:    v_mad_u64_u32 v[18:19], s[4:5], v16, v12, 0
1604; GFX7-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v17, v13, v[0:1]
1605; GFX7-NEXT:    v_mul_lo_u32 v27, v3, v12
1606; GFX7-NEXT:    v_mul_lo_u32 v26, v5, v10
1607; GFX7-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v2, v12, v[0:1]
1608; GFX7-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v3, v11, v[0:1]
1609; GFX7-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v4, v10, v[0:1]
1610; GFX7-NEXT:    v_mad_u64_u32 v[18:19], s[4:5], v17, v11, v[18:19]
1611; GFX7-NEXT:    v_cndmask_b32_e64 v20, 0, 1, s[4:5]
1612; GFX7-NEXT:    v_mad_u64_u32 v[0:1], s[6:7], v5, v9, v[0:1]
1613; GFX7-NEXT:    v_mad_u64_u32 v[18:19], vcc, v2, v10, v[18:19]
1614; GFX7-NEXT:    v_addc_u32_e32 v20, vcc, 0, v20, vcc
1615; GFX7-NEXT:    v_mad_u64_u32 v[18:19], vcc, v3, v9, v[18:19]
1616; GFX7-NEXT:    v_addc_u32_e32 v22, vcc, 0, v20, vcc
1617; GFX7-NEXT:    v_mad_u64_u32 v[18:19], vcc, v4, v8, v[18:19]
1618; GFX7-NEXT:    v_addc_u32_e32 v24, vcc, 0, v22, vcc
1619; GFX7-NEXT:    v_mad_u64_u32 v[22:23], s[4:5], v16, v10, 0
1620; GFX7-NEXT:    v_mad_u64_u32 v[20:21], s[4:5], v6, v8, v[0:1]
1621; GFX7-NEXT:    v_mad_u64_u32 v[22:23], s[4:5], v17, v9, v[22:23]
1622; GFX7-NEXT:    v_mov_b32_e32 v1, v18
1623; GFX7-NEXT:    v_mov_b32_e32 v18, v19
1624; GFX7-NEXT:    v_mov_b32_e32 v19, v20
1625; GFX7-NEXT:    v_mad_u64_u32 v[18:19], vcc, v16, v13, v[18:19]
1626; GFX7-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[4:5]
1627; GFX7-NEXT:    v_mad_u64_u32 v[22:23], s[4:5], v2, v8, v[22:23]
1628; GFX7-NEXT:    v_addc_u32_e64 v25, s[4:5], 0, v0, s[4:5]
1629; GFX7-NEXT:    v_mad_u64_u32 v[18:19], s[4:5], v17, v12, v[18:19]
1630; GFX7-NEXT:    v_mov_b32_e32 v0, v23
1631; GFX7-NEXT:    v_mul_lo_u32 v23, v4, v11
1632; GFX7-NEXT:    v_mad_u64_u32 v[18:19], s[6:7], v2, v11, v[18:19]
1633; GFX7-NEXT:    v_mad_u64_u32 v[0:1], s[12:13], v16, v11, v[0:1]
1634; GFX7-NEXT:    v_mad_u64_u32 v[11:12], s[8:9], v3, v10, v[18:19]
1635; GFX7-NEXT:    v_mul_lo_u32 v13, v2, v13
1636; GFX7-NEXT:    v_mul_lo_u32 v20, v6, v9
1637; GFX7-NEXT:    v_mad_u64_u32 v[11:12], s[10:11], v4, v9, v[11:12]
1638; GFX7-NEXT:    v_cndmask_b32_e64 v4, 0, 1, s[12:13]
1639; GFX7-NEXT:    v_mad_u64_u32 v[0:1], s[12:13], v17, v10, v[0:1]
1640; GFX7-NEXT:    v_addc_u32_e64 v4, s[12:13], 0, v4, s[12:13]
1641; GFX7-NEXT:    v_mad_u64_u32 v[18:19], s[12:13], v2, v9, v[0:1]
1642; GFX7-NEXT:    v_mad_u64_u32 v[0:1], s[14:15], v16, v8, 0
1643; GFX7-NEXT:    v_mov_b32_e32 v2, v22
1644; GFX7-NEXT:    v_addc_u32_e64 v10, s[12:13], 0, v4, s[12:13]
1645; GFX7-NEXT:    v_mad_u64_u32 v[1:2], s[16:17], v16, v9, v[1:2]
1646; GFX7-NEXT:    v_mad_u64_u32 v[3:4], s[12:13], v3, v8, v[18:19]
1647; GFX7-NEXT:    v_mad_u64_u32 v[5:6], s[14:15], v5, v8, v[11:12]
1648; GFX7-NEXT:    v_addc_u32_e64 v10, s[12:13], 0, v10, s[12:13]
1649; GFX7-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[16:17]
1650; GFX7-NEXT:    v_mad_u64_u32 v[1:2], s[12:13], v17, v8, v[1:2]
1651; GFX7-NEXT:    v_addc_u32_e64 v3, s[12:13], v11, v3, s[12:13]
1652; GFX7-NEXT:    v_mul_lo_u32 v11, v16, v15
1653; GFX7-NEXT:    v_mul_lo_u32 v9, v17, v14
1654; GFX7-NEXT:    v_addc_u32_e64 v4, s[12:13], v25, v4, s[12:13]
1655; GFX7-NEXT:    v_addc_u32_e64 v5, s[12:13], v10, v5, s[12:13]
1656; GFX7-NEXT:    v_addc_u32_e64 v6, s[12:13], v24, v6, s[12:13]
1657; GFX7-NEXT:    v_addc_u32_e64 v10, s[12:13], v21, v11, s[12:13]
1658; GFX7-NEXT:    v_addc_u32_e64 v9, s[12:13], v10, v9, s[14:15]
1659; GFX7-NEXT:    v_addc_u32_e64 v9, s[10:11], v9, v13, s[10:11]
1660; GFX7-NEXT:    v_addc_u32_e64 v9, s[8:9], v9, v27, s[8:9]
1661; GFX7-NEXT:    v_addc_u32_e64 v9, s[6:7], v9, v23, s[6:7]
1662; GFX7-NEXT:    v_addc_u32_e64 v9, s[4:5], v9, v26, s[4:5]
1663; GFX7-NEXT:    v_addc_u32_e32 v9, vcc, v9, v20, vcc
1664; GFX7-NEXT:    v_mad_u64_u32 v[7:8], s[4:5], v7, v8, v[9:10]
1665; GFX7-NEXT:    s_setpc_b64 s[30:31]
1666;
1667; GFX8-LABEL: v_mul_i256:
1668; GFX8:       ; %bb.0:
1669; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1670; GFX8-NEXT:    v_mov_b32_e32 v16, v0
1671; GFX8-NEXT:    v_mov_b32_e32 v17, v1
1672; GFX8-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v16, v14, 0
1673; GFX8-NEXT:    v_mad_u64_u32 v[18:19], s[4:5], v16, v12, 0
1674; GFX8-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v17, v13, v[0:1]
1675; GFX8-NEXT:    v_mul_lo_u32 v27, v3, v12
1676; GFX8-NEXT:    v_mul_lo_u32 v26, v5, v10
1677; GFX8-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v2, v12, v[0:1]
1678; GFX8-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v3, v11, v[0:1]
1679; GFX8-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v4, v10, v[0:1]
1680; GFX8-NEXT:    v_mad_u64_u32 v[18:19], s[4:5], v17, v11, v[18:19]
1681; GFX8-NEXT:    v_cndmask_b32_e64 v20, 0, 1, s[4:5]
1682; GFX8-NEXT:    v_mad_u64_u32 v[0:1], s[6:7], v5, v9, v[0:1]
1683; GFX8-NEXT:    v_mad_u64_u32 v[18:19], vcc, v2, v10, v[18:19]
1684; GFX8-NEXT:    v_addc_u32_e32 v20, vcc, 0, v20, vcc
1685; GFX8-NEXT:    v_mad_u64_u32 v[18:19], vcc, v3, v9, v[18:19]
1686; GFX8-NEXT:    v_addc_u32_e32 v22, vcc, 0, v20, vcc
1687; GFX8-NEXT:    v_mad_u64_u32 v[18:19], vcc, v4, v8, v[18:19]
1688; GFX8-NEXT:    v_addc_u32_e32 v24, vcc, 0, v22, vcc
1689; GFX8-NEXT:    v_mad_u64_u32 v[22:23], s[4:5], v16, v10, 0
1690; GFX8-NEXT:    v_mad_u64_u32 v[20:21], s[4:5], v6, v8, v[0:1]
1691; GFX8-NEXT:    v_mad_u64_u32 v[22:23], s[4:5], v17, v9, v[22:23]
1692; GFX8-NEXT:    v_mov_b32_e32 v1, v18
1693; GFX8-NEXT:    v_mov_b32_e32 v18, v19
1694; GFX8-NEXT:    v_mov_b32_e32 v19, v20
1695; GFX8-NEXT:    v_mad_u64_u32 v[18:19], vcc, v16, v13, v[18:19]
1696; GFX8-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[4:5]
1697; GFX8-NEXT:    v_mad_u64_u32 v[22:23], s[4:5], v2, v8, v[22:23]
1698; GFX8-NEXT:    v_addc_u32_e64 v25, s[4:5], 0, v0, s[4:5]
1699; GFX8-NEXT:    v_mad_u64_u32 v[18:19], s[4:5], v17, v12, v[18:19]
1700; GFX8-NEXT:    v_mov_b32_e32 v0, v23
1701; GFX8-NEXT:    v_mul_lo_u32 v23, v4, v11
1702; GFX8-NEXT:    v_mad_u64_u32 v[18:19], s[6:7], v2, v11, v[18:19]
1703; GFX8-NEXT:    v_mad_u64_u32 v[0:1], s[12:13], v16, v11, v[0:1]
1704; GFX8-NEXT:    v_mad_u64_u32 v[11:12], s[8:9], v3, v10, v[18:19]
1705; GFX8-NEXT:    v_mul_lo_u32 v13, v2, v13
1706; GFX8-NEXT:    v_mul_lo_u32 v20, v6, v9
1707; GFX8-NEXT:    v_mad_u64_u32 v[11:12], s[10:11], v4, v9, v[11:12]
1708; GFX8-NEXT:    v_cndmask_b32_e64 v4, 0, 1, s[12:13]
1709; GFX8-NEXT:    v_mad_u64_u32 v[0:1], s[12:13], v17, v10, v[0:1]
1710; GFX8-NEXT:    v_addc_u32_e64 v4, s[12:13], 0, v4, s[12:13]
1711; GFX8-NEXT:    v_mad_u64_u32 v[18:19], s[12:13], v2, v9, v[0:1]
1712; GFX8-NEXT:    v_mad_u64_u32 v[0:1], s[14:15], v16, v8, 0
1713; GFX8-NEXT:    v_mov_b32_e32 v2, v22
1714; GFX8-NEXT:    v_addc_u32_e64 v10, s[12:13], 0, v4, s[12:13]
1715; GFX8-NEXT:    v_mad_u64_u32 v[1:2], s[16:17], v16, v9, v[1:2]
1716; GFX8-NEXT:    v_mad_u64_u32 v[3:4], s[12:13], v3, v8, v[18:19]
1717; GFX8-NEXT:    v_mad_u64_u32 v[5:6], s[14:15], v5, v8, v[11:12]
1718; GFX8-NEXT:    v_addc_u32_e64 v10, s[12:13], 0, v10, s[12:13]
1719; GFX8-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[16:17]
1720; GFX8-NEXT:    v_mad_u64_u32 v[1:2], s[12:13], v17, v8, v[1:2]
1721; GFX8-NEXT:    v_addc_u32_e64 v3, s[12:13], v11, v3, s[12:13]
1722; GFX8-NEXT:    v_mul_lo_u32 v11, v16, v15
1723; GFX8-NEXT:    v_mul_lo_u32 v9, v17, v14
1724; GFX8-NEXT:    v_addc_u32_e64 v4, s[12:13], v25, v4, s[12:13]
1725; GFX8-NEXT:    v_addc_u32_e64 v5, s[12:13], v10, v5, s[12:13]
1726; GFX8-NEXT:    v_addc_u32_e64 v6, s[12:13], v24, v6, s[12:13]
1727; GFX8-NEXT:    v_addc_u32_e64 v10, s[12:13], v21, v11, s[12:13]
1728; GFX8-NEXT:    v_addc_u32_e64 v9, s[12:13], v10, v9, s[14:15]
1729; GFX8-NEXT:    v_addc_u32_e64 v9, s[10:11], v9, v13, s[10:11]
1730; GFX8-NEXT:    v_addc_u32_e64 v9, s[8:9], v9, v27, s[8:9]
1731; GFX8-NEXT:    v_addc_u32_e64 v9, s[6:7], v9, v23, s[6:7]
1732; GFX8-NEXT:    v_addc_u32_e64 v9, s[4:5], v9, v26, s[4:5]
1733; GFX8-NEXT:    v_addc_u32_e32 v9, vcc, v9, v20, vcc
1734; GFX8-NEXT:    v_mad_u64_u32 v[7:8], s[4:5], v7, v8, v[9:10]
1735; GFX8-NEXT:    s_setpc_b64 s[30:31]
1736;
1737; GFX9-LABEL: v_mul_i256:
1738; GFX9:       ; %bb.0:
1739; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1740; GFX9-NEXT:    v_mov_b32_e32 v16, v0
1741; GFX9-NEXT:    v_mov_b32_e32 v17, v1
1742; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v16, v14, 0
1743; GFX9-NEXT:    v_mad_u64_u32 v[18:19], s[4:5], v16, v12, 0
1744; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v17, v13, v[0:1]
1745; GFX9-NEXT:    v_mul_lo_u32 v27, v3, v12
1746; GFX9-NEXT:    v_mul_lo_u32 v26, v5, v10
1747; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v2, v12, v[0:1]
1748; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v3, v11, v[0:1]
1749; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v4, v10, v[0:1]
1750; GFX9-NEXT:    v_mad_u64_u32 v[18:19], s[4:5], v17, v11, v[18:19]
1751; GFX9-NEXT:    v_cndmask_b32_e64 v20, 0, 1, s[4:5]
1752; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[6:7], v5, v9, v[0:1]
1753; GFX9-NEXT:    v_mad_u64_u32 v[18:19], vcc, v2, v10, v[18:19]
1754; GFX9-NEXT:    v_addc_co_u32_e32 v20, vcc, 0, v20, vcc
1755; GFX9-NEXT:    v_mad_u64_u32 v[18:19], vcc, v3, v9, v[18:19]
1756; GFX9-NEXT:    v_addc_co_u32_e32 v22, vcc, 0, v20, vcc
1757; GFX9-NEXT:    v_mad_u64_u32 v[18:19], vcc, v4, v8, v[18:19]
1758; GFX9-NEXT:    v_addc_co_u32_e32 v24, vcc, 0, v22, vcc
1759; GFX9-NEXT:    v_mad_u64_u32 v[22:23], s[4:5], v16, v10, 0
1760; GFX9-NEXT:    v_mad_u64_u32 v[20:21], s[4:5], v6, v8, v[0:1]
1761; GFX9-NEXT:    v_mad_u64_u32 v[22:23], s[4:5], v17, v9, v[22:23]
1762; GFX9-NEXT:    v_mov_b32_e32 v1, v18
1763; GFX9-NEXT:    v_mov_b32_e32 v18, v19
1764; GFX9-NEXT:    v_mov_b32_e32 v19, v20
1765; GFX9-NEXT:    v_mad_u64_u32 v[18:19], vcc, v16, v13, v[18:19]
1766; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[4:5]
1767; GFX9-NEXT:    v_mad_u64_u32 v[22:23], s[4:5], v2, v8, v[22:23]
1768; GFX9-NEXT:    v_addc_co_u32_e64 v25, s[4:5], 0, v0, s[4:5]
1769; GFX9-NEXT:    v_mad_u64_u32 v[18:19], s[4:5], v17, v12, v[18:19]
1770; GFX9-NEXT:    v_mov_b32_e32 v0, v23
1771; GFX9-NEXT:    v_mul_lo_u32 v23, v4, v11
1772; GFX9-NEXT:    v_mad_u64_u32 v[18:19], s[6:7], v2, v11, v[18:19]
1773; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[12:13], v16, v11, v[0:1]
1774; GFX9-NEXT:    v_mad_u64_u32 v[11:12], s[8:9], v3, v10, v[18:19]
1775; GFX9-NEXT:    v_mul_lo_u32 v13, v2, v13
1776; GFX9-NEXT:    v_mul_lo_u32 v20, v6, v9
1777; GFX9-NEXT:    v_mad_u64_u32 v[11:12], s[10:11], v4, v9, v[11:12]
1778; GFX9-NEXT:    v_cndmask_b32_e64 v4, 0, 1, s[12:13]
1779; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[12:13], v17, v10, v[0:1]
1780; GFX9-NEXT:    v_addc_co_u32_e64 v4, s[12:13], 0, v4, s[12:13]
1781; GFX9-NEXT:    v_mad_u64_u32 v[18:19], s[12:13], v2, v9, v[0:1]
1782; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[14:15], v16, v8, 0
1783; GFX9-NEXT:    v_mov_b32_e32 v2, v22
1784; GFX9-NEXT:    v_addc_co_u32_e64 v10, s[12:13], 0, v4, s[12:13]
1785; GFX9-NEXT:    v_mad_u64_u32 v[1:2], s[16:17], v16, v9, v[1:2]
1786; GFX9-NEXT:    v_mad_u64_u32 v[3:4], s[12:13], v3, v8, v[18:19]
1787; GFX9-NEXT:    v_mad_u64_u32 v[5:6], s[14:15], v5, v8, v[11:12]
1788; GFX9-NEXT:    v_addc_co_u32_e64 v10, s[12:13], 0, v10, s[12:13]
1789; GFX9-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[16:17]
1790; GFX9-NEXT:    v_mad_u64_u32 v[1:2], s[12:13], v17, v8, v[1:2]
1791; GFX9-NEXT:    v_addc_co_u32_e64 v3, s[12:13], v11, v3, s[12:13]
1792; GFX9-NEXT:    v_mul_lo_u32 v11, v16, v15
1793; GFX9-NEXT:    v_mul_lo_u32 v9, v17, v14
1794; GFX9-NEXT:    v_addc_co_u32_e64 v4, s[12:13], v25, v4, s[12:13]
1795; GFX9-NEXT:    v_addc_co_u32_e64 v5, s[12:13], v10, v5, s[12:13]
1796; GFX9-NEXT:    v_addc_co_u32_e64 v6, s[12:13], v24, v6, s[12:13]
1797; GFX9-NEXT:    v_addc_co_u32_e64 v10, s[12:13], v21, v11, s[12:13]
1798; GFX9-NEXT:    v_addc_co_u32_e64 v9, s[12:13], v10, v9, s[14:15]
1799; GFX9-NEXT:    v_addc_co_u32_e64 v9, s[10:11], v9, v13, s[10:11]
1800; GFX9-NEXT:    v_addc_co_u32_e64 v9, s[8:9], v9, v27, s[8:9]
1801; GFX9-NEXT:    v_addc_co_u32_e64 v9, s[6:7], v9, v23, s[6:7]
1802; GFX9-NEXT:    v_addc_co_u32_e64 v9, s[4:5], v9, v26, s[4:5]
1803; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, v9, v20, vcc
1804; GFX9-NEXT:    v_mad_u64_u32 v[7:8], s[4:5], v7, v8, v[9:10]
1805; GFX9-NEXT:    s_setpc_b64 s[30:31]
1806;
1807; GFX10-LABEL: v_mul_i256:
1808; GFX10:       ; %bb.0:
1809; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1810; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
1811; GFX10-NEXT:    v_mov_b32_e32 v16, v0
1812; GFX10-NEXT:    v_mov_b32_e32 v17, v1
1813; GFX10-NEXT:    v_mul_lo_u32 v27, v6, v9
1814; GFX10-NEXT:    v_mul_lo_u32 v28, v5, v10
1815; GFX10-NEXT:    v_mul_lo_u32 v7, v7, v8
1816; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s4, v16, v14, 0
1817; GFX10-NEXT:    v_mad_u64_u32 v[18:19], s4, v16, v12, 0
1818; GFX10-NEXT:    v_mul_lo_u32 v30, v17, v14
1819; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s4, v17, v13, v[0:1]
1820; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s4, v2, v12, v[0:1]
1821; GFX10-NEXT:    v_mad_u64_u32 v[18:19], s4, v17, v11, v[18:19]
1822; GFX10-NEXT:    v_cndmask_b32_e64 v20, 0, 1, s4
1823; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s5, v3, v11, v[0:1]
1824; GFX10-NEXT:    v_mad_u64_u32 v[18:19], vcc_lo, v2, v10, v[18:19]
1825; GFX10-NEXT:    v_add_co_ci_u32_e32 v22, vcc_lo, 0, v20, vcc_lo
1826; GFX10-NEXT:    v_mad_u64_u32 v[20:21], s4, v16, v10, 0
1827; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s4, v4, v10, v[0:1]
1828; GFX10-NEXT:    v_mad_u64_u32 v[18:19], vcc_lo, v3, v9, v[18:19]
1829; GFX10-NEXT:    v_add_co_ci_u32_e32 v24, vcc_lo, 0, v22, vcc_lo
1830; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s4, v5, v9, v[0:1]
1831; GFX10-NEXT:    v_mad_u64_u32 v[18:19], vcc_lo, v4, v8, v[18:19]
1832; GFX10-NEXT:    v_add_co_ci_u32_e32 v26, vcc_lo, 0, v24, vcc_lo
1833; GFX10-NEXT:    v_mad_u64_u32 v[22:23], s4, v6, v8, v[0:1]
1834; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s4, v17, v9, v[20:21]
1835; GFX10-NEXT:    v_cndmask_b32_e64 v25, 0, 1, s4
1836; GFX10-NEXT:    v_mov_b32_e32 v20, v22
1837; GFX10-NEXT:    v_mad_u64_u32 v[21:22], vcc_lo, v2, v8, v[0:1]
1838; GFX10-NEXT:    v_add_co_ci_u32_e32 v29, vcc_lo, 0, v25, vcc_lo
1839; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s4, v16, v13, v[19:20]
1840; GFX10-NEXT:    v_mov_b32_e32 v20, v18
1841; GFX10-NEXT:    v_mov_b32_e32 v19, v22
1842; GFX10-NEXT:    v_mul_lo_u32 v22, v16, v15
1843; GFX10-NEXT:    v_mad_u64_u32 v[24:25], vcc_lo, v17, v12, v[0:1]
1844; GFX10-NEXT:    v_mad_u64_u32 v[14:15], s6, v16, v11, v[19:20]
1845; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s5, v16, v8, 0
1846; GFX10-NEXT:    v_mul_lo_u32 v20, v4, v11
1847; GFX10-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s6
1848; GFX10-NEXT:    v_mad_u64_u32 v[18:19], s5, v2, v11, v[24:25]
1849; GFX10-NEXT:    v_mul_lo_u32 v25, v3, v12
1850; GFX10-NEXT:    v_mad_u64_u32 v[11:12], s6, v17, v10, v[14:15]
1851; GFX10-NEXT:    v_add_co_ci_u32_e64 v6, s6, 0, v6, s6
1852; GFX10-NEXT:    v_mul_lo_u32 v24, v2, v13
1853; GFX10-NEXT:    v_mad_u64_u32 v[18:19], s7, v3, v10, v[18:19]
1854; GFX10-NEXT:    v_mov_b32_e32 v13, v1
1855; GFX10-NEXT:    v_mad_u64_u32 v[1:2], s6, v2, v9, v[11:12]
1856; GFX10-NEXT:    v_mov_b32_e32 v14, v21
1857; GFX10-NEXT:    v_add_co_ci_u32_e64 v6, s6, 0, v6, s6
1858; GFX10-NEXT:    v_mad_u64_u32 v[10:11], s6, v4, v9, v[18:19]
1859; GFX10-NEXT:    v_mad_u64_u32 v[12:13], s8, v16, v9, v[13:14]
1860; GFX10-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s8
1861; GFX10-NEXT:    v_mad_u64_u32 v[3:4], s8, v3, v8, v[1:2]
1862; GFX10-NEXT:    v_add_co_ci_u32_e64 v14, s8, 0, v6, s8
1863; GFX10-NEXT:    v_mad_u64_u32 v[5:6], s8, v5, v8, v[10:11]
1864; GFX10-NEXT:    v_mad_u64_u32 v[1:2], s9, v17, v8, v[12:13]
1865; GFX10-NEXT:    v_add_co_ci_u32_e64 v3, s9, v9, v3, s9
1866; GFX10-NEXT:    v_add_co_ci_u32_e64 v4, s9, v29, v4, s9
1867; GFX10-NEXT:    v_add_co_ci_u32_e64 v5, s9, v14, v5, s9
1868; GFX10-NEXT:    v_add_co_ci_u32_e64 v6, s9, v26, v6, s9
1869; GFX10-NEXT:    v_add_co_ci_u32_e64 v9, s9, v23, v22, s9
1870; GFX10-NEXT:    v_add_co_ci_u32_e64 v9, s8, v9, v30, s8
1871; GFX10-NEXT:    v_add_co_ci_u32_e64 v9, s6, v9, v24, s6
1872; GFX10-NEXT:    v_add_co_ci_u32_e64 v9, s6, v9, v25, s7
1873; GFX10-NEXT:    v_add_co_ci_u32_e64 v9, s5, v9, v20, s5
1874; GFX10-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, v9, v28, vcc_lo
1875; GFX10-NEXT:    v_add_co_ci_u32_e64 v8, vcc_lo, v9, v27, s4
1876; GFX10-NEXT:    v_add_nc_u32_e32 v7, v8, v7
1877; GFX10-NEXT:    s_setpc_b64 s[30:31]
1878  %result = mul i256 %num, %den
1879  ret i256 %result
1880}
1881