1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -global-isel -march=amdgcn -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX7 %s
3; RUN: llc -global-isel -march=amdgcn -mcpu=gfx801 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX8 %s
4; RUN: llc -global-isel -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9 %s
5; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10 %s
6
7define amdgpu_ps i16 @s_mul_i16(i16 inreg %num, i16 inreg %den) {
8; GFX7-LABEL: s_mul_i16:
9; GFX7:       ; %bb.0:
10; GFX7-NEXT:    s_mul_i32 s0, s0, s1
11; GFX7-NEXT:    ; return to shader part epilog
12;
13; GFX8-LABEL: s_mul_i16:
14; GFX8:       ; %bb.0:
15; GFX8-NEXT:    s_mov_b32 s2, 0xffff
16; GFX8-NEXT:    s_and_b32 s0, s0, s2
17; GFX8-NEXT:    s_and_b32 s1, s1, s2
18; GFX8-NEXT:    s_mul_i32 s0, s0, s1
19; GFX8-NEXT:    ; return to shader part epilog
20;
21; GFX9-LABEL: s_mul_i16:
22; GFX9:       ; %bb.0:
23; GFX9-NEXT:    s_mov_b32 s2, 0xffff
24; GFX9-NEXT:    s_and_b32 s0, s0, s2
25; GFX9-NEXT:    s_and_b32 s1, s1, s2
26; GFX9-NEXT:    s_mul_i32 s0, s0, s1
27; GFX9-NEXT:    ; return to shader part epilog
28;
29; GFX10-LABEL: s_mul_i16:
30; GFX10:       ; %bb.0:
31; GFX10-NEXT:    s_mov_b32 s2, 0xffff
32; GFX10-NEXT:    s_and_b32 s0, s0, s2
33; GFX10-NEXT:    s_and_b32 s1, s1, s2
34; GFX10-NEXT:    s_mul_i32 s0, s0, s1
35; GFX10-NEXT:    ; return to shader part epilog
36  %result = mul i16 %num, %den
37  ret i16 %result
38}
39
40define i16 @v_mul_i16(i16 %num, i16 %den) {
41; GFX7-LABEL: v_mul_i16:
42; GFX7:       ; %bb.0:
43; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
44; GFX7-NEXT:    s_mov_b32 s4, 0xffff
45; GFX7-NEXT:    v_and_b32_e32 v0, s4, v0
46; GFX7-NEXT:    v_and_b32_e32 v1, s4, v1
47; GFX7-NEXT:    v_mul_u32_u24_e32 v0, v0, v1
48; GFX7-NEXT:    s_setpc_b64 s[30:31]
49;
50; GFX8-LABEL: v_mul_i16:
51; GFX8:       ; %bb.0:
52; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
53; GFX8-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
54; GFX8-NEXT:    s_setpc_b64 s[30:31]
55;
56; GFX9-LABEL: v_mul_i16:
57; GFX9:       ; %bb.0:
58; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
59; GFX9-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
60; GFX9-NEXT:    s_setpc_b64 s[30:31]
61;
62; GFX10-LABEL: v_mul_i16:
63; GFX10:       ; %bb.0:
64; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
65; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
66; GFX10-NEXT:    v_mul_lo_u16 v0, v0, v1
67; GFX10-NEXT:    s_setpc_b64 s[30:31]
68  %result = mul i16 %num, %den
69  ret i16 %result
70}
71
72define amdgpu_ps zeroext i16 @s_mul_i16_zeroext(i16 inreg zeroext %num, i16 inreg zeroext %den) {
73; GFX7-LABEL: s_mul_i16_zeroext:
74; GFX7:       ; %bb.0:
75; GFX7-NEXT:    s_mul_i32 s0, s0, s1
76; GFX7-NEXT:    s_and_b32 s0, s0, 0xffff
77; GFX7-NEXT:    ; return to shader part epilog
78;
79; GFX8-LABEL: s_mul_i16_zeroext:
80; GFX8:       ; %bb.0:
81; GFX8-NEXT:    s_mov_b32 s2, 0xffff
82; GFX8-NEXT:    s_and_b32 s0, s0, s2
83; GFX8-NEXT:    s_and_b32 s1, s1, s2
84; GFX8-NEXT:    s_mul_i32 s0, s0, s1
85; GFX8-NEXT:    s_and_b32 s0, s0, s2
86; GFX8-NEXT:    ; return to shader part epilog
87;
88; GFX9-LABEL: s_mul_i16_zeroext:
89; GFX9:       ; %bb.0:
90; GFX9-NEXT:    s_mov_b32 s2, 0xffff
91; GFX9-NEXT:    s_and_b32 s0, s0, s2
92; GFX9-NEXT:    s_and_b32 s1, s1, s2
93; GFX9-NEXT:    s_mul_i32 s0, s0, s1
94; GFX9-NEXT:    s_and_b32 s0, s0, s2
95; GFX9-NEXT:    ; return to shader part epilog
96;
97; GFX10-LABEL: s_mul_i16_zeroext:
98; GFX10:       ; %bb.0:
99; GFX10-NEXT:    s_mov_b32 s2, 0xffff
100; GFX10-NEXT:    s_and_b32 s0, s0, s2
101; GFX10-NEXT:    s_and_b32 s1, s1, s2
102; GFX10-NEXT:    s_mul_i32 s0, s0, s1
103; GFX10-NEXT:    s_and_b32 s0, s0, s2
104; GFX10-NEXT:    ; return to shader part epilog
105  %result = mul i16 %num, %den
106  ret i16 %result
107}
108
109define zeroext i16 @v_mul_i16_zeroext(i16 zeroext %num, i16 zeroext %den) {
110; GFX7-LABEL: v_mul_i16_zeroext:
111; GFX7:       ; %bb.0:
112; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
113; GFX7-NEXT:    v_mul_u32_u24_e32 v0, v0, v1
114; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff, v0
115; GFX7-NEXT:    s_setpc_b64 s[30:31]
116;
117; GFX8-LABEL: v_mul_i16_zeroext:
118; GFX8:       ; %bb.0:
119; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
120; GFX8-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
121; GFX8-NEXT:    s_setpc_b64 s[30:31]
122;
123; GFX9-LABEL: v_mul_i16_zeroext:
124; GFX9:       ; %bb.0:
125; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
126; GFX9-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
127; GFX9-NEXT:    s_setpc_b64 s[30:31]
128;
129; GFX10-LABEL: v_mul_i16_zeroext:
130; GFX10:       ; %bb.0:
131; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
132; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
133; GFX10-NEXT:    v_mul_lo_u16 v0, v0, v1
134; GFX10-NEXT:    v_bfe_u32 v0, v0, 0, 16
135; GFX10-NEXT:    s_setpc_b64 s[30:31]
136  %result = mul i16 %num, %den
137  ret i16 %result
138}
139
140define amdgpu_ps signext i16 @s_mul_i16_signext(i16 inreg signext %num, i16 inreg signext %den) {
141; GFX7-LABEL: s_mul_i16_signext:
142; GFX7:       ; %bb.0:
143; GFX7-NEXT:    s_mul_i32 s0, s0, s1
144; GFX7-NEXT:    s_sext_i32_i16 s0, s0
145; GFX7-NEXT:    ; return to shader part epilog
146;
147; GFX8-LABEL: s_mul_i16_signext:
148; GFX8:       ; %bb.0:
149; GFX8-NEXT:    s_mov_b32 s2, 0xffff
150; GFX8-NEXT:    s_and_b32 s0, s0, s2
151; GFX8-NEXT:    s_and_b32 s1, s1, s2
152; GFX8-NEXT:    s_mul_i32 s0, s0, s1
153; GFX8-NEXT:    s_sext_i32_i16 s0, s0
154; GFX8-NEXT:    ; return to shader part epilog
155;
156; GFX9-LABEL: s_mul_i16_signext:
157; GFX9:       ; %bb.0:
158; GFX9-NEXT:    s_mov_b32 s2, 0xffff
159; GFX9-NEXT:    s_and_b32 s0, s0, s2
160; GFX9-NEXT:    s_and_b32 s1, s1, s2
161; GFX9-NEXT:    s_mul_i32 s0, s0, s1
162; GFX9-NEXT:    s_sext_i32_i16 s0, s0
163; GFX9-NEXT:    ; return to shader part epilog
164;
165; GFX10-LABEL: s_mul_i16_signext:
166; GFX10:       ; %bb.0:
167; GFX10-NEXT:    s_mov_b32 s2, 0xffff
168; GFX10-NEXT:    s_and_b32 s0, s0, s2
169; GFX10-NEXT:    s_and_b32 s1, s1, s2
170; GFX10-NEXT:    s_mul_i32 s0, s0, s1
171; GFX10-NEXT:    s_sext_i32_i16 s0, s0
172; GFX10-NEXT:    ; return to shader part epilog
173  %result = mul i16 %num, %den
174  ret i16 %result
175}
176
177define signext i16 @v_mul_i16_signext(i16 signext %num, i16 signext %den) {
178; GFX7-LABEL: v_mul_i16_signext:
179; GFX7:       ; %bb.0:
180; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
181; GFX7-NEXT:    s_mov_b32 s4, 0xffff
182; GFX7-NEXT:    v_and_b32_e32 v0, s4, v0
183; GFX7-NEXT:    v_and_b32_e32 v1, s4, v1
184; GFX7-NEXT:    v_mul_u32_u24_e32 v0, v0, v1
185; GFX7-NEXT:    v_bfe_i32 v0, v0, 0, 16
186; GFX7-NEXT:    s_setpc_b64 s[30:31]
187;
188; GFX8-LABEL: v_mul_i16_signext:
189; GFX8:       ; %bb.0:
190; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
191; GFX8-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
192; GFX8-NEXT:    v_bfe_i32 v0, v0, 0, 16
193; GFX8-NEXT:    s_setpc_b64 s[30:31]
194;
195; GFX9-LABEL: v_mul_i16_signext:
196; GFX9:       ; %bb.0:
197; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
198; GFX9-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
199; GFX9-NEXT:    v_bfe_i32 v0, v0, 0, 16
200; GFX9-NEXT:    s_setpc_b64 s[30:31]
201;
202; GFX10-LABEL: v_mul_i16_signext:
203; GFX10:       ; %bb.0:
204; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
205; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
206; GFX10-NEXT:    v_mul_lo_u16 v0, v0, v1
207; GFX10-NEXT:    v_bfe_i32 v0, v0, 0, 16
208; GFX10-NEXT:    s_setpc_b64 s[30:31]
209  %result = mul i16 %num, %den
210  ret i16 %result
211}
212
213define amdgpu_ps i32 @s_mul_i32(i32 inreg %num, i32 inreg %den) {
214; GCN-LABEL: s_mul_i32:
215; GCN:       ; %bb.0:
216; GCN-NEXT:    s_mul_i32 s0, s0, s1
217; GCN-NEXT:    ; return to shader part epilog
218;
219; GFX10-LABEL: s_mul_i32:
220; GFX10:       ; %bb.0:
221; GFX10-NEXT:    s_mul_i32 s0, s0, s1
222; GFX10-NEXT:    ; return to shader part epilog
223  %result = mul i32 %num, %den
224  ret i32 %result
225}
226
227define i32 @v_mul_i32(i32 %num, i32 %den) {
228; GCN-LABEL: v_mul_i32:
229; GCN:       ; %bb.0:
230; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
231; GCN-NEXT:    v_mul_lo_u32 v0, v0, v1
232; GCN-NEXT:    s_setpc_b64 s[30:31]
233;
234; GFX10-LABEL: v_mul_i32:
235; GFX10:       ; %bb.0:
236; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
237; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
238; GFX10-NEXT:    v_mul_lo_u32 v0, v0, v1
239; GFX10-NEXT:    s_setpc_b64 s[30:31]
240  %result = mul i32 %num, %den
241  ret i32 %result
242}
243
244define amdgpu_ps <2 x i32> @s_mul_v2i32(<2 x i32> inreg %num, <2 x i32> inreg %den) {
245; GCN-LABEL: s_mul_v2i32:
246; GCN:       ; %bb.0:
247; GCN-NEXT:    s_mul_i32 s0, s0, s2
248; GCN-NEXT:    s_mul_i32 s1, s1, s3
249; GCN-NEXT:    ; return to shader part epilog
250;
251; GFX10-LABEL: s_mul_v2i32:
252; GFX10:       ; %bb.0:
253; GFX10-NEXT:    s_mul_i32 s0, s0, s2
254; GFX10-NEXT:    s_mul_i32 s1, s1, s3
255; GFX10-NEXT:    ; return to shader part epilog
256  %result = mul <2 x i32> %num, %den
257  ret <2 x i32> %result
258}
259
260define <2 x i32> @v_mul_v2i32(<2 x i32> %num, <2 x i32> %den) {
261; GCN-LABEL: v_mul_v2i32:
262; GCN:       ; %bb.0:
263; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
264; GCN-NEXT:    v_mul_lo_u32 v0, v0, v2
265; GCN-NEXT:    v_mul_lo_u32 v1, v1, v3
266; GCN-NEXT:    s_setpc_b64 s[30:31]
267;
268; GFX10-LABEL: v_mul_v2i32:
269; GFX10:       ; %bb.0:
270; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
271; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
272; GFX10-NEXT:    v_mul_lo_u32 v0, v0, v2
273; GFX10-NEXT:    v_mul_lo_u32 v1, v1, v3
274; GFX10-NEXT:    s_setpc_b64 s[30:31]
275  %result = mul <2 x i32> %num, %den
276  ret <2 x i32> %result
277}
278
279define amdgpu_cs i33 @s_mul_i33(i33 inreg %num,  i33 inreg %den) {
280; GFX7-LABEL: s_mul_i33:
281; GFX7:       ; %bb.0:
282; GFX7-NEXT:    v_mov_b32_e32 v0, s2
283; GFX7-NEXT:    v_mul_hi_u32 v0, s0, v0
284; GFX7-NEXT:    s_mul_i32 s4, s0, s2
285; GFX7-NEXT:    s_mul_i32 s1, s1, s2
286; GFX7-NEXT:    s_mul_i32 s0, s0, s3
287; GFX7-NEXT:    s_add_i32 s1, s1, s0
288; GFX7-NEXT:    v_add_i32_e32 v0, vcc, s1, v0
289; GFX7-NEXT:    v_readfirstlane_b32 s1, v0
290; GFX7-NEXT:    s_mov_b32 s0, s4
291; GFX7-NEXT:    ; return to shader part epilog
292;
293; GFX8-LABEL: s_mul_i33:
294; GFX8:       ; %bb.0:
295; GFX8-NEXT:    v_mov_b32_e32 v0, s2
296; GFX8-NEXT:    v_mul_hi_u32 v0, s0, v0
297; GFX8-NEXT:    s_mul_i32 s4, s0, s2
298; GFX8-NEXT:    s_mul_i32 s1, s1, s2
299; GFX8-NEXT:    s_mul_i32 s0, s0, s3
300; GFX8-NEXT:    s_add_i32 s1, s1, s0
301; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s1, v0
302; GFX8-NEXT:    v_readfirstlane_b32 s1, v0
303; GFX8-NEXT:    s_mov_b32 s0, s4
304; GFX8-NEXT:    ; return to shader part epilog
305;
306; GFX9-LABEL: s_mul_i33:
307; GFX9:       ; %bb.0:
308; GFX9-NEXT:    s_mul_i32 s1, s1, s2
309; GFX9-NEXT:    s_mul_i32 s3, s0, s3
310; GFX9-NEXT:    s_mul_i32 s4, s0, s2
311; GFX9-NEXT:    s_mul_hi_u32 s0, s0, s2
312; GFX9-NEXT:    s_add_i32 s1, s1, s3
313; GFX9-NEXT:    s_add_i32 s1, s1, s0
314; GFX9-NEXT:    s_mov_b32 s0, s4
315; GFX9-NEXT:    ; return to shader part epilog
316;
317; GFX10-LABEL: s_mul_i33:
318; GFX10:       ; %bb.0:
319; GFX10-NEXT:    s_mul_i32 s1, s1, s2
320; GFX10-NEXT:    s_mul_i32 s3, s0, s3
321; GFX10-NEXT:    s_mul_hi_u32 s4, s0, s2
322; GFX10-NEXT:    s_add_i32 s1, s1, s3
323; GFX10-NEXT:    s_mul_i32 s0, s0, s2
324; GFX10-NEXT:    s_add_i32 s1, s1, s4
325; GFX10-NEXT:    ; return to shader part epilog
326  %result = mul i33 %num, %den
327  ret i33 %result
328}
329
330define amdgpu_ps i64 @s_mul_i64(i64 inreg %num, i64 inreg %den) {
331; GFX7-LABEL: s_mul_i64:
332; GFX7:       ; %bb.0:
333; GFX7-NEXT:    v_mov_b32_e32 v0, s2
334; GFX7-NEXT:    v_mul_hi_u32 v0, s0, v0
335; GFX7-NEXT:    s_mul_i32 s4, s0, s2
336; GFX7-NEXT:    s_mul_i32 s1, s1, s2
337; GFX7-NEXT:    s_mul_i32 s0, s0, s3
338; GFX7-NEXT:    s_add_i32 s1, s1, s0
339; GFX7-NEXT:    v_add_i32_e32 v0, vcc, s1, v0
340; GFX7-NEXT:    v_readfirstlane_b32 s1, v0
341; GFX7-NEXT:    s_mov_b32 s0, s4
342; GFX7-NEXT:    ; return to shader part epilog
343;
344; GFX8-LABEL: s_mul_i64:
345; GFX8:       ; %bb.0:
346; GFX8-NEXT:    v_mov_b32_e32 v0, s2
347; GFX8-NEXT:    v_mul_hi_u32 v0, s0, v0
348; GFX8-NEXT:    s_mul_i32 s4, s0, s2
349; GFX8-NEXT:    s_mul_i32 s1, s1, s2
350; GFX8-NEXT:    s_mul_i32 s0, s0, s3
351; GFX8-NEXT:    s_add_i32 s1, s1, s0
352; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s1, v0
353; GFX8-NEXT:    v_readfirstlane_b32 s1, v0
354; GFX8-NEXT:    s_mov_b32 s0, s4
355; GFX8-NEXT:    ; return to shader part epilog
356;
357; GFX9-LABEL: s_mul_i64:
358; GFX9:       ; %bb.0:
359; GFX9-NEXT:    s_mul_i32 s1, s1, s2
360; GFX9-NEXT:    s_mul_i32 s3, s0, s3
361; GFX9-NEXT:    s_mul_i32 s4, s0, s2
362; GFX9-NEXT:    s_mul_hi_u32 s0, s0, s2
363; GFX9-NEXT:    s_add_i32 s1, s1, s3
364; GFX9-NEXT:    s_add_i32 s1, s1, s0
365; GFX9-NEXT:    s_mov_b32 s0, s4
366; GFX9-NEXT:    ; return to shader part epilog
367;
368; GFX10-LABEL: s_mul_i64:
369; GFX10:       ; %bb.0:
370; GFX10-NEXT:    s_mul_i32 s1, s1, s2
371; GFX10-NEXT:    s_mul_i32 s3, s0, s3
372; GFX10-NEXT:    s_mul_hi_u32 s4, s0, s2
373; GFX10-NEXT:    s_add_i32 s1, s1, s3
374; GFX10-NEXT:    s_mul_i32 s0, s0, s2
375; GFX10-NEXT:    s_add_i32 s1, s1, s4
376; GFX10-NEXT:    ; return to shader part epilog
377  %result = mul i64 %num, %den
378  ret i64 %result
379}
380
381define i64 @v_mul_i64(i64 %num, i64 %den) {
382; GFX7-LABEL: v_mul_i64:
383; GFX7:       ; %bb.0:
384; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
385; GFX7-NEXT:    v_mul_lo_u32 v1, v1, v2
386; GFX7-NEXT:    v_mul_lo_u32 v4, v0, v3
387; GFX7-NEXT:    v_mul_lo_u32 v3, v0, v2
388; GFX7-NEXT:    v_mul_hi_u32 v0, v0, v2
389; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v1, v4
390; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v1, v0
391; GFX7-NEXT:    v_mov_b32_e32 v0, v3
392; GFX7-NEXT:    s_setpc_b64 s[30:31]
393;
394; GFX8-LABEL: v_mul_i64:
395; GFX8:       ; %bb.0:
396; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
397; GFX8-NEXT:    v_mul_lo_u32 v1, v1, v2
398; GFX8-NEXT:    v_mul_lo_u32 v4, v0, v3
399; GFX8-NEXT:    v_mul_lo_u32 v3, v0, v2
400; GFX8-NEXT:    v_mul_hi_u32 v0, v0, v2
401; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v4
402; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v0
403; GFX8-NEXT:    v_mov_b32_e32 v0, v3
404; GFX8-NEXT:    s_setpc_b64 s[30:31]
405;
406; GFX9-LABEL: v_mul_i64:
407; GFX9:       ; %bb.0:
408; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
409; GFX9-NEXT:    v_mul_lo_u32 v1, v1, v2
410; GFX9-NEXT:    v_mul_lo_u32 v3, v0, v3
411; GFX9-NEXT:    v_mul_hi_u32 v4, v0, v2
412; GFX9-NEXT:    v_mul_lo_u32 v0, v0, v2
413; GFX9-NEXT:    v_add3_u32 v1, v1, v3, v4
414; GFX9-NEXT:    s_setpc_b64 s[30:31]
415;
416; GFX10-LABEL: v_mul_i64:
417; GFX10:       ; %bb.0:
418; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
419; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
420; GFX10-NEXT:    v_mul_lo_u32 v1, v1, v2
421; GFX10-NEXT:    v_mul_lo_u32 v3, v0, v3
422; GFX10-NEXT:    v_mul_hi_u32 v4, v0, v2
423; GFX10-NEXT:    v_mul_lo_u32 v0, v0, v2
424; GFX10-NEXT:    v_add3_u32 v1, v1, v3, v4
425; GFX10-NEXT:    s_setpc_b64 s[30:31]
426  %result = mul i64 %num, %den
427  ret i64 %result
428}
429
430define amdgpu_ps <3 x i32> @s_mul_i96(i96 inreg %num, i96 inreg %den) {
431; GFX7-LABEL: s_mul_i96:
432; GFX7:       ; %bb.0:
433; GFX7-NEXT:    v_mov_b32_e32 v0, s3
434; GFX7-NEXT:    v_mul_hi_u32 v0, s0, v0
435; GFX7-NEXT:    s_mul_i32 s7, s1, s3
436; GFX7-NEXT:    s_mul_i32 s8, s0, s4
437; GFX7-NEXT:    v_mov_b32_e32 v2, s1
438; GFX7-NEXT:    s_add_u32 s7, s7, s8
439; GFX7-NEXT:    v_mul_hi_u32 v2, v2, s3
440; GFX7-NEXT:    v_mov_b32_e32 v3, s4
441; GFX7-NEXT:    v_add_i32_e32 v0, vcc, s7, v0
442; GFX7-NEXT:    s_mul_i32 s2, s2, s3
443; GFX7-NEXT:    s_mul_i32 s7, s1, s4
444; GFX7-NEXT:    v_mul_hi_u32 v3, s0, v3
445; GFX7-NEXT:    s_mul_i32 s6, s0, s3
446; GFX7-NEXT:    s_cselect_b32 s8, 1, 0
447; GFX7-NEXT:    s_mul_i32 s5, s0, s5
448; GFX7-NEXT:    s_add_i32 s0, s2, s7
449; GFX7-NEXT:    s_add_i32 s0, s0, s5
450; GFX7-NEXT:    s_and_b32 s8, s8, 1
451; GFX7-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
452; GFX7-NEXT:    v_add_i32_e32 v2, vcc, s0, v2
453; GFX7-NEXT:    v_add_i32_e32 v1, vcc, s8, v1
454; GFX7-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
455; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
456; GFX7-NEXT:    v_readfirstlane_b32 s1, v0
457; GFX7-NEXT:    v_readfirstlane_b32 s2, v1
458; GFX7-NEXT:    s_mov_b32 s0, s6
459; GFX7-NEXT:    ; return to shader part epilog
460;
461; GFX8-LABEL: s_mul_i96:
462; GFX8:       ; %bb.0:
463; GFX8-NEXT:    v_mov_b32_e32 v0, s3
464; GFX8-NEXT:    v_mul_hi_u32 v0, s0, v0
465; GFX8-NEXT:    s_mul_i32 s7, s1, s3
466; GFX8-NEXT:    s_mul_i32 s8, s0, s4
467; GFX8-NEXT:    v_mov_b32_e32 v2, s1
468; GFX8-NEXT:    s_add_u32 s7, s7, s8
469; GFX8-NEXT:    v_mul_hi_u32 v2, v2, s3
470; GFX8-NEXT:    v_mov_b32_e32 v3, s4
471; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s7, v0
472; GFX8-NEXT:    s_mul_i32 s2, s2, s3
473; GFX8-NEXT:    s_mul_i32 s7, s1, s4
474; GFX8-NEXT:    v_mul_hi_u32 v3, s0, v3
475; GFX8-NEXT:    s_mul_i32 s6, s0, s3
476; GFX8-NEXT:    s_cselect_b32 s8, 1, 0
477; GFX8-NEXT:    s_mul_i32 s5, s0, s5
478; GFX8-NEXT:    s_add_i32 s0, s2, s7
479; GFX8-NEXT:    s_add_i32 s0, s0, s5
480; GFX8-NEXT:    s_and_b32 s8, s8, 1
481; GFX8-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
482; GFX8-NEXT:    v_add_u32_e32 v2, vcc, s0, v2
483; GFX8-NEXT:    v_add_u32_e32 v1, vcc, s8, v1
484; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v3
485; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v2, v1
486; GFX8-NEXT:    v_readfirstlane_b32 s1, v0
487; GFX8-NEXT:    v_readfirstlane_b32 s2, v1
488; GFX8-NEXT:    s_mov_b32 s0, s6
489; GFX8-NEXT:    ; return to shader part epilog
490;
491; GFX9-LABEL: s_mul_i96:
492; GFX9:       ; %bb.0:
493; GFX9-NEXT:    s_mul_i32 s7, s1, s3
494; GFX9-NEXT:    s_mul_i32 s8, s0, s4
495; GFX9-NEXT:    s_add_u32 s7, s7, s8
496; GFX9-NEXT:    s_cselect_b32 s8, 1, 0
497; GFX9-NEXT:    s_mul_hi_u32 s9, s0, s3
498; GFX9-NEXT:    s_and_b32 s8, s8, 1
499; GFX9-NEXT:    s_add_u32 s7, s7, s9
500; GFX9-NEXT:    s_cselect_b32 s9, 1, 0
501; GFX9-NEXT:    s_and_b32 s9, s9, 1
502; GFX9-NEXT:    s_add_i32 s8, s8, s9
503; GFX9-NEXT:    s_mul_i32 s2, s2, s3
504; GFX9-NEXT:    s_mul_i32 s9, s1, s4
505; GFX9-NEXT:    s_mul_i32 s5, s0, s5
506; GFX9-NEXT:    s_add_i32 s2, s2, s9
507; GFX9-NEXT:    s_mul_hi_u32 s1, s1, s3
508; GFX9-NEXT:    s_add_i32 s2, s2, s5
509; GFX9-NEXT:    s_mul_i32 s6, s0, s3
510; GFX9-NEXT:    s_mul_hi_u32 s0, s0, s4
511; GFX9-NEXT:    s_add_i32 s1, s2, s1
512; GFX9-NEXT:    s_add_i32 s0, s1, s0
513; GFX9-NEXT:    s_add_i32 s2, s0, s8
514; GFX9-NEXT:    s_mov_b32 s0, s6
515; GFX9-NEXT:    s_mov_b32 s1, s7
516; GFX9-NEXT:    ; return to shader part epilog
517;
518; GFX10-LABEL: s_mul_i96:
519; GFX10:       ; %bb.0:
520; GFX10-NEXT:    s_mul_i32 s6, s1, s3
521; GFX10-NEXT:    s_mul_i32 s7, s0, s4
522; GFX10-NEXT:    s_mul_hi_u32 s8, s0, s3
523; GFX10-NEXT:    s_add_u32 s6, s6, s7
524; GFX10-NEXT:    s_cselect_b32 s7, 1, 0
525; GFX10-NEXT:    s_mul_i32 s2, s2, s3
526; GFX10-NEXT:    s_and_b32 s7, s7, 1
527; GFX10-NEXT:    s_mul_i32 s9, s1, s4
528; GFX10-NEXT:    s_add_u32 s6, s6, s8
529; GFX10-NEXT:    s_cselect_b32 s8, 1, 0
530; GFX10-NEXT:    s_mul_i32 s5, s0, s5
531; GFX10-NEXT:    s_add_i32 s2, s2, s9
532; GFX10-NEXT:    s_mul_hi_u32 s1, s1, s3
533; GFX10-NEXT:    s_add_i32 s2, s2, s5
534; GFX10-NEXT:    s_and_b32 s8, s8, 1
535; GFX10-NEXT:    s_mul_hi_u32 s4, s0, s4
536; GFX10-NEXT:    s_add_i32 s1, s2, s1
537; GFX10-NEXT:    s_add_i32 s7, s7, s8
538; GFX10-NEXT:    s_add_i32 s1, s1, s4
539; GFX10-NEXT:    s_mul_i32 s0, s0, s3
540; GFX10-NEXT:    s_add_i32 s2, s1, s7
541; GFX10-NEXT:    s_mov_b32 s1, s6
542; GFX10-NEXT:    ; return to shader part epilog
543  %result = mul i96 %num, %den
544  %cast = bitcast i96 %result to <3 x i32>
545  ret <3 x i32> %cast
546}
547
548define i96 @v_mul_i96(i96 %num, i96 %den) {
549; GFX7-LABEL: v_mul_i96:
550; GFX7:       ; %bb.0:
551; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
552; GFX7-NEXT:    v_mul_lo_u32 v7, v1, v3
553; GFX7-NEXT:    v_mul_lo_u32 v8, v0, v4
554; GFX7-NEXT:    v_mul_hi_u32 v9, v0, v3
555; GFX7-NEXT:    v_mul_lo_u32 v2, v2, v3
556; GFX7-NEXT:    v_mul_lo_u32 v5, v0, v5
557; GFX7-NEXT:    v_add_i32_e32 v7, vcc, v7, v8
558; GFX7-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
559; GFX7-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
560; GFX7-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
561; GFX7-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
562; GFX7-NEXT:    v_mul_lo_u32 v9, v1, v4
563; GFX7-NEXT:    v_mul_hi_u32 v1, v1, v3
564; GFX7-NEXT:    v_mul_lo_u32 v6, v0, v3
565; GFX7-NEXT:    v_mul_hi_u32 v0, v0, v4
566; GFX7-NEXT:    v_add_i32_e32 v2, vcc, v2, v9
567; GFX7-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
568; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
569; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
570; GFX7-NEXT:    v_add_i32_e32 v2, vcc, v0, v8
571; GFX7-NEXT:    v_mov_b32_e32 v0, v6
572; GFX7-NEXT:    v_mov_b32_e32 v1, v7
573; GFX7-NEXT:    s_setpc_b64 s[30:31]
574;
575; GFX8-LABEL: v_mul_i96:
576; GFX8:       ; %bb.0:
577; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
578; GFX8-NEXT:    v_mul_lo_u32 v7, v1, v3
579; GFX8-NEXT:    v_mul_lo_u32 v8, v0, v4
580; GFX8-NEXT:    v_mul_hi_u32 v9, v0, v3
581; GFX8-NEXT:    v_mul_lo_u32 v2, v2, v3
582; GFX8-NEXT:    v_mul_lo_u32 v5, v0, v5
583; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v7, v8
584; GFX8-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
585; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v7, v9
586; GFX8-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
587; GFX8-NEXT:    v_add_u32_e32 v8, vcc, v8, v9
588; GFX8-NEXT:    v_mul_lo_u32 v9, v1, v4
589; GFX8-NEXT:    v_mul_hi_u32 v1, v1, v3
590; GFX8-NEXT:    v_mul_lo_u32 v6, v0, v3
591; GFX8-NEXT:    v_mul_hi_u32 v0, v0, v4
592; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v9
593; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v5
594; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v2, v1
595; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v1, v0
596; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v0, v8
597; GFX8-NEXT:    v_mov_b32_e32 v0, v6
598; GFX8-NEXT:    v_mov_b32_e32 v1, v7
599; GFX8-NEXT:    s_setpc_b64 s[30:31]
600;
601; GFX9-LABEL: v_mul_i96:
602; GFX9:       ; %bb.0:
603; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
604; GFX9-NEXT:    v_mul_lo_u32 v7, v1, v3
605; GFX9-NEXT:    v_mul_lo_u32 v8, v0, v4
606; GFX9-NEXT:    v_mul_hi_u32 v9, v0, v3
607; GFX9-NEXT:    v_mul_lo_u32 v2, v2, v3
608; GFX9-NEXT:    v_mul_lo_u32 v10, v1, v4
609; GFX9-NEXT:    v_mul_lo_u32 v5, v0, v5
610; GFX9-NEXT:    v_mul_hi_u32 v1, v1, v3
611; GFX9-NEXT:    v_mul_lo_u32 v6, v0, v3
612; GFX9-NEXT:    v_add_co_u32_e32 v7, vcc, v7, v8
613; GFX9-NEXT:    v_mul_hi_u32 v0, v0, v4
614; GFX9-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
615; GFX9-NEXT:    v_add_co_u32_e32 v7, vcc, v7, v9
616; GFX9-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
617; GFX9-NEXT:    v_add_u32_e32 v2, v2, v10
618; GFX9-NEXT:    v_add_u32_e32 v3, v8, v9
619; GFX9-NEXT:    v_add3_u32 v1, v2, v5, v1
620; GFX9-NEXT:    v_add3_u32 v2, v1, v0, v3
621; GFX9-NEXT:    v_mov_b32_e32 v0, v6
622; GFX9-NEXT:    v_mov_b32_e32 v1, v7
623; GFX9-NEXT:    s_setpc_b64 s[30:31]
624;
625; GFX10-LABEL: v_mul_i96:
626; GFX10:       ; %bb.0:
627; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
628; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
629; GFX10-NEXT:    v_mul_lo_u32 v6, v1, v3
630; GFX10-NEXT:    v_mul_lo_u32 v7, v0, v4
631; GFX10-NEXT:    v_mul_hi_u32 v8, v0, v3
632; GFX10-NEXT:    v_mul_lo_u32 v2, v2, v3
633; GFX10-NEXT:    v_mul_lo_u32 v9, v1, v4
634; GFX10-NEXT:    v_mul_lo_u32 v5, v0, v5
635; GFX10-NEXT:    v_mul_hi_u32 v4, v0, v4
636; GFX10-NEXT:    v_mul_lo_u32 v0, v0, v3
637; GFX10-NEXT:    v_add_co_u32 v6, s4, v6, v7
638; GFX10-NEXT:    v_mul_hi_u32 v7, v1, v3
639; GFX10-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s4
640; GFX10-NEXT:    v_add_nc_u32_e32 v2, v2, v9
641; GFX10-NEXT:    v_add_co_u32 v1, s4, v6, v8
642; GFX10-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s4
643; GFX10-NEXT:    v_add3_u32 v2, v2, v5, v7
644; GFX10-NEXT:    v_add_nc_u32_e32 v3, v10, v6
645; GFX10-NEXT:    v_add3_u32 v2, v2, v4, v3
646; GFX10-NEXT:    s_setpc_b64 s[30:31]
647  %result = mul i96 %num, %den
648  ret i96 %result
649}
650
651define amdgpu_ps <4 x i32> @s_mul_i128(i128 inreg %num, i128 inreg %den) {
652; GFX7-LABEL: s_mul_i128:
653; GFX7:       ; %bb.0:
654; GFX7-NEXT:    v_mov_b32_e32 v0, s4
655; GFX7-NEXT:    v_mul_hi_u32 v0, s0, v0
656; GFX7-NEXT:    s_mul_i32 s9, s1, s4
657; GFX7-NEXT:    s_mul_i32 s10, s0, s5
658; GFX7-NEXT:    s_add_u32 s9, s9, s10
659; GFX7-NEXT:    s_cselect_b32 s10, 1, 0
660; GFX7-NEXT:    v_add_i32_e32 v0, vcc, s9, v0
661; GFX7-NEXT:    s_and_b32 s10, s10, 1
662; GFX7-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
663; GFX7-NEXT:    v_add_i32_e32 v1, vcc, s10, v1
664; GFX7-NEXT:    s_mul_i32 s9, s2, s4
665; GFX7-NEXT:    s_mul_i32 s10, s1, s5
666; GFX7-NEXT:    v_mov_b32_e32 v2, s1
667; GFX7-NEXT:    s_add_u32 s9, s9, s10
668; GFX7-NEXT:    v_mul_hi_u32 v2, v2, s4
669; GFX7-NEXT:    s_cselect_b32 s10, 1, 0
670; GFX7-NEXT:    s_mul_i32 s11, s0, s6
671; GFX7-NEXT:    s_and_b32 s10, s10, 1
672; GFX7-NEXT:    v_mov_b32_e32 v3, s5
673; GFX7-NEXT:    s_add_u32 s9, s9, s11
674; GFX7-NEXT:    v_mul_hi_u32 v4, s0, v3
675; GFX7-NEXT:    s_cselect_b32 s11, 1, 0
676; GFX7-NEXT:    s_and_b32 s11, s11, 1
677; GFX7-NEXT:    v_add_i32_e32 v2, vcc, s9, v2
678; GFX7-NEXT:    s_add_i32 s10, s10, s11
679; GFX7-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
680; GFX7-NEXT:    v_add_i32_e32 v5, vcc, s10, v5
681; GFX7-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
682; GFX7-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
683; GFX7-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
684; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
685; GFX7-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
686; GFX7-NEXT:    v_add_i32_e32 v2, vcc, v4, v2
687; GFX7-NEXT:    v_mov_b32_e32 v4, s2
688; GFX7-NEXT:    s_mul_i32 s3, s3, s4
689; GFX7-NEXT:    s_mul_i32 s5, s2, s5
690; GFX7-NEXT:    v_mul_hi_u32 v4, v4, s4
691; GFX7-NEXT:    v_mov_b32_e32 v5, s6
692; GFX7-NEXT:    s_mul_i32 s8, s0, s4
693; GFX7-NEXT:    s_mul_i32 s9, s1, s6
694; GFX7-NEXT:    s_mul_i32 s7, s0, s7
695; GFX7-NEXT:    v_mul_hi_u32 v3, s1, v3
696; GFX7-NEXT:    v_mul_hi_u32 v5, s0, v5
697; GFX7-NEXT:    s_add_i32 s0, s3, s5
698; GFX7-NEXT:    s_add_i32 s0, s0, s9
699; GFX7-NEXT:    s_add_i32 s0, s0, s7
700; GFX7-NEXT:    v_add_i32_e32 v4, vcc, s0, v4
701; GFX7-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
702; GFX7-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
703; GFX7-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
704; GFX7-NEXT:    v_readfirstlane_b32 s1, v0
705; GFX7-NEXT:    v_readfirstlane_b32 s2, v1
706; GFX7-NEXT:    v_readfirstlane_b32 s3, v2
707; GFX7-NEXT:    s_mov_b32 s0, s8
708; GFX7-NEXT:    ; return to shader part epilog
709;
710; GFX8-LABEL: s_mul_i128:
711; GFX8:       ; %bb.0:
712; GFX8-NEXT:    v_mov_b32_e32 v0, s4
713; GFX8-NEXT:    v_mul_hi_u32 v0, s0, v0
714; GFX8-NEXT:    s_mul_i32 s9, s1, s4
715; GFX8-NEXT:    s_mul_i32 s10, s0, s5
716; GFX8-NEXT:    s_add_u32 s9, s9, s10
717; GFX8-NEXT:    s_cselect_b32 s10, 1, 0
718; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s9, v0
719; GFX8-NEXT:    s_and_b32 s10, s10, 1
720; GFX8-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
721; GFX8-NEXT:    v_add_u32_e32 v1, vcc, s10, v1
722; GFX8-NEXT:    s_mul_i32 s9, s2, s4
723; GFX8-NEXT:    s_mul_i32 s10, s1, s5
724; GFX8-NEXT:    v_mov_b32_e32 v2, s1
725; GFX8-NEXT:    s_add_u32 s9, s9, s10
726; GFX8-NEXT:    v_mul_hi_u32 v2, v2, s4
727; GFX8-NEXT:    s_cselect_b32 s10, 1, 0
728; GFX8-NEXT:    s_mul_i32 s11, s0, s6
729; GFX8-NEXT:    s_and_b32 s10, s10, 1
730; GFX8-NEXT:    v_mov_b32_e32 v3, s5
731; GFX8-NEXT:    s_add_u32 s9, s9, s11
732; GFX8-NEXT:    v_mul_hi_u32 v4, s0, v3
733; GFX8-NEXT:    s_cselect_b32 s11, 1, 0
734; GFX8-NEXT:    s_and_b32 s11, s11, 1
735; GFX8-NEXT:    v_add_u32_e32 v2, vcc, s9, v2
736; GFX8-NEXT:    s_add_i32 s10, s10, s11
737; GFX8-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
738; GFX8-NEXT:    v_add_u32_e32 v5, vcc, s10, v5
739; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v4
740; GFX8-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
741; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v5, v4
742; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v2, v1
743; GFX8-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
744; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v4, v2
745; GFX8-NEXT:    v_mov_b32_e32 v4, s2
746; GFX8-NEXT:    s_mul_i32 s3, s3, s4
747; GFX8-NEXT:    s_mul_i32 s5, s2, s5
748; GFX8-NEXT:    v_mul_hi_u32 v4, v4, s4
749; GFX8-NEXT:    v_mov_b32_e32 v5, s6
750; GFX8-NEXT:    s_mul_i32 s8, s0, s4
751; GFX8-NEXT:    s_mul_i32 s9, s1, s6
752; GFX8-NEXT:    s_mul_i32 s7, s0, s7
753; GFX8-NEXT:    v_mul_hi_u32 v3, s1, v3
754; GFX8-NEXT:    v_mul_hi_u32 v5, s0, v5
755; GFX8-NEXT:    s_add_i32 s0, s3, s5
756; GFX8-NEXT:    s_add_i32 s0, s0, s9
757; GFX8-NEXT:    s_add_i32 s0, s0, s7
758; GFX8-NEXT:    v_add_u32_e32 v4, vcc, s0, v4
759; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v4, v3
760; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v3, v5
761; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v3, v2
762; GFX8-NEXT:    v_readfirstlane_b32 s1, v0
763; GFX8-NEXT:    v_readfirstlane_b32 s2, v1
764; GFX8-NEXT:    v_readfirstlane_b32 s3, v2
765; GFX8-NEXT:    s_mov_b32 s0, s8
766; GFX8-NEXT:    ; return to shader part epilog
767;
768; GFX9-LABEL: s_mul_i128:
769; GFX9:       ; %bb.0:
770; GFX9-NEXT:    s_mul_i32 s9, s1, s4
771; GFX9-NEXT:    s_mul_i32 s10, s0, s5
772; GFX9-NEXT:    s_add_u32 s9, s9, s10
773; GFX9-NEXT:    s_cselect_b32 s10, 1, 0
774; GFX9-NEXT:    s_mul_hi_u32 s11, s0, s4
775; GFX9-NEXT:    s_and_b32 s10, s10, 1
776; GFX9-NEXT:    s_add_u32 s9, s9, s11
777; GFX9-NEXT:    s_cselect_b32 s11, 1, 0
778; GFX9-NEXT:    s_and_b32 s11, s11, 1
779; GFX9-NEXT:    s_add_i32 s10, s10, s11
780; GFX9-NEXT:    s_mul_i32 s11, s2, s4
781; GFX9-NEXT:    s_mul_i32 s12, s1, s5
782; GFX9-NEXT:    s_add_u32 s11, s11, s12
783; GFX9-NEXT:    s_cselect_b32 s12, 1, 0
784; GFX9-NEXT:    s_mul_i32 s13, s0, s6
785; GFX9-NEXT:    s_and_b32 s12, s12, 1
786; GFX9-NEXT:    s_add_u32 s11, s11, s13
787; GFX9-NEXT:    s_cselect_b32 s13, 1, 0
788; GFX9-NEXT:    s_and_b32 s13, s13, 1
789; GFX9-NEXT:    s_mul_hi_u32 s14, s1, s4
790; GFX9-NEXT:    s_add_i32 s12, s12, s13
791; GFX9-NEXT:    s_add_u32 s11, s11, s14
792; GFX9-NEXT:    s_cselect_b32 s13, 1, 0
793; GFX9-NEXT:    s_and_b32 s13, s13, 1
794; GFX9-NEXT:    s_mul_hi_u32 s15, s0, s5
795; GFX9-NEXT:    s_add_i32 s12, s12, s13
796; GFX9-NEXT:    s_add_u32 s11, s11, s15
797; GFX9-NEXT:    s_cselect_b32 s13, 1, 0
798; GFX9-NEXT:    s_and_b32 s13, s13, 1
799; GFX9-NEXT:    s_add_i32 s12, s12, s13
800; GFX9-NEXT:    s_add_u32 s10, s11, s10
801; GFX9-NEXT:    s_cselect_b32 s11, 1, 0
802; GFX9-NEXT:    s_and_b32 s11, s11, 1
803; GFX9-NEXT:    s_add_i32 s12, s12, s11
804; GFX9-NEXT:    s_mul_i32 s3, s3, s4
805; GFX9-NEXT:    s_mul_i32 s11, s2, s5
806; GFX9-NEXT:    s_mul_i32 s13, s1, s6
807; GFX9-NEXT:    s_add_i32 s3, s3, s11
808; GFX9-NEXT:    s_mul_i32 s7, s0, s7
809; GFX9-NEXT:    s_add_i32 s3, s3, s13
810; GFX9-NEXT:    s_mul_hi_u32 s2, s2, s4
811; GFX9-NEXT:    s_add_i32 s3, s3, s7
812; GFX9-NEXT:    s_mul_hi_u32 s1, s1, s5
813; GFX9-NEXT:    s_add_i32 s2, s3, s2
814; GFX9-NEXT:    s_mul_i32 s8, s0, s4
815; GFX9-NEXT:    s_mul_hi_u32 s0, s0, s6
816; GFX9-NEXT:    s_add_i32 s1, s2, s1
817; GFX9-NEXT:    s_add_i32 s0, s1, s0
818; GFX9-NEXT:    s_add_i32 s3, s0, s12
819; GFX9-NEXT:    s_mov_b32 s0, s8
820; GFX9-NEXT:    s_mov_b32 s1, s9
821; GFX9-NEXT:    s_mov_b32 s2, s10
822; GFX9-NEXT:    ; return to shader part epilog
823;
824; GFX10-LABEL: s_mul_i128:
825; GFX10:       ; %bb.0:
826; GFX10-NEXT:    s_mul_i32 s8, s1, s4
827; GFX10-NEXT:    s_mul_i32 s9, s0, s5
828; GFX10-NEXT:    s_mul_hi_u32 s10, s0, s4
829; GFX10-NEXT:    s_add_u32 s8, s8, s9
830; GFX10-NEXT:    s_cselect_b32 s9, 1, 0
831; GFX10-NEXT:    s_mul_i32 s11, s1, s5
832; GFX10-NEXT:    s_and_b32 s9, s9, 1
833; GFX10-NEXT:    s_add_u32 s8, s8, s10
834; GFX10-NEXT:    s_cselect_b32 s10, 1, 0
835; GFX10-NEXT:    s_mul_i32 s12, s0, s6
836; GFX10-NEXT:    s_and_b32 s10, s10, 1
837; GFX10-NEXT:    s_mul_hi_u32 s13, s1, s4
838; GFX10-NEXT:    s_add_i32 s9, s9, s10
839; GFX10-NEXT:    s_mul_i32 s10, s2, s4
840; GFX10-NEXT:    s_mul_i32 s3, s3, s4
841; GFX10-NEXT:    s_add_u32 s10, s10, s11
842; GFX10-NEXT:    s_cselect_b32 s11, 1, 0
843; GFX10-NEXT:    s_mul_i32 s7, s0, s7
844; GFX10-NEXT:    s_and_b32 s11, s11, 1
845; GFX10-NEXT:    s_add_u32 s10, s10, s12
846; GFX10-NEXT:    s_cselect_b32 s12, 1, 0
847; GFX10-NEXT:    s_and_b32 s12, s12, 1
848; GFX10-NEXT:    s_add_i32 s11, s11, s12
849; GFX10-NEXT:    s_add_u32 s10, s10, s13
850; GFX10-NEXT:    s_cselect_b32 s12, 1, 0
851; GFX10-NEXT:    s_mul_hi_u32 s13, s0, s5
852; GFX10-NEXT:    s_and_b32 s12, s12, 1
853; GFX10-NEXT:    s_add_i32 s11, s11, s12
854; GFX10-NEXT:    s_add_u32 s10, s10, s13
855; GFX10-NEXT:    s_cselect_b32 s12, 1, 0
856; GFX10-NEXT:    s_mul_i32 s13, s1, s6
857; GFX10-NEXT:    s_and_b32 s12, s12, 1
858; GFX10-NEXT:    s_mul_hi_u32 s1, s1, s5
859; GFX10-NEXT:    s_add_i32 s11, s11, s12
860; GFX10-NEXT:    s_mul_i32 s12, s2, s5
861; GFX10-NEXT:    s_add_u32 s9, s10, s9
862; GFX10-NEXT:    s_cselect_b32 s10, 1, 0
863; GFX10-NEXT:    s_add_i32 s3, s3, s12
864; GFX10-NEXT:    s_mul_hi_u32 s2, s2, s4
865; GFX10-NEXT:    s_add_i32 s3, s3, s13
866; GFX10-NEXT:    s_and_b32 s10, s10, 1
867; GFX10-NEXT:    s_add_i32 s3, s3, s7
868; GFX10-NEXT:    s_add_i32 s11, s11, s10
869; GFX10-NEXT:    s_add_i32 s2, s3, s2
870; GFX10-NEXT:    s_mul_hi_u32 s3, s0, s6
871; GFX10-NEXT:    s_add_i32 s1, s2, s1
872; GFX10-NEXT:    s_mul_i32 s0, s0, s4
873; GFX10-NEXT:    s_add_i32 s1, s1, s3
874; GFX10-NEXT:    s_mov_b32 s2, s9
875; GFX10-NEXT:    s_add_i32 s3, s1, s11
876; GFX10-NEXT:    s_mov_b32 s1, s8
877; GFX10-NEXT:    ; return to shader part epilog
878  %result = mul i128 %num, %den
879  %cast = bitcast i128 %result to <4 x i32>
880  ret <4 x i32> %cast
881}
882
883define i128 @v_mul_i128(i128 %num, i128 %den) {
884; GFX7-LABEL: v_mul_i128:
885; GFX7:       ; %bb.0:
886; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
887; GFX7-NEXT:    v_mul_lo_u32 v9, v1, v4
888; GFX7-NEXT:    v_mul_lo_u32 v10, v0, v5
889; GFX7-NEXT:    v_mul_hi_u32 v11, v0, v4
890; GFX7-NEXT:    v_mul_lo_u32 v12, v1, v5
891; GFX7-NEXT:    v_mul_lo_u32 v13, v0, v6
892; GFX7-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
893; GFX7-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
894; GFX7-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
895; GFX7-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
896; GFX7-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
897; GFX7-NEXT:    v_mul_lo_u32 v11, v2, v4
898; GFX7-NEXT:    v_mul_hi_u32 v14, v1, v4
899; GFX7-NEXT:    v_mul_hi_u32 v15, v0, v5
900; GFX7-NEXT:    v_mul_lo_u32 v3, v3, v4
901; GFX7-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
902; GFX7-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
903; GFX7-NEXT:    v_add_i32_e32 v11, vcc, v11, v13
904; GFX7-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
905; GFX7-NEXT:    v_add_i32_e32 v12, vcc, v12, v13
906; GFX7-NEXT:    v_add_i32_e32 v11, vcc, v11, v14
907; GFX7-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
908; GFX7-NEXT:    v_add_i32_e32 v12, vcc, v12, v13
909; GFX7-NEXT:    v_add_i32_e32 v11, vcc, v11, v15
910; GFX7-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
911; GFX7-NEXT:    v_add_i32_e32 v12, vcc, v12, v13
912; GFX7-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
913; GFX7-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
914; GFX7-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
915; GFX7-NEXT:    v_mul_lo_u32 v12, v2, v5
916; GFX7-NEXT:    v_mul_lo_u32 v13, v1, v6
917; GFX7-NEXT:    v_mul_lo_u32 v7, v0, v7
918; GFX7-NEXT:    v_mul_hi_u32 v2, v2, v4
919; GFX7-NEXT:    v_mul_hi_u32 v1, v1, v5
920; GFX7-NEXT:    v_add_i32_e32 v3, vcc, v3, v12
921; GFX7-NEXT:    v_mul_lo_u32 v8, v0, v4
922; GFX7-NEXT:    v_mul_hi_u32 v0, v0, v6
923; GFX7-NEXT:    v_add_i32_e32 v3, vcc, v3, v13
924; GFX7-NEXT:    v_add_i32_e32 v3, vcc, v3, v7
925; GFX7-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
926; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
927; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
928; GFX7-NEXT:    v_add_i32_e32 v3, vcc, v0, v11
929; GFX7-NEXT:    v_mov_b32_e32 v0, v8
930; GFX7-NEXT:    v_mov_b32_e32 v1, v9
931; GFX7-NEXT:    v_mov_b32_e32 v2, v10
932; GFX7-NEXT:    s_setpc_b64 s[30:31]
933;
934; GFX8-LABEL: v_mul_i128:
935; GFX8:       ; %bb.0:
936; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
937; GFX8-NEXT:    v_mul_lo_u32 v9, v1, v4
938; GFX8-NEXT:    v_mul_lo_u32 v10, v0, v5
939; GFX8-NEXT:    v_mul_hi_u32 v11, v0, v4
940; GFX8-NEXT:    v_mul_lo_u32 v12, v1, v5
941; GFX8-NEXT:    v_mul_lo_u32 v13, v0, v6
942; GFX8-NEXT:    v_add_u32_e32 v9, vcc, v9, v10
943; GFX8-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
944; GFX8-NEXT:    v_add_u32_e32 v9, vcc, v9, v11
945; GFX8-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
946; GFX8-NEXT:    v_add_u32_e32 v10, vcc, v10, v11
947; GFX8-NEXT:    v_mul_lo_u32 v11, v2, v4
948; GFX8-NEXT:    v_mul_hi_u32 v14, v1, v4
949; GFX8-NEXT:    v_mul_hi_u32 v15, v0, v5
950; GFX8-NEXT:    v_mul_lo_u32 v3, v3, v4
951; GFX8-NEXT:    v_add_u32_e32 v11, vcc, v11, v12
952; GFX8-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
953; GFX8-NEXT:    v_add_u32_e32 v11, vcc, v11, v13
954; GFX8-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
955; GFX8-NEXT:    v_add_u32_e32 v12, vcc, v12, v13
956; GFX8-NEXT:    v_add_u32_e32 v11, vcc, v11, v14
957; GFX8-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
958; GFX8-NEXT:    v_add_u32_e32 v12, vcc, v12, v13
959; GFX8-NEXT:    v_add_u32_e32 v11, vcc, v11, v15
960; GFX8-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
961; GFX8-NEXT:    v_add_u32_e32 v12, vcc, v12, v13
962; GFX8-NEXT:    v_add_u32_e32 v10, vcc, v11, v10
963; GFX8-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
964; GFX8-NEXT:    v_add_u32_e32 v11, vcc, v12, v11
965; GFX8-NEXT:    v_mul_lo_u32 v12, v2, v5
966; GFX8-NEXT:    v_mul_lo_u32 v13, v1, v6
967; GFX8-NEXT:    v_mul_lo_u32 v7, v0, v7
968; GFX8-NEXT:    v_mul_hi_u32 v2, v2, v4
969; GFX8-NEXT:    v_mul_hi_u32 v1, v1, v5
970; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v3, v12
971; GFX8-NEXT:    v_mul_lo_u32 v8, v0, v4
972; GFX8-NEXT:    v_mul_hi_u32 v0, v0, v6
973; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v3, v13
974; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v3, v7
975; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v3, v2
976; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v2, v1
977; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v1, v0
978; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v0, v11
979; GFX8-NEXT:    v_mov_b32_e32 v0, v8
980; GFX8-NEXT:    v_mov_b32_e32 v1, v9
981; GFX8-NEXT:    v_mov_b32_e32 v2, v10
982; GFX8-NEXT:    s_setpc_b64 s[30:31]
983;
984; GFX9-LABEL: v_mul_i128:
985; GFX9:       ; %bb.0:
986; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
987; GFX9-NEXT:    v_mul_lo_u32 v9, v1, v4
988; GFX9-NEXT:    v_mul_lo_u32 v10, v0, v5
989; GFX9-NEXT:    v_mul_hi_u32 v11, v0, v4
990; GFX9-NEXT:    v_mul_lo_u32 v12, v1, v5
991; GFX9-NEXT:    v_mul_lo_u32 v13, v0, v6
992; GFX9-NEXT:    v_add_co_u32_e32 v9, vcc, v9, v10
993; GFX9-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
994; GFX9-NEXT:    v_add_co_u32_e32 v9, vcc, v9, v11
995; GFX9-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
996; GFX9-NEXT:    v_add_u32_e32 v10, v10, v11
997; GFX9-NEXT:    v_mul_lo_u32 v11, v2, v4
998; GFX9-NEXT:    v_mul_hi_u32 v14, v1, v4
999; GFX9-NEXT:    v_mul_hi_u32 v15, v0, v5
1000; GFX9-NEXT:    v_mul_lo_u32 v3, v3, v4
1001; GFX9-NEXT:    v_add_co_u32_e32 v11, vcc, v11, v12
1002; GFX9-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
1003; GFX9-NEXT:    v_add_co_u32_e32 v11, vcc, v11, v13
1004; GFX9-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
1005; GFX9-NEXT:    v_add_co_u32_e32 v11, vcc, v11, v14
1006; GFX9-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
1007; GFX9-NEXT:    v_add_co_u32_e32 v11, vcc, v11, v15
1008; GFX9-NEXT:    v_add3_u32 v12, v12, v13, v14
1009; GFX9-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
1010; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, v11, v10
1011; GFX9-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
1012; GFX9-NEXT:    v_add3_u32 v11, v12, v13, v11
1013; GFX9-NEXT:    v_mul_lo_u32 v12, v2, v5
1014; GFX9-NEXT:    v_mul_lo_u32 v13, v1, v6
1015; GFX9-NEXT:    v_mul_lo_u32 v7, v0, v7
1016; GFX9-NEXT:    v_mul_hi_u32 v2, v2, v4
1017; GFX9-NEXT:    v_mul_hi_u32 v1, v1, v5
1018; GFX9-NEXT:    v_mul_lo_u32 v8, v0, v4
1019; GFX9-NEXT:    v_mul_hi_u32 v0, v0, v6
1020; GFX9-NEXT:    v_add_u32_e32 v3, v3, v12
1021; GFX9-NEXT:    v_add3_u32 v3, v3, v13, v7
1022; GFX9-NEXT:    v_add3_u32 v1, v3, v2, v1
1023; GFX9-NEXT:    v_add3_u32 v3, v1, v0, v11
1024; GFX9-NEXT:    v_mov_b32_e32 v0, v8
1025; GFX9-NEXT:    v_mov_b32_e32 v1, v9
1026; GFX9-NEXT:    v_mov_b32_e32 v2, v10
1027; GFX9-NEXT:    s_setpc_b64 s[30:31]
1028;
1029; GFX10-LABEL: v_mul_i128:
1030; GFX10:       ; %bb.0:
1031; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1032; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
1033; GFX10-NEXT:    v_mul_lo_u32 v8, v2, v4
1034; GFX10-NEXT:    v_mul_lo_u32 v9, v1, v5
1035; GFX10-NEXT:    v_mul_lo_u32 v10, v1, v4
1036; GFX10-NEXT:    v_mul_lo_u32 v11, v0, v5
1037; GFX10-NEXT:    v_mul_hi_u32 v12, v0, v4
1038; GFX10-NEXT:    v_mul_lo_u32 v13, v0, v6
1039; GFX10-NEXT:    v_mul_hi_u32 v15, v0, v5
1040; GFX10-NEXT:    v_mul_lo_u32 v3, v3, v4
1041; GFX10-NEXT:    v_mul_lo_u32 v7, v0, v7
1042; GFX10-NEXT:    v_add_co_u32 v8, s4, v8, v9
1043; GFX10-NEXT:    v_add_co_u32 v9, s5, v10, v11
1044; GFX10-NEXT:    v_mul_hi_u32 v11, v1, v4
1045; GFX10-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s5
1046; GFX10-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s4
1047; GFX10-NEXT:    v_add_co_u32 v13, s4, v8, v13
1048; GFX10-NEXT:    v_add_co_u32 v8, s5, v9, v12
1049; GFX10-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s5
1050; GFX10-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s4
1051; GFX10-NEXT:    v_add_co_u32 v11, s4, v13, v11
1052; GFX10-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s4
1053; GFX10-NEXT:    v_add_nc_u32_e32 v9, v10, v9
1054; GFX10-NEXT:    v_mul_lo_u32 v10, v2, v5
1055; GFX10-NEXT:    v_add_co_u32 v11, s4, v11, v15
1056; GFX10-NEXT:    v_add3_u32 v12, v14, v12, v13
1057; GFX10-NEXT:    v_mul_lo_u32 v13, v1, v6
1058; GFX10-NEXT:    v_mul_hi_u32 v15, v2, v4
1059; GFX10-NEXT:    v_mul_hi_u32 v1, v1, v5
1060; GFX10-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s4
1061; GFX10-NEXT:    v_add_nc_u32_e32 v3, v3, v10
1062; GFX10-NEXT:    v_add_co_u32 v2, s4, v11, v9
1063; GFX10-NEXT:    v_cndmask_b32_e64 v5, 0, 1, s4
1064; GFX10-NEXT:    v_mul_hi_u32 v6, v0, v6
1065; GFX10-NEXT:    v_add3_u32 v3, v3, v13, v7
1066; GFX10-NEXT:    v_mul_lo_u32 v0, v0, v4
1067; GFX10-NEXT:    v_add3_u32 v4, v12, v14, v5
1068; GFX10-NEXT:    v_add3_u32 v1, v3, v15, v1
1069; GFX10-NEXT:    v_add3_u32 v3, v1, v6, v4
1070; GFX10-NEXT:    v_mov_b32_e32 v1, v8
1071; GFX10-NEXT:    s_setpc_b64 s[30:31]
1072  %result = mul i128 %num, %den
1073  ret i128 %result
1074}
1075
1076define amdgpu_ps <8 x i32> @s_mul_i256(i256 inreg %num, i256 inreg %den) {
1077; GFX7-LABEL: s_mul_i256:
1078; GFX7:       ; %bb.0:
1079; GFX7-NEXT:    s_mov_b32 s16, s0
1080; GFX7-NEXT:    v_mov_b32_e32 v0, s8
1081; GFX7-NEXT:    v_mul_hi_u32 v0, s16, v0
1082; GFX7-NEXT:    s_mul_i32 s17, s1, s8
1083; GFX7-NEXT:    s_mul_i32 s18, s16, s9
1084; GFX7-NEXT:    s_add_u32 s17, s17, s18
1085; GFX7-NEXT:    s_cselect_b32 s18, 1, 0
1086; GFX7-NEXT:    v_add_i32_e32 v0, vcc, s17, v0
1087; GFX7-NEXT:    s_and_b32 s18, s18, 1
1088; GFX7-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
1089; GFX7-NEXT:    v_add_i32_e32 v1, vcc, s18, v1
1090; GFX7-NEXT:    s_mul_i32 s17, s2, s8
1091; GFX7-NEXT:    s_mul_i32 s18, s1, s9
1092; GFX7-NEXT:    v_mov_b32_e32 v2, s1
1093; GFX7-NEXT:    s_add_u32 s17, s17, s18
1094; GFX7-NEXT:    v_mul_hi_u32 v2, v2, s8
1095; GFX7-NEXT:    s_cselect_b32 s18, 1, 0
1096; GFX7-NEXT:    s_mul_i32 s19, s16, s10
1097; GFX7-NEXT:    s_and_b32 s18, s18, 1
1098; GFX7-NEXT:    v_mov_b32_e32 v3, s9
1099; GFX7-NEXT:    s_add_u32 s17, s17, s19
1100; GFX7-NEXT:    v_mul_hi_u32 v4, s16, v3
1101; GFX7-NEXT:    s_cselect_b32 s19, 1, 0
1102; GFX7-NEXT:    s_and_b32 s19, s19, 1
1103; GFX7-NEXT:    v_add_i32_e32 v2, vcc, s17, v2
1104; GFX7-NEXT:    s_add_i32 s18, s18, s19
1105; GFX7-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
1106; GFX7-NEXT:    v_add_i32_e32 v5, vcc, s18, v5
1107; GFX7-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
1108; GFX7-NEXT:    s_mul_i32 s17, s3, s8
1109; GFX7-NEXT:    s_mul_i32 s18, s2, s9
1110; GFX7-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
1111; GFX7-NEXT:    s_add_u32 s17, s17, s18
1112; GFX7-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
1113; GFX7-NEXT:    s_cselect_b32 s18, 1, 0
1114; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
1115; GFX7-NEXT:    s_mul_i32 s19, s1, s10
1116; GFX7-NEXT:    s_and_b32 s18, s18, 1
1117; GFX7-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
1118; GFX7-NEXT:    s_add_u32 s17, s17, s19
1119; GFX7-NEXT:    v_add_i32_e32 v2, vcc, v4, v2
1120; GFX7-NEXT:    v_mov_b32_e32 v4, s2
1121; GFX7-NEXT:    s_cselect_b32 s19, 1, 0
1122; GFX7-NEXT:    v_mul_hi_u32 v5, v4, s8
1123; GFX7-NEXT:    s_and_b32 s19, s19, 1
1124; GFX7-NEXT:    s_mul_i32 s20, s16, s11
1125; GFX7-NEXT:    s_add_i32 s18, s18, s19
1126; GFX7-NEXT:    s_add_u32 s17, s17, s20
1127; GFX7-NEXT:    v_mul_hi_u32 v3, s1, v3
1128; GFX7-NEXT:    s_cselect_b32 s19, 1, 0
1129; GFX7-NEXT:    s_and_b32 s19, s19, 1
1130; GFX7-NEXT:    v_add_i32_e32 v5, vcc, s17, v5
1131; GFX7-NEXT:    v_mov_b32_e32 v6, s10
1132; GFX7-NEXT:    s_add_i32 s18, s18, s19
1133; GFX7-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
1134; GFX7-NEXT:    v_mul_hi_u32 v7, s16, v6
1135; GFX7-NEXT:    v_add_i32_e32 v8, vcc, s18, v8
1136; GFX7-NEXT:    s_mul_i32 s17, s4, s8
1137; GFX7-NEXT:    s_mul_i32 s18, s3, s9
1138; GFX7-NEXT:    v_add_i32_e32 v3, vcc, v5, v3
1139; GFX7-NEXT:    s_add_u32 s17, s17, s18
1140; GFX7-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
1141; GFX7-NEXT:    s_cselect_b32 s18, 1, 0
1142; GFX7-NEXT:    v_add_i32_e32 v5, vcc, v8, v5
1143; GFX7-NEXT:    s_mul_i32 s19, s2, s10
1144; GFX7-NEXT:    s_and_b32 s18, s18, 1
1145; GFX7-NEXT:    v_add_i32_e32 v3, vcc, v3, v7
1146; GFX7-NEXT:    s_add_u32 s17, s17, s19
1147; GFX7-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
1148; GFX7-NEXT:    s_cselect_b32 s19, 1, 0
1149; GFX7-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
1150; GFX7-NEXT:    s_and_b32 s19, s19, 1
1151; GFX7-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
1152; GFX7-NEXT:    s_mul_i32 s20, s1, s11
1153; GFX7-NEXT:    s_add_i32 s18, s18, s19
1154; GFX7-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
1155; GFX7-NEXT:    s_add_u32 s17, s17, s20
1156; GFX7-NEXT:    v_add_i32_e32 v3, vcc, v5, v3
1157; GFX7-NEXT:    v_mov_b32_e32 v5, s3
1158; GFX7-NEXT:    s_cselect_b32 s19, 1, 0
1159; GFX7-NEXT:    v_mul_hi_u32 v7, v5, s8
1160; GFX7-NEXT:    s_and_b32 s19, s19, 1
1161; GFX7-NEXT:    s_mul_i32 s21, s16, s12
1162; GFX7-NEXT:    s_add_i32 s18, s18, s19
1163; GFX7-NEXT:    s_add_u32 s17, s17, s21
1164; GFX7-NEXT:    s_cselect_b32 s19, 1, 0
1165; GFX7-NEXT:    v_mul_hi_u32 v4, v4, s9
1166; GFX7-NEXT:    s_and_b32 s19, s19, 1
1167; GFX7-NEXT:    v_add_i32_e32 v7, vcc, s17, v7
1168; GFX7-NEXT:    s_add_i32 s18, s18, s19
1169; GFX7-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
1170; GFX7-NEXT:    v_add_i32_e32 v11, vcc, s18, v11
1171; GFX7-NEXT:    s_mul_i32 s17, s5, s8
1172; GFX7-NEXT:    s_mul_i32 s18, s4, s9
1173; GFX7-NEXT:    v_mul_hi_u32 v8, s1, v6
1174; GFX7-NEXT:    s_add_u32 s17, s17, s18
1175; GFX7-NEXT:    v_add_i32_e32 v4, vcc, v7, v4
1176; GFX7-NEXT:    s_cselect_b32 s18, 1, 0
1177; GFX7-NEXT:    v_mov_b32_e32 v9, s11
1178; GFX7-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
1179; GFX7-NEXT:    s_mul_i32 s19, s3, s10
1180; GFX7-NEXT:    s_and_b32 s18, s18, 1
1181; GFX7-NEXT:    v_mul_hi_u32 v10, s16, v9
1182; GFX7-NEXT:    v_add_i32_e32 v7, vcc, v11, v7
1183; GFX7-NEXT:    s_add_u32 s17, s17, s19
1184; GFX7-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
1185; GFX7-NEXT:    s_cselect_b32 s19, 1, 0
1186; GFX7-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
1187; GFX7-NEXT:    s_and_b32 s19, s19, 1
1188; GFX7-NEXT:    v_add_i32_e32 v7, vcc, v7, v8
1189; GFX7-NEXT:    s_mul_i32 s20, s2, s11
1190; GFX7-NEXT:    s_add_i32 s18, s18, s19
1191; GFX7-NEXT:    v_add_i32_e32 v4, vcc, v4, v10
1192; GFX7-NEXT:    s_add_u32 s17, s17, s20
1193; GFX7-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
1194; GFX7-NEXT:    s_cselect_b32 s19, 1, 0
1195; GFX7-NEXT:    v_add_i32_e32 v7, vcc, v7, v8
1196; GFX7-NEXT:    s_and_b32 s19, s19, 1
1197; GFX7-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
1198; GFX7-NEXT:    s_mul_i32 s21, s1, s12
1199; GFX7-NEXT:    s_add_i32 s18, s18, s19
1200; GFX7-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
1201; GFX7-NEXT:    s_add_u32 s17, s17, s21
1202; GFX7-NEXT:    v_add_i32_e32 v4, vcc, v7, v4
1203; GFX7-NEXT:    v_mov_b32_e32 v7, s4
1204; GFX7-NEXT:    s_cselect_b32 s19, 1, 0
1205; GFX7-NEXT:    v_mul_hi_u32 v8, v7, s8
1206; GFX7-NEXT:    s_and_b32 s19, s19, 1
1207; GFX7-NEXT:    s_mul_i32 s22, s16, s13
1208; GFX7-NEXT:    s_add_i32 s18, s18, s19
1209; GFX7-NEXT:    s_add_u32 s17, s17, s22
1210; GFX7-NEXT:    s_cselect_b32 s19, 1, 0
1211; GFX7-NEXT:    s_and_b32 s19, s19, 1
1212; GFX7-NEXT:    v_add_i32_e32 v8, vcc, s17, v8
1213; GFX7-NEXT:    v_mul_hi_u32 v10, v5, s9
1214; GFX7-NEXT:    s_add_i32 s18, s18, s19
1215; GFX7-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
1216; GFX7-NEXT:    v_add_i32_e32 v14, vcc, s18, v14
1217; GFX7-NEXT:    s_mul_i32 s17, s6, s8
1218; GFX7-NEXT:    s_mul_i32 s18, s5, s9
1219; GFX7-NEXT:    s_add_u32 s17, s17, s18
1220; GFX7-NEXT:    v_mul_hi_u32 v6, s2, v6
1221; GFX7-NEXT:    s_cselect_b32 s18, 1, 0
1222; GFX7-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
1223; GFX7-NEXT:    s_mul_i32 s19, s4, s10
1224; GFX7-NEXT:    s_and_b32 s18, s18, 1
1225; GFX7-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
1226; GFX7-NEXT:    s_add_u32 s17, s17, s19
1227; GFX7-NEXT:    v_mul_hi_u32 v11, s1, v9
1228; GFX7-NEXT:    v_add_i32_e32 v10, vcc, v14, v10
1229; GFX7-NEXT:    s_cselect_b32 s19, 1, 0
1230; GFX7-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
1231; GFX7-NEXT:    s_and_b32 s19, s19, 1
1232; GFX7-NEXT:    v_mov_b32_e32 v12, s12
1233; GFX7-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
1234; GFX7-NEXT:    s_mul_i32 s20, s3, s11
1235; GFX7-NEXT:    s_add_i32 s18, s18, s19
1236; GFX7-NEXT:    v_mul_hi_u32 v13, s16, v12
1237; GFX7-NEXT:    v_add_i32_e32 v8, vcc, v10, v8
1238; GFX7-NEXT:    s_add_u32 s17, s17, s20
1239; GFX7-NEXT:    v_add_i32_e32 v6, vcc, v6, v11
1240; GFX7-NEXT:    s_cselect_b32 s19, 1, 0
1241; GFX7-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
1242; GFX7-NEXT:    s_and_b32 s19, s19, 1
1243; GFX7-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
1244; GFX7-NEXT:    s_mul_i32 s21, s2, s12
1245; GFX7-NEXT:    s_add_i32 s18, s18, s19
1246; GFX7-NEXT:    v_add_i32_e32 v6, vcc, v6, v13
1247; GFX7-NEXT:    s_add_u32 s17, s17, s21
1248; GFX7-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
1249; GFX7-NEXT:    s_cselect_b32 s19, 1, 0
1250; GFX7-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
1251; GFX7-NEXT:    s_and_b32 s19, s19, 1
1252; GFX7-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
1253; GFX7-NEXT:    s_mul_i32 s22, s1, s13
1254; GFX7-NEXT:    s_add_i32 s18, s18, s19
1255; GFX7-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
1256; GFX7-NEXT:    s_add_u32 s17, s17, s22
1257; GFX7-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
1258; GFX7-NEXT:    v_mov_b32_e32 v8, s5
1259; GFX7-NEXT:    s_cselect_b32 s19, 1, 0
1260; GFX7-NEXT:    v_mul_hi_u32 v10, v8, s8
1261; GFX7-NEXT:    s_and_b32 s19, s19, 1
1262; GFX7-NEXT:    s_mul_i32 s23, s16, s14
1263; GFX7-NEXT:    s_add_i32 s18, s18, s19
1264; GFX7-NEXT:    s_add_u32 s17, s17, s23
1265; GFX7-NEXT:    v_mul_hi_u32 v11, v7, s9
1266; GFX7-NEXT:    s_cselect_b32 s19, 1, 0
1267; GFX7-NEXT:    s_and_b32 s19, s19, 1
1268; GFX7-NEXT:    v_add_i32_e32 v10, vcc, s17, v10
1269; GFX7-NEXT:    s_add_i32 s18, s18, s19
1270; GFX7-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
1271; GFX7-NEXT:    v_mul_hi_u32 v5, v5, s10
1272; GFX7-NEXT:    v_add_i32_e32 v17, vcc, s18, v17
1273; GFX7-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
1274; GFX7-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
1275; GFX7-NEXT:    v_mul_hi_u32 v13, s2, v9
1276; GFX7-NEXT:    v_add_i32_e32 v11, vcc, v17, v11
1277; GFX7-NEXT:    v_add_i32_e32 v5, vcc, v10, v5
1278; GFX7-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
1279; GFX7-NEXT:    v_mul_hi_u32 v14, s1, v12
1280; GFX7-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
1281; GFX7-NEXT:    v_add_i32_e32 v5, vcc, v5, v13
1282; GFX7-NEXT:    v_mov_b32_e32 v15, s13
1283; GFX7-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
1284; GFX7-NEXT:    v_mul_hi_u32 v16, s16, v15
1285; GFX7-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
1286; GFX7-NEXT:    v_add_i32_e32 v5, vcc, v5, v14
1287; GFX7-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
1288; GFX7-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
1289; GFX7-NEXT:    v_add_i32_e32 v5, vcc, v5, v16
1290; GFX7-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
1291; GFX7-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
1292; GFX7-NEXT:    s_mul_i32 s7, s7, s8
1293; GFX7-NEXT:    s_mul_i32 s17, s6, s9
1294; GFX7-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
1295; GFX7-NEXT:    s_mul_i32 s5, s5, s10
1296; GFX7-NEXT:    s_mul_i32 s4, s4, s11
1297; GFX7-NEXT:    s_mul_i32 s11, s3, s12
1298; GFX7-NEXT:    s_mul_i32 s12, s2, s13
1299; GFX7-NEXT:    s_mul_i32 s13, s1, s14
1300; GFX7-NEXT:    v_mul_hi_u32 v11, s2, v12
1301; GFX7-NEXT:    v_mul_hi_u32 v12, s1, v15
1302; GFX7-NEXT:    s_add_i32 s1, s7, s17
1303; GFX7-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
1304; GFX7-NEXT:    s_add_i32 s1, s1, s5
1305; GFX7-NEXT:    v_add_i32_e32 v6, vcc, v10, v6
1306; GFX7-NEXT:    v_mov_b32_e32 v10, s6
1307; GFX7-NEXT:    s_add_i32 s1, s1, s4
1308; GFX7-NEXT:    v_mul_hi_u32 v10, v10, s8
1309; GFX7-NEXT:    s_add_i32 s1, s1, s11
1310; GFX7-NEXT:    v_mul_hi_u32 v8, v8, s9
1311; GFX7-NEXT:    s_add_i32 s1, s1, s12
1312; GFX7-NEXT:    s_mul_i32 s15, s16, s15
1313; GFX7-NEXT:    v_mul_hi_u32 v7, v7, s10
1314; GFX7-NEXT:    s_add_i32 s1, s1, s13
1315; GFX7-NEXT:    v_mul_hi_u32 v9, s3, v9
1316; GFX7-NEXT:    s_add_i32 s1, s1, s15
1317; GFX7-NEXT:    v_add_i32_e32 v10, vcc, s1, v10
1318; GFX7-NEXT:    v_mov_b32_e32 v13, s14
1319; GFX7-NEXT:    v_add_i32_e32 v8, vcc, v10, v8
1320; GFX7-NEXT:    v_mul_hi_u32 v13, s16, v13
1321; GFX7-NEXT:    v_add_i32_e32 v7, vcc, v8, v7
1322; GFX7-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
1323; GFX7-NEXT:    v_add_i32_e32 v7, vcc, v7, v11
1324; GFX7-NEXT:    v_add_i32_e32 v7, vcc, v7, v12
1325; GFX7-NEXT:    v_add_i32_e32 v7, vcc, v7, v13
1326; GFX7-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
1327; GFX7-NEXT:    s_mul_i32 s0, s0, s8
1328; GFX7-NEXT:    v_readfirstlane_b32 s1, v0
1329; GFX7-NEXT:    v_readfirstlane_b32 s2, v1
1330; GFX7-NEXT:    v_readfirstlane_b32 s3, v2
1331; GFX7-NEXT:    v_readfirstlane_b32 s4, v3
1332; GFX7-NEXT:    v_readfirstlane_b32 s5, v4
1333; GFX7-NEXT:    v_readfirstlane_b32 s6, v5
1334; GFX7-NEXT:    v_readfirstlane_b32 s7, v6
1335; GFX7-NEXT:    ; return to shader part epilog
1336;
1337; GFX8-LABEL: s_mul_i256:
1338; GFX8:       ; %bb.0:
1339; GFX8-NEXT:    s_mov_b32 s16, s0
1340; GFX8-NEXT:    v_mov_b32_e32 v0, s8
1341; GFX8-NEXT:    v_mul_hi_u32 v0, s16, v0
1342; GFX8-NEXT:    s_mul_i32 s17, s1, s8
1343; GFX8-NEXT:    s_mul_i32 s18, s16, s9
1344; GFX8-NEXT:    s_add_u32 s17, s17, s18
1345; GFX8-NEXT:    s_cselect_b32 s18, 1, 0
1346; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s17, v0
1347; GFX8-NEXT:    s_and_b32 s18, s18, 1
1348; GFX8-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
1349; GFX8-NEXT:    v_add_u32_e32 v1, vcc, s18, v1
1350; GFX8-NEXT:    s_mul_i32 s17, s2, s8
1351; GFX8-NEXT:    s_mul_i32 s18, s1, s9
1352; GFX8-NEXT:    v_mov_b32_e32 v2, s1
1353; GFX8-NEXT:    s_add_u32 s17, s17, s18
1354; GFX8-NEXT:    v_mul_hi_u32 v2, v2, s8
1355; GFX8-NEXT:    s_cselect_b32 s18, 1, 0
1356; GFX8-NEXT:    s_mul_i32 s19, s16, s10
1357; GFX8-NEXT:    s_and_b32 s18, s18, 1
1358; GFX8-NEXT:    v_mov_b32_e32 v3, s9
1359; GFX8-NEXT:    s_add_u32 s17, s17, s19
1360; GFX8-NEXT:    v_mul_hi_u32 v4, s16, v3
1361; GFX8-NEXT:    s_cselect_b32 s19, 1, 0
1362; GFX8-NEXT:    s_and_b32 s19, s19, 1
1363; GFX8-NEXT:    v_add_u32_e32 v2, vcc, s17, v2
1364; GFX8-NEXT:    s_add_i32 s18, s18, s19
1365; GFX8-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
1366; GFX8-NEXT:    v_add_u32_e32 v5, vcc, s18, v5
1367; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v4
1368; GFX8-NEXT:    s_mul_i32 s17, s3, s8
1369; GFX8-NEXT:    s_mul_i32 s18, s2, s9
1370; GFX8-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
1371; GFX8-NEXT:    s_add_u32 s17, s17, s18
1372; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v5, v4
1373; GFX8-NEXT:    s_cselect_b32 s18, 1, 0
1374; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v2, v1
1375; GFX8-NEXT:    s_mul_i32 s19, s1, s10
1376; GFX8-NEXT:    s_and_b32 s18, s18, 1
1377; GFX8-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
1378; GFX8-NEXT:    s_add_u32 s17, s17, s19
1379; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v4, v2
1380; GFX8-NEXT:    v_mov_b32_e32 v4, s2
1381; GFX8-NEXT:    s_cselect_b32 s19, 1, 0
1382; GFX8-NEXT:    v_mul_hi_u32 v5, v4, s8
1383; GFX8-NEXT:    s_and_b32 s19, s19, 1
1384; GFX8-NEXT:    s_mul_i32 s20, s16, s11
1385; GFX8-NEXT:    s_add_i32 s18, s18, s19
1386; GFX8-NEXT:    s_add_u32 s17, s17, s20
1387; GFX8-NEXT:    v_mul_hi_u32 v3, s1, v3
1388; GFX8-NEXT:    s_cselect_b32 s19, 1, 0
1389; GFX8-NEXT:    s_and_b32 s19, s19, 1
1390; GFX8-NEXT:    v_add_u32_e32 v5, vcc, s17, v5
1391; GFX8-NEXT:    v_mov_b32_e32 v6, s10
1392; GFX8-NEXT:    s_add_i32 s18, s18, s19
1393; GFX8-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
1394; GFX8-NEXT:    v_mul_hi_u32 v7, s16, v6
1395; GFX8-NEXT:    v_add_u32_e32 v8, vcc, s18, v8
1396; GFX8-NEXT:    s_mul_i32 s17, s4, s8
1397; GFX8-NEXT:    s_mul_i32 s18, s3, s9
1398; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v5, v3
1399; GFX8-NEXT:    s_add_u32 s17, s17, s18
1400; GFX8-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
1401; GFX8-NEXT:    s_cselect_b32 s18, 1, 0
1402; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v8, v5
1403; GFX8-NEXT:    s_mul_i32 s19, s2, s10
1404; GFX8-NEXT:    s_and_b32 s18, s18, 1
1405; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v3, v7
1406; GFX8-NEXT:    s_add_u32 s17, s17, s19
1407; GFX8-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
1408; GFX8-NEXT:    s_cselect_b32 s19, 1, 0
1409; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v5, v7
1410; GFX8-NEXT:    s_and_b32 s19, s19, 1
1411; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v3, v2
1412; GFX8-NEXT:    s_mul_i32 s20, s1, s11
1413; GFX8-NEXT:    s_add_i32 s18, s18, s19
1414; GFX8-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
1415; GFX8-NEXT:    s_add_u32 s17, s17, s20
1416; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v5, v3
1417; GFX8-NEXT:    v_mov_b32_e32 v5, s3
1418; GFX8-NEXT:    s_cselect_b32 s19, 1, 0
1419; GFX8-NEXT:    v_mul_hi_u32 v7, v5, s8
1420; GFX8-NEXT:    s_and_b32 s19, s19, 1
1421; GFX8-NEXT:    s_mul_i32 s21, s16, s12
1422; GFX8-NEXT:    s_add_i32 s18, s18, s19
1423; GFX8-NEXT:    s_add_u32 s17, s17, s21
1424; GFX8-NEXT:    s_cselect_b32 s19, 1, 0
1425; GFX8-NEXT:    v_mul_hi_u32 v4, v4, s9
1426; GFX8-NEXT:    s_and_b32 s19, s19, 1
1427; GFX8-NEXT:    v_add_u32_e32 v7, vcc, s17, v7
1428; GFX8-NEXT:    s_add_i32 s18, s18, s19
1429; GFX8-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
1430; GFX8-NEXT:    v_add_u32_e32 v11, vcc, s18, v11
1431; GFX8-NEXT:    s_mul_i32 s17, s5, s8
1432; GFX8-NEXT:    s_mul_i32 s18, s4, s9
1433; GFX8-NEXT:    v_mul_hi_u32 v8, s1, v6
1434; GFX8-NEXT:    s_add_u32 s17, s17, s18
1435; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v7, v4
1436; GFX8-NEXT:    s_cselect_b32 s18, 1, 0
1437; GFX8-NEXT:    v_mov_b32_e32 v9, s11
1438; GFX8-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
1439; GFX8-NEXT:    s_mul_i32 s19, s3, s10
1440; GFX8-NEXT:    s_and_b32 s18, s18, 1
1441; GFX8-NEXT:    v_mul_hi_u32 v10, s16, v9
1442; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v11, v7
1443; GFX8-NEXT:    s_add_u32 s17, s17, s19
1444; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v4, v8
1445; GFX8-NEXT:    s_cselect_b32 s19, 1, 0
1446; GFX8-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
1447; GFX8-NEXT:    s_and_b32 s19, s19, 1
1448; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v7, v8
1449; GFX8-NEXT:    s_mul_i32 s20, s2, s11
1450; GFX8-NEXT:    s_add_i32 s18, s18, s19
1451; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v4, v10
1452; GFX8-NEXT:    s_add_u32 s17, s17, s20
1453; GFX8-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
1454; GFX8-NEXT:    s_cselect_b32 s19, 1, 0
1455; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v7, v8
1456; GFX8-NEXT:    s_and_b32 s19, s19, 1
1457; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v4, v3
1458; GFX8-NEXT:    s_mul_i32 s21, s1, s12
1459; GFX8-NEXT:    s_add_i32 s18, s18, s19
1460; GFX8-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
1461; GFX8-NEXT:    s_add_u32 s17, s17, s21
1462; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v7, v4
1463; GFX8-NEXT:    v_mov_b32_e32 v7, s4
1464; GFX8-NEXT:    s_cselect_b32 s19, 1, 0
1465; GFX8-NEXT:    v_mul_hi_u32 v8, v7, s8
1466; GFX8-NEXT:    s_and_b32 s19, s19, 1
1467; GFX8-NEXT:    s_mul_i32 s22, s16, s13
1468; GFX8-NEXT:    s_add_i32 s18, s18, s19
1469; GFX8-NEXT:    s_add_u32 s17, s17, s22
1470; GFX8-NEXT:    s_cselect_b32 s19, 1, 0
1471; GFX8-NEXT:    s_and_b32 s19, s19, 1
1472; GFX8-NEXT:    v_add_u32_e32 v8, vcc, s17, v8
1473; GFX8-NEXT:    v_mul_hi_u32 v10, v5, s9
1474; GFX8-NEXT:    s_add_i32 s18, s18, s19
1475; GFX8-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
1476; GFX8-NEXT:    v_add_u32_e32 v14, vcc, s18, v14
1477; GFX8-NEXT:    s_mul_i32 s17, s6, s8
1478; GFX8-NEXT:    s_mul_i32 s18, s5, s9
1479; GFX8-NEXT:    s_add_u32 s17, s17, s18
1480; GFX8-NEXT:    v_mul_hi_u32 v6, s2, v6
1481; GFX8-NEXT:    s_cselect_b32 s18, 1, 0
1482; GFX8-NEXT:    v_add_u32_e32 v8, vcc, v8, v10
1483; GFX8-NEXT:    s_mul_i32 s19, s4, s10
1484; GFX8-NEXT:    s_and_b32 s18, s18, 1
1485; GFX8-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
1486; GFX8-NEXT:    s_add_u32 s17, s17, s19
1487; GFX8-NEXT:    v_mul_hi_u32 v11, s1, v9
1488; GFX8-NEXT:    v_add_u32_e32 v10, vcc, v14, v10
1489; GFX8-NEXT:    s_cselect_b32 s19, 1, 0
1490; GFX8-NEXT:    v_add_u32_e32 v6, vcc, v8, v6
1491; GFX8-NEXT:    s_and_b32 s19, s19, 1
1492; GFX8-NEXT:    v_mov_b32_e32 v12, s12
1493; GFX8-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
1494; GFX8-NEXT:    s_mul_i32 s20, s3, s11
1495; GFX8-NEXT:    s_add_i32 s18, s18, s19
1496; GFX8-NEXT:    v_mul_hi_u32 v13, s16, v12
1497; GFX8-NEXT:    v_add_u32_e32 v8, vcc, v10, v8
1498; GFX8-NEXT:    s_add_u32 s17, s17, s20
1499; GFX8-NEXT:    v_add_u32_e32 v6, vcc, v6, v11
1500; GFX8-NEXT:    s_cselect_b32 s19, 1, 0
1501; GFX8-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
1502; GFX8-NEXT:    s_and_b32 s19, s19, 1
1503; GFX8-NEXT:    v_add_u32_e32 v8, vcc, v8, v10
1504; GFX8-NEXT:    s_mul_i32 s21, s2, s12
1505; GFX8-NEXT:    s_add_i32 s18, s18, s19
1506; GFX8-NEXT:    v_add_u32_e32 v6, vcc, v6, v13
1507; GFX8-NEXT:    s_add_u32 s17, s17, s21
1508; GFX8-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
1509; GFX8-NEXT:    s_cselect_b32 s19, 1, 0
1510; GFX8-NEXT:    v_add_u32_e32 v8, vcc, v8, v10
1511; GFX8-NEXT:    s_and_b32 s19, s19, 1
1512; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v6, v4
1513; GFX8-NEXT:    s_mul_i32 s22, s1, s13
1514; GFX8-NEXT:    s_add_i32 s18, s18, s19
1515; GFX8-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
1516; GFX8-NEXT:    s_add_u32 s17, s17, s22
1517; GFX8-NEXT:    v_add_u32_e32 v6, vcc, v8, v6
1518; GFX8-NEXT:    v_mov_b32_e32 v8, s5
1519; GFX8-NEXT:    s_cselect_b32 s19, 1, 0
1520; GFX8-NEXT:    v_mul_hi_u32 v10, v8, s8
1521; GFX8-NEXT:    s_and_b32 s19, s19, 1
1522; GFX8-NEXT:    s_mul_i32 s23, s16, s14
1523; GFX8-NEXT:    s_add_i32 s18, s18, s19
1524; GFX8-NEXT:    s_add_u32 s17, s17, s23
1525; GFX8-NEXT:    v_mul_hi_u32 v11, v7, s9
1526; GFX8-NEXT:    s_cselect_b32 s19, 1, 0
1527; GFX8-NEXT:    s_and_b32 s19, s19, 1
1528; GFX8-NEXT:    v_add_u32_e32 v10, vcc, s17, v10
1529; GFX8-NEXT:    s_add_i32 s18, s18, s19
1530; GFX8-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
1531; GFX8-NEXT:    v_mul_hi_u32 v5, v5, s10
1532; GFX8-NEXT:    v_add_u32_e32 v17, vcc, s18, v17
1533; GFX8-NEXT:    v_add_u32_e32 v10, vcc, v10, v11
1534; GFX8-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
1535; GFX8-NEXT:    v_mul_hi_u32 v13, s2, v9
1536; GFX8-NEXT:    v_add_u32_e32 v11, vcc, v17, v11
1537; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v10, v5
1538; GFX8-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
1539; GFX8-NEXT:    v_mul_hi_u32 v14, s1, v12
1540; GFX8-NEXT:    v_add_u32_e32 v10, vcc, v11, v10
1541; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v5, v13
1542; GFX8-NEXT:    v_mov_b32_e32 v15, s13
1543; GFX8-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
1544; GFX8-NEXT:    v_mul_hi_u32 v16, s16, v15
1545; GFX8-NEXT:    v_add_u32_e32 v10, vcc, v10, v11
1546; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v5, v14
1547; GFX8-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
1548; GFX8-NEXT:    v_add_u32_e32 v10, vcc, v10, v11
1549; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v5, v16
1550; GFX8-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
1551; GFX8-NEXT:    v_add_u32_e32 v10, vcc, v10, v11
1552; GFX8-NEXT:    s_mul_i32 s7, s7, s8
1553; GFX8-NEXT:    s_mul_i32 s17, s6, s9
1554; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v5, v6
1555; GFX8-NEXT:    s_mul_i32 s5, s5, s10
1556; GFX8-NEXT:    s_mul_i32 s4, s4, s11
1557; GFX8-NEXT:    s_mul_i32 s11, s3, s12
1558; GFX8-NEXT:    s_mul_i32 s12, s2, s13
1559; GFX8-NEXT:    s_mul_i32 s13, s1, s14
1560; GFX8-NEXT:    v_mul_hi_u32 v11, s2, v12
1561; GFX8-NEXT:    v_mul_hi_u32 v12, s1, v15
1562; GFX8-NEXT:    s_add_i32 s1, s7, s17
1563; GFX8-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
1564; GFX8-NEXT:    s_add_i32 s1, s1, s5
1565; GFX8-NEXT:    v_add_u32_e32 v6, vcc, v10, v6
1566; GFX8-NEXT:    v_mov_b32_e32 v10, s6
1567; GFX8-NEXT:    s_add_i32 s1, s1, s4
1568; GFX8-NEXT:    v_mul_hi_u32 v10, v10, s8
1569; GFX8-NEXT:    s_add_i32 s1, s1, s11
1570; GFX8-NEXT:    v_mul_hi_u32 v8, v8, s9
1571; GFX8-NEXT:    s_add_i32 s1, s1, s12
1572; GFX8-NEXT:    s_mul_i32 s15, s16, s15
1573; GFX8-NEXT:    v_mul_hi_u32 v7, v7, s10
1574; GFX8-NEXT:    s_add_i32 s1, s1, s13
1575; GFX8-NEXT:    v_mul_hi_u32 v9, s3, v9
1576; GFX8-NEXT:    s_add_i32 s1, s1, s15
1577; GFX8-NEXT:    v_add_u32_e32 v10, vcc, s1, v10
1578; GFX8-NEXT:    v_mov_b32_e32 v13, s14
1579; GFX8-NEXT:    v_add_u32_e32 v8, vcc, v10, v8
1580; GFX8-NEXT:    v_mul_hi_u32 v13, s16, v13
1581; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v8, v7
1582; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v7, v9
1583; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v7, v11
1584; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v7, v12
1585; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v7, v13
1586; GFX8-NEXT:    v_add_u32_e32 v6, vcc, v7, v6
1587; GFX8-NEXT:    s_mul_i32 s0, s0, s8
1588; GFX8-NEXT:    v_readfirstlane_b32 s1, v0
1589; GFX8-NEXT:    v_readfirstlane_b32 s2, v1
1590; GFX8-NEXT:    v_readfirstlane_b32 s3, v2
1591; GFX8-NEXT:    v_readfirstlane_b32 s4, v3
1592; GFX8-NEXT:    v_readfirstlane_b32 s5, v4
1593; GFX8-NEXT:    v_readfirstlane_b32 s6, v5
1594; GFX8-NEXT:    v_readfirstlane_b32 s7, v6
1595; GFX8-NEXT:    ; return to shader part epilog
1596;
1597; GFX9-LABEL: s_mul_i256:
1598; GFX9:       ; %bb.0:
1599; GFX9-NEXT:    s_mov_b32 s16, s0
1600; GFX9-NEXT:    s_mul_i32 s17, s1, s8
1601; GFX9-NEXT:    s_mul_i32 s18, s16, s9
1602; GFX9-NEXT:    s_add_u32 s17, s17, s18
1603; GFX9-NEXT:    s_cselect_b32 s18, 1, 0
1604; GFX9-NEXT:    s_mul_hi_u32 s19, s16, s8
1605; GFX9-NEXT:    s_and_b32 s18, s18, 1
1606; GFX9-NEXT:    s_add_u32 s17, s17, s19
1607; GFX9-NEXT:    s_cselect_b32 s19, 1, 0
1608; GFX9-NEXT:    s_and_b32 s19, s19, 1
1609; GFX9-NEXT:    s_add_i32 s18, s18, s19
1610; GFX9-NEXT:    s_mul_i32 s19, s2, s8
1611; GFX9-NEXT:    s_mul_i32 s20, s1, s9
1612; GFX9-NEXT:    s_add_u32 s19, s19, s20
1613; GFX9-NEXT:    s_cselect_b32 s20, 1, 0
1614; GFX9-NEXT:    s_mul_i32 s21, s16, s10
1615; GFX9-NEXT:    s_and_b32 s20, s20, 1
1616; GFX9-NEXT:    s_add_u32 s19, s19, s21
1617; GFX9-NEXT:    s_cselect_b32 s21, 1, 0
1618; GFX9-NEXT:    s_and_b32 s21, s21, 1
1619; GFX9-NEXT:    s_mul_hi_u32 s22, s1, s8
1620; GFX9-NEXT:    s_add_i32 s20, s20, s21
1621; GFX9-NEXT:    s_add_u32 s19, s19, s22
1622; GFX9-NEXT:    s_cselect_b32 s21, 1, 0
1623; GFX9-NEXT:    s_and_b32 s21, s21, 1
1624; GFX9-NEXT:    s_mul_hi_u32 s23, s16, s9
1625; GFX9-NEXT:    s_add_i32 s20, s20, s21
1626; GFX9-NEXT:    s_add_u32 s19, s19, s23
1627; GFX9-NEXT:    s_cselect_b32 s21, 1, 0
1628; GFX9-NEXT:    s_and_b32 s21, s21, 1
1629; GFX9-NEXT:    s_add_i32 s20, s20, s21
1630; GFX9-NEXT:    s_add_u32 s18, s19, s18
1631; GFX9-NEXT:    s_cselect_b32 s19, 1, 0
1632; GFX9-NEXT:    s_and_b32 s19, s19, 1
1633; GFX9-NEXT:    s_add_i32 s20, s20, s19
1634; GFX9-NEXT:    s_mul_i32 s19, s3, s8
1635; GFX9-NEXT:    s_mul_i32 s21, s2, s9
1636; GFX9-NEXT:    s_add_u32 s19, s19, s21
1637; GFX9-NEXT:    s_cselect_b32 s21, 1, 0
1638; GFX9-NEXT:    s_mul_i32 s22, s1, s10
1639; GFX9-NEXT:    s_and_b32 s21, s21, 1
1640; GFX9-NEXT:    s_add_u32 s19, s19, s22
1641; GFX9-NEXT:    s_cselect_b32 s22, 1, 0
1642; GFX9-NEXT:    s_and_b32 s22, s22, 1
1643; GFX9-NEXT:    s_mul_i32 s23, s16, s11
1644; GFX9-NEXT:    s_add_i32 s21, s21, s22
1645; GFX9-NEXT:    s_add_u32 s19, s19, s23
1646; GFX9-NEXT:    s_cselect_b32 s22, 1, 0
1647; GFX9-NEXT:    s_and_b32 s22, s22, 1
1648; GFX9-NEXT:    s_mul_hi_u32 s24, s2, s8
1649; GFX9-NEXT:    s_add_i32 s21, s21, s22
1650; GFX9-NEXT:    s_add_u32 s19, s19, s24
1651; GFX9-NEXT:    s_cselect_b32 s22, 1, 0
1652; GFX9-NEXT:    s_and_b32 s22, s22, 1
1653; GFX9-NEXT:    s_mul_hi_u32 s25, s1, s9
1654; GFX9-NEXT:    s_add_i32 s21, s21, s22
1655; GFX9-NEXT:    s_add_u32 s19, s19, s25
1656; GFX9-NEXT:    s_cselect_b32 s22, 1, 0
1657; GFX9-NEXT:    s_and_b32 s22, s22, 1
1658; GFX9-NEXT:    s_mul_hi_u32 s26, s16, s10
1659; GFX9-NEXT:    s_add_i32 s21, s21, s22
1660; GFX9-NEXT:    s_add_u32 s19, s19, s26
1661; GFX9-NEXT:    s_cselect_b32 s22, 1, 0
1662; GFX9-NEXT:    s_and_b32 s22, s22, 1
1663; GFX9-NEXT:    s_add_i32 s21, s21, s22
1664; GFX9-NEXT:    s_add_u32 s19, s19, s20
1665; GFX9-NEXT:    s_cselect_b32 s20, 1, 0
1666; GFX9-NEXT:    s_and_b32 s20, s20, 1
1667; GFX9-NEXT:    s_add_i32 s21, s21, s20
1668; GFX9-NEXT:    s_mul_i32 s20, s4, s8
1669; GFX9-NEXT:    s_mul_i32 s22, s3, s9
1670; GFX9-NEXT:    s_add_u32 s20, s20, s22
1671; GFX9-NEXT:    s_cselect_b32 s22, 1, 0
1672; GFX9-NEXT:    s_mul_i32 s23, s2, s10
1673; GFX9-NEXT:    s_and_b32 s22, s22, 1
1674; GFX9-NEXT:    s_add_u32 s20, s20, s23
1675; GFX9-NEXT:    s_cselect_b32 s23, 1, 0
1676; GFX9-NEXT:    s_and_b32 s23, s23, 1
1677; GFX9-NEXT:    s_mul_i32 s24, s1, s11
1678; GFX9-NEXT:    s_add_i32 s22, s22, s23
1679; GFX9-NEXT:    s_add_u32 s20, s20, s24
1680; GFX9-NEXT:    s_cselect_b32 s23, 1, 0
1681; GFX9-NEXT:    s_and_b32 s23, s23, 1
1682; GFX9-NEXT:    s_mul_i32 s25, s16, s12
1683; GFX9-NEXT:    s_add_i32 s22, s22, s23
1684; GFX9-NEXT:    s_add_u32 s20, s20, s25
1685; GFX9-NEXT:    s_cselect_b32 s23, 1, 0
1686; GFX9-NEXT:    s_and_b32 s23, s23, 1
1687; GFX9-NEXT:    s_mul_hi_u32 s26, s3, s8
1688; GFX9-NEXT:    s_add_i32 s22, s22, s23
1689; GFX9-NEXT:    s_add_u32 s20, s20, s26
1690; GFX9-NEXT:    s_cselect_b32 s23, 1, 0
1691; GFX9-NEXT:    s_and_b32 s23, s23, 1
1692; GFX9-NEXT:    s_mul_hi_u32 s27, s2, s9
1693; GFX9-NEXT:    s_add_i32 s22, s22, s23
1694; GFX9-NEXT:    s_add_u32 s20, s20, s27
1695; GFX9-NEXT:    s_cselect_b32 s23, 1, 0
1696; GFX9-NEXT:    s_and_b32 s23, s23, 1
1697; GFX9-NEXT:    s_mul_hi_u32 s28, s1, s10
1698; GFX9-NEXT:    s_add_i32 s22, s22, s23
1699; GFX9-NEXT:    s_add_u32 s20, s20, s28
1700; GFX9-NEXT:    s_cselect_b32 s23, 1, 0
1701; GFX9-NEXT:    s_and_b32 s23, s23, 1
1702; GFX9-NEXT:    s_mul_hi_u32 s29, s16, s11
1703; GFX9-NEXT:    s_add_i32 s22, s22, s23
1704; GFX9-NEXT:    s_add_u32 s20, s20, s29
1705; GFX9-NEXT:    s_cselect_b32 s23, 1, 0
1706; GFX9-NEXT:    s_and_b32 s23, s23, 1
1707; GFX9-NEXT:    s_add_i32 s22, s22, s23
1708; GFX9-NEXT:    s_add_u32 s20, s20, s21
1709; GFX9-NEXT:    s_cselect_b32 s21, 1, 0
1710; GFX9-NEXT:    s_and_b32 s21, s21, 1
1711; GFX9-NEXT:    s_add_i32 s22, s22, s21
1712; GFX9-NEXT:    s_mul_i32 s21, s5, s8
1713; GFX9-NEXT:    s_mul_i32 s23, s4, s9
1714; GFX9-NEXT:    s_add_u32 s21, s21, s23
1715; GFX9-NEXT:    s_cselect_b32 s23, 1, 0
1716; GFX9-NEXT:    s_mul_i32 s24, s3, s10
1717; GFX9-NEXT:    s_and_b32 s23, s23, 1
1718; GFX9-NEXT:    s_add_u32 s21, s21, s24
1719; GFX9-NEXT:    s_cselect_b32 s24, 1, 0
1720; GFX9-NEXT:    s_and_b32 s24, s24, 1
1721; GFX9-NEXT:    s_mul_i32 s25, s2, s11
1722; GFX9-NEXT:    s_add_i32 s23, s23, s24
1723; GFX9-NEXT:    s_add_u32 s21, s21, s25
1724; GFX9-NEXT:    s_cselect_b32 s24, 1, 0
1725; GFX9-NEXT:    s_and_b32 s24, s24, 1
1726; GFX9-NEXT:    s_mul_i32 s26, s1, s12
1727; GFX9-NEXT:    s_add_i32 s23, s23, s24
1728; GFX9-NEXT:    s_add_u32 s21, s21, s26
1729; GFX9-NEXT:    s_cselect_b32 s24, 1, 0
1730; GFX9-NEXT:    s_and_b32 s24, s24, 1
1731; GFX9-NEXT:    s_mul_i32 s27, s16, s13
1732; GFX9-NEXT:    s_add_i32 s23, s23, s24
1733; GFX9-NEXT:    s_add_u32 s21, s21, s27
1734; GFX9-NEXT:    s_cselect_b32 s24, 1, 0
1735; GFX9-NEXT:    s_and_b32 s24, s24, 1
1736; GFX9-NEXT:    s_mul_hi_u32 s28, s4, s8
1737; GFX9-NEXT:    s_add_i32 s23, s23, s24
1738; GFX9-NEXT:    s_add_u32 s21, s21, s28
1739; GFX9-NEXT:    s_cselect_b32 s24, 1, 0
1740; GFX9-NEXT:    s_and_b32 s24, s24, 1
1741; GFX9-NEXT:    s_mul_hi_u32 s29, s3, s9
1742; GFX9-NEXT:    s_add_i32 s23, s23, s24
1743; GFX9-NEXT:    s_add_u32 s21, s21, s29
1744; GFX9-NEXT:    s_cselect_b32 s24, 1, 0
1745; GFX9-NEXT:    s_and_b32 s24, s24, 1
1746; GFX9-NEXT:    s_mul_hi_u32 s30, s2, s10
1747; GFX9-NEXT:    s_add_i32 s23, s23, s24
1748; GFX9-NEXT:    s_add_u32 s21, s21, s30
1749; GFX9-NEXT:    s_cselect_b32 s24, 1, 0
1750; GFX9-NEXT:    s_and_b32 s24, s24, 1
1751; GFX9-NEXT:    s_mul_hi_u32 s31, s1, s11
1752; GFX9-NEXT:    s_add_i32 s23, s23, s24
1753; GFX9-NEXT:    s_add_u32 s21, s21, s31
1754; GFX9-NEXT:    s_cselect_b32 s24, 1, 0
1755; GFX9-NEXT:    s_and_b32 s24, s24, 1
1756; GFX9-NEXT:    s_mul_hi_u32 s33, s16, s12
1757; GFX9-NEXT:    s_add_i32 s23, s23, s24
1758; GFX9-NEXT:    s_add_u32 s21, s21, s33
1759; GFX9-NEXT:    s_cselect_b32 s24, 1, 0
1760; GFX9-NEXT:    s_and_b32 s24, s24, 1
1761; GFX9-NEXT:    s_add_i32 s23, s23, s24
1762; GFX9-NEXT:    s_add_u32 s21, s21, s22
1763; GFX9-NEXT:    s_cselect_b32 s22, 1, 0
1764; GFX9-NEXT:    s_and_b32 s22, s22, 1
1765; GFX9-NEXT:    s_add_i32 s23, s23, s22
1766; GFX9-NEXT:    s_mul_i32 s22, s6, s8
1767; GFX9-NEXT:    s_mul_i32 s24, s5, s9
1768; GFX9-NEXT:    s_add_u32 s22, s22, s24
1769; GFX9-NEXT:    s_cselect_b32 s24, 1, 0
1770; GFX9-NEXT:    s_mul_i32 s25, s4, s10
1771; GFX9-NEXT:    s_and_b32 s24, s24, 1
1772; GFX9-NEXT:    s_add_u32 s22, s22, s25
1773; GFX9-NEXT:    s_cselect_b32 s25, 1, 0
1774; GFX9-NEXT:    s_and_b32 s25, s25, 1
1775; GFX9-NEXT:    s_mul_i32 s26, s3, s11
1776; GFX9-NEXT:    s_add_i32 s24, s24, s25
1777; GFX9-NEXT:    s_add_u32 s22, s22, s26
1778; GFX9-NEXT:    s_cselect_b32 s25, 1, 0
1779; GFX9-NEXT:    s_and_b32 s25, s25, 1
1780; GFX9-NEXT:    s_mul_i32 s27, s2, s12
1781; GFX9-NEXT:    s_add_i32 s24, s24, s25
1782; GFX9-NEXT:    s_add_u32 s22, s22, s27
1783; GFX9-NEXT:    s_cselect_b32 s25, 1, 0
1784; GFX9-NEXT:    s_and_b32 s25, s25, 1
1785; GFX9-NEXT:    s_mul_i32 s28, s1, s13
1786; GFX9-NEXT:    s_add_i32 s24, s24, s25
1787; GFX9-NEXT:    s_add_u32 s22, s22, s28
1788; GFX9-NEXT:    s_cselect_b32 s25, 1, 0
1789; GFX9-NEXT:    s_and_b32 s25, s25, 1
1790; GFX9-NEXT:    s_mul_i32 s29, s16, s14
1791; GFX9-NEXT:    s_add_i32 s24, s24, s25
1792; GFX9-NEXT:    s_add_u32 s22, s22, s29
1793; GFX9-NEXT:    s_cselect_b32 s25, 1, 0
1794; GFX9-NEXT:    s_and_b32 s25, s25, 1
1795; GFX9-NEXT:    s_mul_hi_u32 s30, s5, s8
1796; GFX9-NEXT:    s_add_i32 s24, s24, s25
1797; GFX9-NEXT:    s_add_u32 s22, s22, s30
1798; GFX9-NEXT:    s_cselect_b32 s25, 1, 0
1799; GFX9-NEXT:    s_and_b32 s25, s25, 1
1800; GFX9-NEXT:    s_mul_hi_u32 s31, s4, s9
1801; GFX9-NEXT:    s_add_i32 s24, s24, s25
1802; GFX9-NEXT:    s_add_u32 s22, s22, s31
1803; GFX9-NEXT:    s_cselect_b32 s25, 1, 0
1804; GFX9-NEXT:    s_and_b32 s25, s25, 1
1805; GFX9-NEXT:    s_mul_hi_u32 s33, s3, s10
1806; GFX9-NEXT:    s_add_i32 s24, s24, s25
1807; GFX9-NEXT:    s_add_u32 s22, s22, s33
1808; GFX9-NEXT:    s_cselect_b32 s25, 1, 0
1809; GFX9-NEXT:    s_and_b32 s25, s25, 1
1810; GFX9-NEXT:    s_mul_hi_u32 s34, s2, s11
1811; GFX9-NEXT:    s_add_i32 s24, s24, s25
1812; GFX9-NEXT:    s_add_u32 s22, s22, s34
1813; GFX9-NEXT:    s_cselect_b32 s25, 1, 0
1814; GFX9-NEXT:    s_and_b32 s25, s25, 1
1815; GFX9-NEXT:    s_mul_hi_u32 s35, s1, s12
1816; GFX9-NEXT:    s_add_i32 s24, s24, s25
1817; GFX9-NEXT:    s_add_u32 s22, s22, s35
1818; GFX9-NEXT:    s_cselect_b32 s25, 1, 0
1819; GFX9-NEXT:    s_and_b32 s25, s25, 1
1820; GFX9-NEXT:    s_mul_hi_u32 s36, s16, s13
1821; GFX9-NEXT:    s_add_i32 s24, s24, s25
1822; GFX9-NEXT:    s_add_u32 s22, s22, s36
1823; GFX9-NEXT:    s_cselect_b32 s25, 1, 0
1824; GFX9-NEXT:    s_and_b32 s25, s25, 1
1825; GFX9-NEXT:    s_add_i32 s24, s24, s25
1826; GFX9-NEXT:    s_add_u32 s22, s22, s23
1827; GFX9-NEXT:    s_cselect_b32 s23, 1, 0
1828; GFX9-NEXT:    s_and_b32 s23, s23, 1
1829; GFX9-NEXT:    s_add_i32 s24, s24, s23
1830; GFX9-NEXT:    s_mul_i32 s7, s7, s8
1831; GFX9-NEXT:    s_mul_i32 s23, s6, s9
1832; GFX9-NEXT:    s_mul_i32 s25, s5, s10
1833; GFX9-NEXT:    s_add_i32 s7, s7, s23
1834; GFX9-NEXT:    s_mul_i32 s26, s4, s11
1835; GFX9-NEXT:    s_add_i32 s7, s7, s25
1836; GFX9-NEXT:    s_mul_i32 s27, s3, s12
1837; GFX9-NEXT:    s_add_i32 s7, s7, s26
1838; GFX9-NEXT:    s_mul_i32 s28, s2, s13
1839; GFX9-NEXT:    s_add_i32 s7, s7, s27
1840; GFX9-NEXT:    s_mul_i32 s29, s1, s14
1841; GFX9-NEXT:    s_add_i32 s7, s7, s28
1842; GFX9-NEXT:    s_mul_i32 s15, s16, s15
1843; GFX9-NEXT:    s_add_i32 s7, s7, s29
1844; GFX9-NEXT:    s_mul_hi_u32 s6, s6, s8
1845; GFX9-NEXT:    s_add_i32 s7, s7, s15
1846; GFX9-NEXT:    s_mul_hi_u32 s5, s5, s9
1847; GFX9-NEXT:    s_add_i32 s6, s7, s6
1848; GFX9-NEXT:    s_mul_hi_u32 s4, s4, s10
1849; GFX9-NEXT:    s_add_i32 s5, s6, s5
1850; GFX9-NEXT:    s_mul_hi_u32 s3, s3, s11
1851; GFX9-NEXT:    s_add_i32 s4, s5, s4
1852; GFX9-NEXT:    s_mul_hi_u32 s2, s2, s12
1853; GFX9-NEXT:    s_add_i32 s3, s4, s3
1854; GFX9-NEXT:    s_mul_hi_u32 s1, s1, s13
1855; GFX9-NEXT:    s_add_i32 s2, s3, s2
1856; GFX9-NEXT:    s_mul_i32 s0, s0, s8
1857; GFX9-NEXT:    s_mul_hi_u32 s8, s16, s14
1858; GFX9-NEXT:    s_add_i32 s1, s2, s1
1859; GFX9-NEXT:    s_add_i32 s1, s1, s8
1860; GFX9-NEXT:    s_add_i32 s7, s1, s24
1861; GFX9-NEXT:    s_mov_b32 s1, s17
1862; GFX9-NEXT:    s_mov_b32 s2, s18
1863; GFX9-NEXT:    s_mov_b32 s3, s19
1864; GFX9-NEXT:    s_mov_b32 s4, s20
1865; GFX9-NEXT:    s_mov_b32 s5, s21
1866; GFX9-NEXT:    s_mov_b32 s6, s22
1867; GFX9-NEXT:    ; return to shader part epilog
1868;
1869; GFX10-LABEL: s_mul_i256:
1870; GFX10:       ; %bb.0:
1871; GFX10-NEXT:    s_mul_i32 s16, s1, s8
1872; GFX10-NEXT:    s_mul_i32 s17, s0, s9
1873; GFX10-NEXT:    s_mul_hi_u32 s18, s0, s8
1874; GFX10-NEXT:    s_add_u32 s16, s16, s17
1875; GFX10-NEXT:    s_cselect_b32 s17, 1, 0
1876; GFX10-NEXT:    s_mul_i32 s19, s1, s9
1877; GFX10-NEXT:    s_and_b32 s17, s17, 1
1878; GFX10-NEXT:    s_add_u32 s16, s16, s18
1879; GFX10-NEXT:    s_cselect_b32 s18, 1, 0
1880; GFX10-NEXT:    s_mul_i32 s20, s0, s10
1881; GFX10-NEXT:    s_and_b32 s18, s18, 1
1882; GFX10-NEXT:    s_mul_hi_u32 s21, s1, s8
1883; GFX10-NEXT:    s_add_i32 s17, s17, s18
1884; GFX10-NEXT:    s_mul_i32 s18, s2, s8
1885; GFX10-NEXT:    s_mul_i32 s22, s0, s11
1886; GFX10-NEXT:    s_add_u32 s18, s18, s19
1887; GFX10-NEXT:    s_cselect_b32 s19, 1, 0
1888; GFX10-NEXT:    s_mul_i32 s23, s1, s11
1889; GFX10-NEXT:    s_and_b32 s19, s19, 1
1890; GFX10-NEXT:    s_add_u32 s18, s18, s20
1891; GFX10-NEXT:    s_cselect_b32 s20, 1, 0
1892; GFX10-NEXT:    s_mul_i32 s24, s0, s12
1893; GFX10-NEXT:    s_and_b32 s20, s20, 1
1894; GFX10-NEXT:    s_mul_i32 s25, s4, s9
1895; GFX10-NEXT:    s_add_i32 s19, s19, s20
1896; GFX10-NEXT:    s_add_u32 s18, s18, s21
1897; GFX10-NEXT:    s_cselect_b32 s20, 1, 0
1898; GFX10-NEXT:    s_mul_hi_u32 s21, s0, s9
1899; GFX10-NEXT:    s_and_b32 s20, s20, 1
1900; GFX10-NEXT:    s_mul_i32 s26, s2, s11
1901; GFX10-NEXT:    s_add_i32 s19, s19, s20
1902; GFX10-NEXT:    s_add_u32 s18, s18, s21
1903; GFX10-NEXT:    s_cselect_b32 s20, 1, 0
1904; GFX10-NEXT:    s_mul_i32 s21, s1, s10
1905; GFX10-NEXT:    s_and_b32 s20, s20, 1
1906; GFX10-NEXT:    s_mul_i32 s27, s0, s13
1907; GFX10-NEXT:    s_add_i32 s19, s19, s20
1908; GFX10-NEXT:    s_add_u32 s17, s18, s17
1909; GFX10-NEXT:    s_cselect_b32 s18, 1, 0
1910; GFX10-NEXT:    s_mul_i32 s20, s2, s9
1911; GFX10-NEXT:    s_and_b32 s18, s18, 1
1912; GFX10-NEXT:    s_mul_hi_u32 s28, s3, s9
1913; GFX10-NEXT:    s_add_i32 s19, s19, s18
1914; GFX10-NEXT:    s_mul_i32 s18, s3, s8
1915; GFX10-NEXT:    s_mul_i32 s7, s7, s8
1916; GFX10-NEXT:    s_add_u32 s18, s18, s20
1917; GFX10-NEXT:    s_cselect_b32 s20, 1, 0
1918; GFX10-NEXT:    s_mul_i32 s15, s0, s15
1919; GFX10-NEXT:    s_and_b32 s20, s20, 1
1920; GFX10-NEXT:    s_add_u32 s18, s18, s21
1921; GFX10-NEXT:    s_cselect_b32 s21, 1, 0
1922; GFX10-NEXT:    s_and_b32 s21, s21, 1
1923; GFX10-NEXT:    s_add_i32 s20, s20, s21
1924; GFX10-NEXT:    s_add_u32 s18, s18, s22
1925; GFX10-NEXT:    s_cselect_b32 s21, 1, 0
1926; GFX10-NEXT:    s_mul_hi_u32 s22, s2, s8
1927; GFX10-NEXT:    s_and_b32 s21, s21, 1
1928; GFX10-NEXT:    s_add_i32 s20, s20, s21
1929; GFX10-NEXT:    s_add_u32 s18, s18, s22
1930; GFX10-NEXT:    s_cselect_b32 s21, 1, 0
1931; GFX10-NEXT:    s_mul_hi_u32 s22, s1, s9
1932; GFX10-NEXT:    s_and_b32 s21, s21, 1
1933; GFX10-NEXT:    s_add_i32 s20, s20, s21
1934; GFX10-NEXT:    s_add_u32 s18, s18, s22
1935; GFX10-NEXT:    s_cselect_b32 s21, 1, 0
1936; GFX10-NEXT:    s_mul_hi_u32 s22, s0, s10
1937; GFX10-NEXT:    s_and_b32 s21, s21, 1
1938; GFX10-NEXT:    s_add_i32 s20, s20, s21
1939; GFX10-NEXT:    s_add_u32 s18, s18, s22
1940; GFX10-NEXT:    s_cselect_b32 s21, 1, 0
1941; GFX10-NEXT:    s_mul_i32 s22, s2, s10
1942; GFX10-NEXT:    s_and_b32 s21, s21, 1
1943; GFX10-NEXT:    s_add_i32 s20, s20, s21
1944; GFX10-NEXT:    s_add_u32 s18, s18, s19
1945; GFX10-NEXT:    s_cselect_b32 s19, 1, 0
1946; GFX10-NEXT:    s_mul_i32 s21, s3, s9
1947; GFX10-NEXT:    s_and_b32 s19, s19, 1
1948; GFX10-NEXT:    s_add_i32 s20, s20, s19
1949; GFX10-NEXT:    s_mul_i32 s19, s4, s8
1950; GFX10-NEXT:    s_add_u32 s19, s19, s21
1951; GFX10-NEXT:    s_cselect_b32 s21, 1, 0
1952; GFX10-NEXT:    s_and_b32 s21, s21, 1
1953; GFX10-NEXT:    s_add_u32 s19, s19, s22
1954; GFX10-NEXT:    s_cselect_b32 s22, 1, 0
1955; GFX10-NEXT:    s_and_b32 s22, s22, 1
1956; GFX10-NEXT:    s_add_i32 s21, s21, s22
1957; GFX10-NEXT:    s_add_u32 s19, s19, s23
1958; GFX10-NEXT:    s_cselect_b32 s22, 1, 0
1959; GFX10-NEXT:    s_mul_hi_u32 s23, s3, s8
1960; GFX10-NEXT:    s_and_b32 s22, s22, 1
1961; GFX10-NEXT:    s_add_i32 s21, s21, s22
1962; GFX10-NEXT:    s_add_u32 s19, s19, s24
1963; GFX10-NEXT:    s_cselect_b32 s22, 1, 0
1964; GFX10-NEXT:    s_mul_hi_u32 s24, s2, s9
1965; GFX10-NEXT:    s_and_b32 s22, s22, 1
1966; GFX10-NEXT:    s_add_i32 s21, s21, s22
1967; GFX10-NEXT:    s_add_u32 s19, s19, s23
1968; GFX10-NEXT:    s_cselect_b32 s22, 1, 0
1969; GFX10-NEXT:    s_mul_hi_u32 s23, s1, s10
1970; GFX10-NEXT:    s_and_b32 s22, s22, 1
1971; GFX10-NEXT:    s_add_i32 s21, s21, s22
1972; GFX10-NEXT:    s_add_u32 s19, s19, s24
1973; GFX10-NEXT:    s_cselect_b32 s22, 1, 0
1974; GFX10-NEXT:    s_mul_hi_u32 s24, s0, s11
1975; GFX10-NEXT:    s_and_b32 s22, s22, 1
1976; GFX10-NEXT:    s_add_i32 s21, s21, s22
1977; GFX10-NEXT:    s_add_u32 s19, s19, s23
1978; GFX10-NEXT:    s_cselect_b32 s22, 1, 0
1979; GFX10-NEXT:    s_mul_i32 s23, s5, s8
1980; GFX10-NEXT:    s_and_b32 s22, s22, 1
1981; GFX10-NEXT:    s_add_i32 s21, s21, s22
1982; GFX10-NEXT:    s_add_u32 s19, s19, s24
1983; GFX10-NEXT:    s_cselect_b32 s22, 1, 0
1984; GFX10-NEXT:    s_mul_i32 s24, s3, s10
1985; GFX10-NEXT:    s_and_b32 s22, s22, 1
1986; GFX10-NEXT:    s_add_i32 s21, s21, s22
1987; GFX10-NEXT:    s_add_u32 s19, s19, s20
1988; GFX10-NEXT:    s_cselect_b32 s20, 1, 0
1989; GFX10-NEXT:    s_mul_i32 s22, s1, s12
1990; GFX10-NEXT:    s_and_b32 s20, s20, 1
1991; GFX10-NEXT:    s_add_i32 s21, s21, s20
1992; GFX10-NEXT:    s_add_u32 s23, s23, s25
1993; GFX10-NEXT:    s_cselect_b32 s25, 1, 0
1994; GFX10-NEXT:    s_mul_hi_u32 s20, s4, s8
1995; GFX10-NEXT:    s_and_b32 s25, s25, 1
1996; GFX10-NEXT:    s_add_u32 s23, s23, s24
1997; GFX10-NEXT:    s_cselect_b32 s24, 1, 0
1998; GFX10-NEXT:    s_and_b32 s24, s24, 1
1999; GFX10-NEXT:    s_add_i32 s24, s25, s24
2000; GFX10-NEXT:    s_add_u32 s23, s23, s26
2001; GFX10-NEXT:    s_cselect_b32 s25, 1, 0
2002; GFX10-NEXT:    s_mul_hi_u32 s26, s2, s10
2003; GFX10-NEXT:    s_and_b32 s25, s25, 1
2004; GFX10-NEXT:    s_add_i32 s24, s24, s25
2005; GFX10-NEXT:    s_add_u32 s22, s23, s22
2006; GFX10-NEXT:    s_cselect_b32 s23, 1, 0
2007; GFX10-NEXT:    s_mul_hi_u32 s25, s1, s11
2008; GFX10-NEXT:    s_and_b32 s23, s23, 1
2009; GFX10-NEXT:    s_add_i32 s23, s24, s23
2010; GFX10-NEXT:    s_add_u32 s22, s22, s27
2011; GFX10-NEXT:    s_cselect_b32 s24, 1, 0
2012; GFX10-NEXT:    s_mul_hi_u32 s27, s0, s12
2013; GFX10-NEXT:    s_and_b32 s24, s24, 1
2014; GFX10-NEXT:    s_add_i32 s23, s23, s24
2015; GFX10-NEXT:    s_add_u32 s20, s22, s20
2016; GFX10-NEXT:    s_cselect_b32 s22, 1, 0
2017; GFX10-NEXT:    s_mul_i32 s24, s6, s8
2018; GFX10-NEXT:    s_and_b32 s22, s22, 1
2019; GFX10-NEXT:    s_add_i32 s22, s23, s22
2020; GFX10-NEXT:    s_add_u32 s20, s20, s28
2021; GFX10-NEXT:    s_cselect_b32 s23, 1, 0
2022; GFX10-NEXT:    s_mul_i32 s28, s5, s9
2023; GFX10-NEXT:    s_and_b32 s23, s23, 1
2024; GFX10-NEXT:    s_add_i32 s22, s22, s23
2025; GFX10-NEXT:    s_add_u32 s20, s20, s26
2026; GFX10-NEXT:    s_cselect_b32 s23, 1, 0
2027; GFX10-NEXT:    s_mul_i32 s26, s4, s10
2028; GFX10-NEXT:    s_and_b32 s23, s23, 1
2029; GFX10-NEXT:    s_add_i32 s22, s22, s23
2030; GFX10-NEXT:    s_add_u32 s20, s20, s25
2031; GFX10-NEXT:    s_cselect_b32 s23, 1, 0
2032; GFX10-NEXT:    s_mul_i32 s25, s3, s11
2033; GFX10-NEXT:    s_and_b32 s23, s23, 1
2034; GFX10-NEXT:    s_add_i32 s22, s22, s23
2035; GFX10-NEXT:    s_add_u32 s20, s20, s27
2036; GFX10-NEXT:    s_cselect_b32 s23, 1, 0
2037; GFX10-NEXT:    s_mul_i32 s27, s2, s12
2038; GFX10-NEXT:    s_and_b32 s23, s23, 1
2039; GFX10-NEXT:    s_add_i32 s22, s22, s23
2040; GFX10-NEXT:    s_add_u32 s20, s20, s21
2041; GFX10-NEXT:    s_cselect_b32 s21, 1, 0
2042; GFX10-NEXT:    s_mul_i32 s23, s1, s13
2043; GFX10-NEXT:    s_and_b32 s21, s21, 1
2044; GFX10-NEXT:    s_add_i32 s22, s22, s21
2045; GFX10-NEXT:    s_add_u32 s21, s24, s28
2046; GFX10-NEXT:    s_cselect_b32 s24, 1, 0
2047; GFX10-NEXT:    s_mul_i32 s28, s0, s14
2048; GFX10-NEXT:    s_and_b32 s24, s24, 1
2049; GFX10-NEXT:    s_add_u32 s21, s21, s26
2050; GFX10-NEXT:    s_cselect_b32 s26, 1, 0
2051; GFX10-NEXT:    s_and_b32 s26, s26, 1
2052; GFX10-NEXT:    s_add_i32 s24, s24, s26
2053; GFX10-NEXT:    s_add_u32 s21, s21, s25
2054; GFX10-NEXT:    s_cselect_b32 s25, 1, 0
2055; GFX10-NEXT:    s_mul_hi_u32 s26, s5, s8
2056; GFX10-NEXT:    s_and_b32 s25, s25, 1
2057; GFX10-NEXT:    s_add_i32 s24, s24, s25
2058; GFX10-NEXT:    s_add_u32 s21, s21, s27
2059; GFX10-NEXT:    s_cselect_b32 s25, 1, 0
2060; GFX10-NEXT:    s_mul_hi_u32 s27, s4, s9
2061; GFX10-NEXT:    s_and_b32 s25, s25, 1
2062; GFX10-NEXT:    s_add_i32 s24, s24, s25
2063; GFX10-NEXT:    s_add_u32 s21, s21, s23
2064; GFX10-NEXT:    s_cselect_b32 s23, 1, 0
2065; GFX10-NEXT:    s_mul_hi_u32 s25, s3, s10
2066; GFX10-NEXT:    s_and_b32 s23, s23, 1
2067; GFX10-NEXT:    s_add_i32 s23, s24, s23
2068; GFX10-NEXT:    s_add_u32 s21, s21, s28
2069; GFX10-NEXT:    s_cselect_b32 s24, 1, 0
2070; GFX10-NEXT:    s_mul_hi_u32 s28, s2, s11
2071; GFX10-NEXT:    s_and_b32 s24, s24, 1
2072; GFX10-NEXT:    s_add_i32 s23, s23, s24
2073; GFX10-NEXT:    s_add_u32 s21, s21, s26
2074; GFX10-NEXT:    s_cselect_b32 s24, 1, 0
2075; GFX10-NEXT:    s_mul_hi_u32 s26, s1, s12
2076; GFX10-NEXT:    s_and_b32 s24, s24, 1
2077; GFX10-NEXT:    s_add_i32 s23, s23, s24
2078; GFX10-NEXT:    s_add_u32 s21, s21, s27
2079; GFX10-NEXT:    s_cselect_b32 s24, 1, 0
2080; GFX10-NEXT:    s_mul_hi_u32 s27, s0, s13
2081; GFX10-NEXT:    s_and_b32 s24, s24, 1
2082; GFX10-NEXT:    s_add_i32 s23, s23, s24
2083; GFX10-NEXT:    s_add_u32 s21, s21, s25
2084; GFX10-NEXT:    s_cselect_b32 s24, 1, 0
2085; GFX10-NEXT:    s_mul_i32 s25, s6, s9
2086; GFX10-NEXT:    s_and_b32 s24, s24, 1
2087; GFX10-NEXT:    s_mul_hi_u32 s6, s6, s8
2088; GFX10-NEXT:    s_add_i32 s23, s23, s24
2089; GFX10-NEXT:    s_add_u32 s21, s21, s28
2090; GFX10-NEXT:    s_cselect_b32 s24, 1, 0
2091; GFX10-NEXT:    s_and_b32 s24, s24, 1
2092; GFX10-NEXT:    s_add_i32 s23, s23, s24
2093; GFX10-NEXT:    s_add_u32 s21, s21, s26
2094; GFX10-NEXT:    s_cselect_b32 s24, 1, 0
2095; GFX10-NEXT:    s_mul_i32 s26, s5, s10
2096; GFX10-NEXT:    s_and_b32 s24, s24, 1
2097; GFX10-NEXT:    s_mul_hi_u32 s5, s5, s9
2098; GFX10-NEXT:    s_add_i32 s23, s23, s24
2099; GFX10-NEXT:    s_add_u32 s21, s21, s27
2100; GFX10-NEXT:    s_cselect_b32 s24, 1, 0
2101; GFX10-NEXT:    s_mul_i32 s27, s4, s11
2102; GFX10-NEXT:    s_and_b32 s24, s24, 1
2103; GFX10-NEXT:    s_mul_hi_u32 s4, s4, s10
2104; GFX10-NEXT:    s_add_i32 s23, s23, s24
2105; GFX10-NEXT:    s_add_u32 s21, s21, s22
2106; GFX10-NEXT:    s_cselect_b32 s22, 1, 0
2107; GFX10-NEXT:    s_add_i32 s7, s7, s25
2108; GFX10-NEXT:    s_mul_i32 s24, s3, s12
2109; GFX10-NEXT:    s_add_i32 s7, s7, s26
2110; GFX10-NEXT:    s_mul_i32 s25, s2, s13
2111; GFX10-NEXT:    s_add_i32 s7, s7, s27
2112; GFX10-NEXT:    s_mul_i32 s26, s1, s14
2113; GFX10-NEXT:    s_add_i32 s7, s7, s24
2114; GFX10-NEXT:    s_mul_hi_u32 s3, s3, s11
2115; GFX10-NEXT:    s_add_i32 s7, s7, s25
2116; GFX10-NEXT:    s_mul_hi_u32 s2, s2, s12
2117; GFX10-NEXT:    s_add_i32 s7, s7, s26
2118; GFX10-NEXT:    s_mul_hi_u32 s1, s1, s13
2119; GFX10-NEXT:    s_add_i32 s7, s7, s15
2120; GFX10-NEXT:    s_add_i32 s6, s7, s6
2121; GFX10-NEXT:    s_add_i32 s5, s6, s5
2122; GFX10-NEXT:    s_mov_b32 s6, s21
2123; GFX10-NEXT:    s_add_i32 s4, s5, s4
2124; GFX10-NEXT:    s_mov_b32 s5, s20
2125; GFX10-NEXT:    s_add_i32 s3, s4, s3
2126; GFX10-NEXT:    s_mul_hi_u32 s4, s0, s14
2127; GFX10-NEXT:    s_add_i32 s2, s3, s2
2128; GFX10-NEXT:    s_and_b32 s3, s22, 1
2129; GFX10-NEXT:    s_add_i32 s1, s2, s1
2130; GFX10-NEXT:    s_add_i32 s23, s23, s3
2131; GFX10-NEXT:    s_add_i32 s1, s1, s4
2132; GFX10-NEXT:    s_mul_i32 s0, s0, s8
2133; GFX10-NEXT:    s_add_i32 s7, s1, s23
2134; GFX10-NEXT:    s_mov_b32 s1, s16
2135; GFX10-NEXT:    s_mov_b32 s2, s17
2136; GFX10-NEXT:    s_mov_b32 s3, s18
2137; GFX10-NEXT:    s_mov_b32 s4, s19
2138; GFX10-NEXT:    ; return to shader part epilog
2139  %result = mul i256 %num, %den
2140  %cast = bitcast i256 %result to <8 x i32>
2141  ret <8 x i32> %cast
2142}
2143
2144define i256 @v_mul_i256(i256 %num, i256 %den) {
2145; GFX7-LABEL: v_mul_i256:
2146; GFX7:       ; %bb.0:
2147; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2148; GFX7-NEXT:    v_mul_lo_u32 v16, v1, v8
2149; GFX7-NEXT:    v_mul_lo_u32 v17, v0, v9
2150; GFX7-NEXT:    v_mul_hi_u32 v18, v0, v8
2151; GFX7-NEXT:    v_mul_lo_u32 v19, v2, v8
2152; GFX7-NEXT:    v_mul_lo_u32 v20, v1, v9
2153; GFX7-NEXT:    v_add_i32_e32 v16, vcc, v16, v17
2154; GFX7-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
2155; GFX7-NEXT:    v_add_i32_e32 v16, vcc, v16, v18
2156; GFX7-NEXT:    v_cndmask_b32_e64 v18, 0, 1, vcc
2157; GFX7-NEXT:    v_add_i32_e32 v17, vcc, v17, v18
2158; GFX7-NEXT:    v_mul_lo_u32 v18, v0, v10
2159; GFX7-NEXT:    v_add_i32_e32 v19, vcc, v19, v20
2160; GFX7-NEXT:    v_mul_hi_u32 v21, v1, v8
2161; GFX7-NEXT:    v_cndmask_b32_e64 v20, 0, 1, vcc
2162; GFX7-NEXT:    v_add_i32_e32 v18, vcc, v19, v18
2163; GFX7-NEXT:    v_cndmask_b32_e64 v19, 0, 1, vcc
2164; GFX7-NEXT:    v_add_i32_e32 v19, vcc, v20, v19
2165; GFX7-NEXT:    v_add_i32_e32 v18, vcc, v18, v21
2166; GFX7-NEXT:    v_mul_hi_u32 v21, v0, v9
2167; GFX7-NEXT:    v_cndmask_b32_e64 v20, 0, 1, vcc
2168; GFX7-NEXT:    v_add_i32_e32 v19, vcc, v19, v20
2169; GFX7-NEXT:    v_add_i32_e32 v18, vcc, v18, v21
2170; GFX7-NEXT:    v_cndmask_b32_e64 v20, 0, 1, vcc
2171; GFX7-NEXT:    v_add_i32_e32 v19, vcc, v19, v20
2172; GFX7-NEXT:    v_add_i32_e32 v17, vcc, v18, v17
2173; GFX7-NEXT:    v_cndmask_b32_e64 v18, 0, 1, vcc
2174; GFX7-NEXT:    v_mul_lo_u32 v20, v3, v8
2175; GFX7-NEXT:    v_mul_lo_u32 v21, v2, v9
2176; GFX7-NEXT:    v_add_i32_e32 v18, vcc, v19, v18
2177; GFX7-NEXT:    v_mul_lo_u32 v19, v1, v10
2178; GFX7-NEXT:    v_add_i32_e32 v20, vcc, v20, v21
2179; GFX7-NEXT:    v_mul_lo_u32 v22, v0, v11
2180; GFX7-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2181; GFX7-NEXT:    v_add_i32_e32 v19, vcc, v20, v19
2182; GFX7-NEXT:    v_cndmask_b32_e64 v20, 0, 1, vcc
2183; GFX7-NEXT:    v_add_i32_e32 v20, vcc, v21, v20
2184; GFX7-NEXT:    v_add_i32_e32 v19, vcc, v19, v22
2185; GFX7-NEXT:    v_mul_hi_u32 v22, v2, v8
2186; GFX7-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2187; GFX7-NEXT:    v_add_i32_e32 v20, vcc, v20, v21
2188; GFX7-NEXT:    v_add_i32_e32 v19, vcc, v19, v22
2189; GFX7-NEXT:    v_mul_hi_u32 v22, v1, v9
2190; GFX7-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2191; GFX7-NEXT:    v_add_i32_e32 v20, vcc, v20, v21
2192; GFX7-NEXT:    v_add_i32_e32 v19, vcc, v19, v22
2193; GFX7-NEXT:    v_mul_hi_u32 v22, v0, v10
2194; GFX7-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2195; GFX7-NEXT:    v_add_i32_e32 v20, vcc, v20, v21
2196; GFX7-NEXT:    v_add_i32_e32 v19, vcc, v19, v22
2197; GFX7-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2198; GFX7-NEXT:    v_add_i32_e32 v20, vcc, v20, v21
2199; GFX7-NEXT:    v_add_i32_e32 v18, vcc, v19, v18
2200; GFX7-NEXT:    v_cndmask_b32_e64 v19, 0, 1, vcc
2201; GFX7-NEXT:    v_mul_lo_u32 v21, v4, v8
2202; GFX7-NEXT:    v_mul_lo_u32 v22, v3, v9
2203; GFX7-NEXT:    v_add_i32_e32 v19, vcc, v20, v19
2204; GFX7-NEXT:    v_mul_lo_u32 v20, v2, v10
2205; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v21, v22
2206; GFX7-NEXT:    v_mul_lo_u32 v23, v1, v11
2207; GFX7-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2208; GFX7-NEXT:    v_add_i32_e32 v20, vcc, v21, v20
2209; GFX7-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2210; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v22, v21
2211; GFX7-NEXT:    v_add_i32_e32 v20, vcc, v20, v23
2212; GFX7-NEXT:    v_mul_lo_u32 v23, v0, v12
2213; GFX7-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2214; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v21, v22
2215; GFX7-NEXT:    v_add_i32_e32 v20, vcc, v20, v23
2216; GFX7-NEXT:    v_mul_hi_u32 v23, v3, v8
2217; GFX7-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2218; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v21, v22
2219; GFX7-NEXT:    v_add_i32_e32 v20, vcc, v20, v23
2220; GFX7-NEXT:    v_mul_hi_u32 v23, v2, v9
2221; GFX7-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2222; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v21, v22
2223; GFX7-NEXT:    v_add_i32_e32 v20, vcc, v20, v23
2224; GFX7-NEXT:    v_mul_hi_u32 v23, v1, v10
2225; GFX7-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2226; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v21, v22
2227; GFX7-NEXT:    v_add_i32_e32 v20, vcc, v20, v23
2228; GFX7-NEXT:    v_mul_hi_u32 v23, v0, v11
2229; GFX7-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2230; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v21, v22
2231; GFX7-NEXT:    v_add_i32_e32 v20, vcc, v20, v23
2232; GFX7-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2233; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v21, v22
2234; GFX7-NEXT:    v_add_i32_e32 v19, vcc, v20, v19
2235; GFX7-NEXT:    v_cndmask_b32_e64 v20, 0, 1, vcc
2236; GFX7-NEXT:    v_mul_lo_u32 v22, v5, v8
2237; GFX7-NEXT:    v_mul_lo_u32 v23, v4, v9
2238; GFX7-NEXT:    v_add_i32_e32 v20, vcc, v21, v20
2239; GFX7-NEXT:    v_mul_lo_u32 v21, v3, v10
2240; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v23
2241; GFX7-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2242; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v22, v21
2243; GFX7-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2244; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v23, v22
2245; GFX7-NEXT:    v_mul_lo_u32 v23, v2, v11
2246; GFX7-NEXT:    v_mul_lo_u32 v7, v7, v8
2247; GFX7-NEXT:    v_mul_lo_u32 v15, v0, v15
2248; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v21, v23
2249; GFX7-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2250; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v23
2251; GFX7-NEXT:    v_mul_lo_u32 v23, v1, v12
2252; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v21, v23
2253; GFX7-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2254; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v23
2255; GFX7-NEXT:    v_mul_lo_u32 v23, v0, v13
2256; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v21, v23
2257; GFX7-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2258; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v23
2259; GFX7-NEXT:    v_mul_hi_u32 v23, v4, v8
2260; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v21, v23
2261; GFX7-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2262; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v23
2263; GFX7-NEXT:    v_mul_hi_u32 v23, v3, v9
2264; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v21, v23
2265; GFX7-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2266; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v23
2267; GFX7-NEXT:    v_mul_hi_u32 v23, v2, v10
2268; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v21, v23
2269; GFX7-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2270; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v23
2271; GFX7-NEXT:    v_mul_hi_u32 v23, v1, v11
2272; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v21, v23
2273; GFX7-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2274; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v23
2275; GFX7-NEXT:    v_mul_hi_u32 v23, v0, v12
2276; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v21, v23
2277; GFX7-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2278; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v23
2279; GFX7-NEXT:    v_add_i32_e32 v20, vcc, v21, v20
2280; GFX7-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2281; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v22, v21
2282; GFX7-NEXT:    v_mul_lo_u32 v22, v6, v8
2283; GFX7-NEXT:    v_mul_lo_u32 v23, v5, v9
2284; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v23
2285; GFX7-NEXT:    v_mul_lo_u32 v23, v4, v10
2286; GFX7-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2287; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v23
2288; GFX7-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2289; GFX7-NEXT:    v_add_i32_e32 v23, vcc, v24, v23
2290; GFX7-NEXT:    v_mul_lo_u32 v24, v3, v11
2291; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v24
2292; GFX7-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2293; GFX7-NEXT:    v_add_i32_e32 v23, vcc, v23, v24
2294; GFX7-NEXT:    v_mul_lo_u32 v24, v2, v12
2295; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v24
2296; GFX7-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2297; GFX7-NEXT:    v_add_i32_e32 v23, vcc, v23, v24
2298; GFX7-NEXT:    v_mul_lo_u32 v24, v1, v13
2299; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v24
2300; GFX7-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2301; GFX7-NEXT:    v_add_i32_e32 v23, vcc, v23, v24
2302; GFX7-NEXT:    v_mul_lo_u32 v24, v0, v14
2303; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v24
2304; GFX7-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2305; GFX7-NEXT:    v_add_i32_e32 v23, vcc, v23, v24
2306; GFX7-NEXT:    v_mul_hi_u32 v24, v5, v8
2307; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v24
2308; GFX7-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2309; GFX7-NEXT:    v_add_i32_e32 v23, vcc, v23, v24
2310; GFX7-NEXT:    v_mul_hi_u32 v24, v4, v9
2311; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v24
2312; GFX7-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2313; GFX7-NEXT:    v_add_i32_e32 v23, vcc, v23, v24
2314; GFX7-NEXT:    v_mul_hi_u32 v24, v3, v10
2315; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v24
2316; GFX7-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2317; GFX7-NEXT:    v_add_i32_e32 v23, vcc, v23, v24
2318; GFX7-NEXT:    v_mul_hi_u32 v24, v2, v11
2319; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v24
2320; GFX7-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2321; GFX7-NEXT:    v_add_i32_e32 v23, vcc, v23, v24
2322; GFX7-NEXT:    v_mul_hi_u32 v24, v1, v12
2323; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v24
2324; GFX7-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2325; GFX7-NEXT:    v_add_i32_e32 v23, vcc, v23, v24
2326; GFX7-NEXT:    v_mul_hi_u32 v24, v0, v13
2327; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v24
2328; GFX7-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2329; GFX7-NEXT:    v_add_i32_e32 v23, vcc, v23, v24
2330; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v22, v21
2331; GFX7-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2332; GFX7-NEXT:    v_add_i32_e32 v23, vcc, v23, v22
2333; GFX7-NEXT:    v_mul_lo_u32 v22, v0, v8
2334; GFX7-NEXT:    v_mul_hi_u32 v8, v6, v8
2335; GFX7-NEXT:    v_mul_lo_u32 v6, v6, v9
2336; GFX7-NEXT:    v_mul_hi_u32 v9, v5, v9
2337; GFX7-NEXT:    v_mul_lo_u32 v5, v5, v10
2338; GFX7-NEXT:    v_mul_hi_u32 v10, v4, v10
2339; GFX7-NEXT:    v_mul_lo_u32 v4, v4, v11
2340; GFX7-NEXT:    v_mul_hi_u32 v11, v3, v11
2341; GFX7-NEXT:    v_mul_lo_u32 v3, v3, v12
2342; GFX7-NEXT:    v_mul_hi_u32 v12, v2, v12
2343; GFX7-NEXT:    v_mul_lo_u32 v2, v2, v13
2344; GFX7-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
2345; GFX7-NEXT:    v_mul_hi_u32 v13, v1, v13
2346; GFX7-NEXT:    v_mul_lo_u32 v1, v1, v14
2347; GFX7-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
2348; GFX7-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
2349; GFX7-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
2350; GFX7-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
2351; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
2352; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v1, v15
2353; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v1, v8
2354; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v1, v9
2355; GFX7-NEXT:    v_mul_hi_u32 v0, v0, v14
2356; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v1, v10
2357; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v1, v11
2358; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v1, v12
2359; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v1, v13
2360; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
2361; GFX7-NEXT:    v_add_i32_e32 v7, vcc, v0, v23
2362; GFX7-NEXT:    v_mov_b32_e32 v0, v22
2363; GFX7-NEXT:    v_mov_b32_e32 v1, v16
2364; GFX7-NEXT:    v_mov_b32_e32 v2, v17
2365; GFX7-NEXT:    v_mov_b32_e32 v3, v18
2366; GFX7-NEXT:    v_mov_b32_e32 v4, v19
2367; GFX7-NEXT:    v_mov_b32_e32 v5, v20
2368; GFX7-NEXT:    v_mov_b32_e32 v6, v21
2369; GFX7-NEXT:    s_setpc_b64 s[30:31]
2370;
2371; GFX8-LABEL: v_mul_i256:
2372; GFX8:       ; %bb.0:
2373; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2374; GFX8-NEXT:    v_mul_lo_u32 v16, v1, v8
2375; GFX8-NEXT:    v_mul_lo_u32 v17, v0, v9
2376; GFX8-NEXT:    v_mul_hi_u32 v18, v0, v8
2377; GFX8-NEXT:    v_mul_lo_u32 v19, v2, v8
2378; GFX8-NEXT:    v_mul_lo_u32 v20, v1, v9
2379; GFX8-NEXT:    v_add_u32_e32 v16, vcc, v16, v17
2380; GFX8-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
2381; GFX8-NEXT:    v_add_u32_e32 v16, vcc, v16, v18
2382; GFX8-NEXT:    v_cndmask_b32_e64 v18, 0, 1, vcc
2383; GFX8-NEXT:    v_add_u32_e32 v17, vcc, v17, v18
2384; GFX8-NEXT:    v_mul_lo_u32 v18, v0, v10
2385; GFX8-NEXT:    v_add_u32_e32 v19, vcc, v19, v20
2386; GFX8-NEXT:    v_mul_hi_u32 v21, v1, v8
2387; GFX8-NEXT:    v_cndmask_b32_e64 v20, 0, 1, vcc
2388; GFX8-NEXT:    v_add_u32_e32 v18, vcc, v19, v18
2389; GFX8-NEXT:    v_cndmask_b32_e64 v19, 0, 1, vcc
2390; GFX8-NEXT:    v_add_u32_e32 v19, vcc, v20, v19
2391; GFX8-NEXT:    v_add_u32_e32 v18, vcc, v18, v21
2392; GFX8-NEXT:    v_mul_hi_u32 v21, v0, v9
2393; GFX8-NEXT:    v_cndmask_b32_e64 v20, 0, 1, vcc
2394; GFX8-NEXT:    v_add_u32_e32 v19, vcc, v19, v20
2395; GFX8-NEXT:    v_add_u32_e32 v18, vcc, v18, v21
2396; GFX8-NEXT:    v_cndmask_b32_e64 v20, 0, 1, vcc
2397; GFX8-NEXT:    v_add_u32_e32 v19, vcc, v19, v20
2398; GFX8-NEXT:    v_add_u32_e32 v17, vcc, v18, v17
2399; GFX8-NEXT:    v_cndmask_b32_e64 v18, 0, 1, vcc
2400; GFX8-NEXT:    v_mul_lo_u32 v20, v3, v8
2401; GFX8-NEXT:    v_mul_lo_u32 v21, v2, v9
2402; GFX8-NEXT:    v_add_u32_e32 v18, vcc, v19, v18
2403; GFX8-NEXT:    v_mul_lo_u32 v19, v1, v10
2404; GFX8-NEXT:    v_add_u32_e32 v20, vcc, v20, v21
2405; GFX8-NEXT:    v_mul_lo_u32 v22, v0, v11
2406; GFX8-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2407; GFX8-NEXT:    v_add_u32_e32 v19, vcc, v20, v19
2408; GFX8-NEXT:    v_cndmask_b32_e64 v20, 0, 1, vcc
2409; GFX8-NEXT:    v_add_u32_e32 v20, vcc, v21, v20
2410; GFX8-NEXT:    v_add_u32_e32 v19, vcc, v19, v22
2411; GFX8-NEXT:    v_mul_hi_u32 v22, v2, v8
2412; GFX8-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2413; GFX8-NEXT:    v_add_u32_e32 v20, vcc, v20, v21
2414; GFX8-NEXT:    v_add_u32_e32 v19, vcc, v19, v22
2415; GFX8-NEXT:    v_mul_hi_u32 v22, v1, v9
2416; GFX8-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2417; GFX8-NEXT:    v_add_u32_e32 v20, vcc, v20, v21
2418; GFX8-NEXT:    v_add_u32_e32 v19, vcc, v19, v22
2419; GFX8-NEXT:    v_mul_hi_u32 v22, v0, v10
2420; GFX8-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2421; GFX8-NEXT:    v_add_u32_e32 v20, vcc, v20, v21
2422; GFX8-NEXT:    v_add_u32_e32 v19, vcc, v19, v22
2423; GFX8-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2424; GFX8-NEXT:    v_add_u32_e32 v20, vcc, v20, v21
2425; GFX8-NEXT:    v_add_u32_e32 v18, vcc, v19, v18
2426; GFX8-NEXT:    v_cndmask_b32_e64 v19, 0, 1, vcc
2427; GFX8-NEXT:    v_mul_lo_u32 v21, v4, v8
2428; GFX8-NEXT:    v_mul_lo_u32 v22, v3, v9
2429; GFX8-NEXT:    v_add_u32_e32 v19, vcc, v20, v19
2430; GFX8-NEXT:    v_mul_lo_u32 v20, v2, v10
2431; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v21, v22
2432; GFX8-NEXT:    v_mul_lo_u32 v23, v1, v11
2433; GFX8-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2434; GFX8-NEXT:    v_add_u32_e32 v20, vcc, v21, v20
2435; GFX8-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2436; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v22, v21
2437; GFX8-NEXT:    v_add_u32_e32 v20, vcc, v20, v23
2438; GFX8-NEXT:    v_mul_lo_u32 v23, v0, v12
2439; GFX8-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2440; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v21, v22
2441; GFX8-NEXT:    v_add_u32_e32 v20, vcc, v20, v23
2442; GFX8-NEXT:    v_mul_hi_u32 v23, v3, v8
2443; GFX8-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2444; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v21, v22
2445; GFX8-NEXT:    v_add_u32_e32 v20, vcc, v20, v23
2446; GFX8-NEXT:    v_mul_hi_u32 v23, v2, v9
2447; GFX8-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2448; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v21, v22
2449; GFX8-NEXT:    v_add_u32_e32 v20, vcc, v20, v23
2450; GFX8-NEXT:    v_mul_hi_u32 v23, v1, v10
2451; GFX8-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2452; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v21, v22
2453; GFX8-NEXT:    v_add_u32_e32 v20, vcc, v20, v23
2454; GFX8-NEXT:    v_mul_hi_u32 v23, v0, v11
2455; GFX8-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2456; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v21, v22
2457; GFX8-NEXT:    v_add_u32_e32 v20, vcc, v20, v23
2458; GFX8-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2459; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v21, v22
2460; GFX8-NEXT:    v_add_u32_e32 v19, vcc, v20, v19
2461; GFX8-NEXT:    v_cndmask_b32_e64 v20, 0, 1, vcc
2462; GFX8-NEXT:    v_mul_lo_u32 v22, v5, v8
2463; GFX8-NEXT:    v_mul_lo_u32 v23, v4, v9
2464; GFX8-NEXT:    v_add_u32_e32 v20, vcc, v21, v20
2465; GFX8-NEXT:    v_mul_lo_u32 v21, v3, v10
2466; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v23
2467; GFX8-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2468; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v22, v21
2469; GFX8-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2470; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v23, v22
2471; GFX8-NEXT:    v_mul_lo_u32 v23, v2, v11
2472; GFX8-NEXT:    v_mul_lo_u32 v7, v7, v8
2473; GFX8-NEXT:    v_mul_lo_u32 v15, v0, v15
2474; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v21, v23
2475; GFX8-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2476; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v23
2477; GFX8-NEXT:    v_mul_lo_u32 v23, v1, v12
2478; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v21, v23
2479; GFX8-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2480; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v23
2481; GFX8-NEXT:    v_mul_lo_u32 v23, v0, v13
2482; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v21, v23
2483; GFX8-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2484; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v23
2485; GFX8-NEXT:    v_mul_hi_u32 v23, v4, v8
2486; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v21, v23
2487; GFX8-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2488; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v23
2489; GFX8-NEXT:    v_mul_hi_u32 v23, v3, v9
2490; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v21, v23
2491; GFX8-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2492; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v23
2493; GFX8-NEXT:    v_mul_hi_u32 v23, v2, v10
2494; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v21, v23
2495; GFX8-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2496; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v23
2497; GFX8-NEXT:    v_mul_hi_u32 v23, v1, v11
2498; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v21, v23
2499; GFX8-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2500; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v23
2501; GFX8-NEXT:    v_mul_hi_u32 v23, v0, v12
2502; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v21, v23
2503; GFX8-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2504; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v23
2505; GFX8-NEXT:    v_add_u32_e32 v20, vcc, v21, v20
2506; GFX8-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2507; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v22, v21
2508; GFX8-NEXT:    v_mul_lo_u32 v22, v6, v8
2509; GFX8-NEXT:    v_mul_lo_u32 v23, v5, v9
2510; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v23
2511; GFX8-NEXT:    v_mul_lo_u32 v23, v4, v10
2512; GFX8-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2513; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v23
2514; GFX8-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2515; GFX8-NEXT:    v_add_u32_e32 v23, vcc, v24, v23
2516; GFX8-NEXT:    v_mul_lo_u32 v24, v3, v11
2517; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v24
2518; GFX8-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2519; GFX8-NEXT:    v_add_u32_e32 v23, vcc, v23, v24
2520; GFX8-NEXT:    v_mul_lo_u32 v24, v2, v12
2521; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v24
2522; GFX8-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2523; GFX8-NEXT:    v_add_u32_e32 v23, vcc, v23, v24
2524; GFX8-NEXT:    v_mul_lo_u32 v24, v1, v13
2525; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v24
2526; GFX8-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2527; GFX8-NEXT:    v_add_u32_e32 v23, vcc, v23, v24
2528; GFX8-NEXT:    v_mul_lo_u32 v24, v0, v14
2529; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v24
2530; GFX8-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2531; GFX8-NEXT:    v_add_u32_e32 v23, vcc, v23, v24
2532; GFX8-NEXT:    v_mul_hi_u32 v24, v5, v8
2533; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v24
2534; GFX8-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2535; GFX8-NEXT:    v_add_u32_e32 v23, vcc, v23, v24
2536; GFX8-NEXT:    v_mul_hi_u32 v24, v4, v9
2537; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v24
2538; GFX8-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2539; GFX8-NEXT:    v_add_u32_e32 v23, vcc, v23, v24
2540; GFX8-NEXT:    v_mul_hi_u32 v24, v3, v10
2541; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v24
2542; GFX8-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2543; GFX8-NEXT:    v_add_u32_e32 v23, vcc, v23, v24
2544; GFX8-NEXT:    v_mul_hi_u32 v24, v2, v11
2545; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v24
2546; GFX8-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2547; GFX8-NEXT:    v_add_u32_e32 v23, vcc, v23, v24
2548; GFX8-NEXT:    v_mul_hi_u32 v24, v1, v12
2549; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v24
2550; GFX8-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2551; GFX8-NEXT:    v_add_u32_e32 v23, vcc, v23, v24
2552; GFX8-NEXT:    v_mul_hi_u32 v24, v0, v13
2553; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v24
2554; GFX8-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2555; GFX8-NEXT:    v_add_u32_e32 v23, vcc, v23, v24
2556; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v22, v21
2557; GFX8-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2558; GFX8-NEXT:    v_add_u32_e32 v23, vcc, v23, v22
2559; GFX8-NEXT:    v_mul_lo_u32 v22, v0, v8
2560; GFX8-NEXT:    v_mul_hi_u32 v8, v6, v8
2561; GFX8-NEXT:    v_mul_lo_u32 v6, v6, v9
2562; GFX8-NEXT:    v_mul_hi_u32 v9, v5, v9
2563; GFX8-NEXT:    v_mul_lo_u32 v5, v5, v10
2564; GFX8-NEXT:    v_mul_hi_u32 v10, v4, v10
2565; GFX8-NEXT:    v_mul_lo_u32 v4, v4, v11
2566; GFX8-NEXT:    v_mul_hi_u32 v11, v3, v11
2567; GFX8-NEXT:    v_mul_lo_u32 v3, v3, v12
2568; GFX8-NEXT:    v_mul_hi_u32 v12, v2, v12
2569; GFX8-NEXT:    v_mul_lo_u32 v2, v2, v13
2570; GFX8-NEXT:    v_add_u32_e32 v6, vcc, v7, v6
2571; GFX8-NEXT:    v_mul_hi_u32 v13, v1, v13
2572; GFX8-NEXT:    v_mul_lo_u32 v1, v1, v14
2573; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v6, v5
2574; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v5, v4
2575; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v4, v3
2576; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v3, v2
2577; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v2, v1
2578; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v15
2579; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v8
2580; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v9
2581; GFX8-NEXT:    v_mul_hi_u32 v0, v0, v14
2582; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v10
2583; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v11
2584; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v12
2585; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v13
2586; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v1, v0
2587; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v0, v23
2588; GFX8-NEXT:    v_mov_b32_e32 v0, v22
2589; GFX8-NEXT:    v_mov_b32_e32 v1, v16
2590; GFX8-NEXT:    v_mov_b32_e32 v2, v17
2591; GFX8-NEXT:    v_mov_b32_e32 v3, v18
2592; GFX8-NEXT:    v_mov_b32_e32 v4, v19
2593; GFX8-NEXT:    v_mov_b32_e32 v5, v20
2594; GFX8-NEXT:    v_mov_b32_e32 v6, v21
2595; GFX8-NEXT:    s_setpc_b64 s[30:31]
2596;
2597; GFX9-LABEL: v_mul_i256:
2598; GFX9:       ; %bb.0:
2599; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2600; GFX9-NEXT:    v_mul_lo_u32 v16, v2, v8
2601; GFX9-NEXT:    v_mul_lo_u32 v17, v1, v9
2602; GFX9-NEXT:    v_mul_lo_u32 v18, v0, v10
2603; GFX9-NEXT:    v_mul_hi_u32 v19, v1, v8
2604; GFX9-NEXT:    v_mul_lo_u32 v20, v1, v8
2605; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, v16, v17
2606; GFX9-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
2607; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, v16, v18
2608; GFX9-NEXT:    v_cndmask_b32_e64 v18, 0, 1, vcc
2609; GFX9-NEXT:    v_add_co_u32_e32 v19, vcc, v16, v19
2610; GFX9-NEXT:    v_mul_lo_u32 v21, v0, v9
2611; GFX9-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
2612; GFX9-NEXT:    v_add3_u32 v18, v17, v18, v16
2613; GFX9-NEXT:    v_mul_hi_u32 v16, v0, v8
2614; GFX9-NEXT:    v_add_co_u32_e32 v17, vcc, v20, v21
2615; GFX9-NEXT:    v_mul_hi_u32 v21, v0, v9
2616; GFX9-NEXT:    v_cndmask_b32_e64 v20, 0, 1, vcc
2617; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, v17, v16
2618; GFX9-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
2619; GFX9-NEXT:    v_add_u32_e32 v17, v20, v17
2620; GFX9-NEXT:    v_add_co_u32_e32 v19, vcc, v19, v21
2621; GFX9-NEXT:    v_mul_lo_u32 v21, v3, v8
2622; GFX9-NEXT:    v_mul_lo_u32 v22, v2, v9
2623; GFX9-NEXT:    v_cndmask_b32_e64 v20, 0, 1, vcc
2624; GFX9-NEXT:    v_add_co_u32_e32 v17, vcc, v19, v17
2625; GFX9-NEXT:    v_cndmask_b32_e64 v19, 0, 1, vcc
2626; GFX9-NEXT:    v_add3_u32 v18, v18, v20, v19
2627; GFX9-NEXT:    v_mul_lo_u32 v19, v1, v10
2628; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, v21, v22
2629; GFX9-NEXT:    v_mul_lo_u32 v22, v0, v11
2630; GFX9-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2631; GFX9-NEXT:    v_add_co_u32_e32 v19, vcc, v20, v19
2632; GFX9-NEXT:    v_cndmask_b32_e64 v20, 0, 1, vcc
2633; GFX9-NEXT:    v_add_co_u32_e32 v19, vcc, v19, v22
2634; GFX9-NEXT:    v_mul_hi_u32 v23, v2, v8
2635; GFX9-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2636; GFX9-NEXT:    v_add3_u32 v20, v21, v20, v22
2637; GFX9-NEXT:    v_mul_hi_u32 v21, v1, v9
2638; GFX9-NEXT:    v_add_co_u32_e32 v19, vcc, v19, v23
2639; GFX9-NEXT:    v_mul_hi_u32 v23, v0, v10
2640; GFX9-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2641; GFX9-NEXT:    v_add_co_u32_e32 v19, vcc, v19, v21
2642; GFX9-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2643; GFX9-NEXT:    v_add3_u32 v20, v20, v22, v21
2644; GFX9-NEXT:    v_add_co_u32_e32 v19, vcc, v19, v23
2645; GFX9-NEXT:    v_mul_lo_u32 v22, v4, v8
2646; GFX9-NEXT:    v_mul_lo_u32 v23, v3, v9
2647; GFX9-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2648; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, v19, v18
2649; GFX9-NEXT:    v_cndmask_b32_e64 v19, 0, 1, vcc
2650; GFX9-NEXT:    v_add3_u32 v19, v20, v21, v19
2651; GFX9-NEXT:    v_mul_lo_u32 v20, v2, v10
2652; GFX9-NEXT:    v_add_co_u32_e32 v21, vcc, v22, v23
2653; GFX9-NEXT:    v_mul_lo_u32 v23, v1, v11
2654; GFX9-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2655; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, v21, v20
2656; GFX9-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2657; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, v20, v23
2658; GFX9-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2659; GFX9-NEXT:    v_add3_u32 v21, v22, v21, v23
2660; GFX9-NEXT:    v_mul_lo_u32 v22, v0, v12
2661; GFX9-NEXT:    v_mul_hi_u32 v23, v3, v8
2662; GFX9-NEXT:    v_mul_lo_u32 v7, v7, v8
2663; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, v20, v22
2664; GFX9-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2665; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, v20, v23
2666; GFX9-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2667; GFX9-NEXT:    v_add3_u32 v21, v21, v22, v23
2668; GFX9-NEXT:    v_mul_hi_u32 v22, v2, v9
2669; GFX9-NEXT:    v_mul_hi_u32 v23, v1, v10
2670; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, v20, v22
2671; GFX9-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2672; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, v20, v23
2673; GFX9-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2674; GFX9-NEXT:    v_add3_u32 v21, v21, v22, v23
2675; GFX9-NEXT:    v_mul_hi_u32 v22, v0, v11
2676; GFX9-NEXT:    v_mul_lo_u32 v23, v3, v10
2677; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, v20, v22
2678; GFX9-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2679; GFX9-NEXT:    v_add_co_u32_e32 v19, vcc, v20, v19
2680; GFX9-NEXT:    v_cndmask_b32_e64 v20, 0, 1, vcc
2681; GFX9-NEXT:    v_add3_u32 v20, v21, v22, v20
2682; GFX9-NEXT:    v_mul_lo_u32 v21, v5, v8
2683; GFX9-NEXT:    v_mul_lo_u32 v22, v4, v9
2684; GFX9-NEXT:    v_add_co_u32_e32 v21, vcc, v21, v22
2685; GFX9-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2686; GFX9-NEXT:    v_add_co_u32_e32 v21, vcc, v21, v23
2687; GFX9-NEXT:    v_mul_lo_u32 v23, v2, v11
2688; GFX9-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2689; GFX9-NEXT:    v_add_co_u32_e32 v21, vcc, v21, v23
2690; GFX9-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2691; GFX9-NEXT:    v_add3_u32 v22, v22, v24, v23
2692; GFX9-NEXT:    v_mul_lo_u32 v23, v1, v12
2693; GFX9-NEXT:    v_add_co_u32_e32 v21, vcc, v21, v23
2694; GFX9-NEXT:    v_mul_lo_u32 v23, v0, v13
2695; GFX9-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2696; GFX9-NEXT:    v_add_co_u32_e32 v21, vcc, v21, v23
2697; GFX9-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2698; GFX9-NEXT:    v_add3_u32 v22, v22, v24, v23
2699; GFX9-NEXT:    v_mul_hi_u32 v23, v4, v8
2700; GFX9-NEXT:    v_add_co_u32_e32 v21, vcc, v21, v23
2701; GFX9-NEXT:    v_mul_hi_u32 v23, v3, v9
2702; GFX9-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2703; GFX9-NEXT:    v_add_co_u32_e32 v21, vcc, v21, v23
2704; GFX9-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2705; GFX9-NEXT:    v_add3_u32 v22, v22, v24, v23
2706; GFX9-NEXT:    v_mul_hi_u32 v23, v2, v10
2707; GFX9-NEXT:    v_add_co_u32_e32 v21, vcc, v21, v23
2708; GFX9-NEXT:    v_mul_hi_u32 v23, v1, v11
2709; GFX9-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2710; GFX9-NEXT:    v_add_co_u32_e32 v21, vcc, v21, v23
2711; GFX9-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2712; GFX9-NEXT:    v_add3_u32 v22, v22, v24, v23
2713; GFX9-NEXT:    v_mul_hi_u32 v23, v0, v12
2714; GFX9-NEXT:    v_add_co_u32_e32 v21, vcc, v21, v23
2715; GFX9-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2716; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, v21, v20
2717; GFX9-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2718; GFX9-NEXT:    v_add3_u32 v21, v22, v23, v21
2719; GFX9-NEXT:    v_mul_lo_u32 v22, v6, v8
2720; GFX9-NEXT:    v_mul_lo_u32 v23, v5, v9
2721; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, v22, v23
2722; GFX9-NEXT:    v_mul_lo_u32 v23, v4, v10
2723; GFX9-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2724; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, v22, v23
2725; GFX9-NEXT:    v_mul_lo_u32 v23, v3, v11
2726; GFX9-NEXT:    v_cndmask_b32_e64 v25, 0, 1, vcc
2727; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, v22, v23
2728; GFX9-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2729; GFX9-NEXT:    v_add3_u32 v23, v24, v25, v23
2730; GFX9-NEXT:    v_mul_lo_u32 v24, v2, v12
2731; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, v22, v24
2732; GFX9-NEXT:    v_mul_lo_u32 v24, v1, v13
2733; GFX9-NEXT:    v_cndmask_b32_e64 v25, 0, 1, vcc
2734; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, v22, v24
2735; GFX9-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2736; GFX9-NEXT:    v_add3_u32 v23, v23, v25, v24
2737; GFX9-NEXT:    v_mul_lo_u32 v24, v0, v14
2738; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, v22, v24
2739; GFX9-NEXT:    v_mul_hi_u32 v24, v5, v8
2740; GFX9-NEXT:    v_cndmask_b32_e64 v25, 0, 1, vcc
2741; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, v22, v24
2742; GFX9-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2743; GFX9-NEXT:    v_add3_u32 v23, v23, v25, v24
2744; GFX9-NEXT:    v_mul_hi_u32 v24, v4, v9
2745; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, v22, v24
2746; GFX9-NEXT:    v_mul_hi_u32 v24, v3, v10
2747; GFX9-NEXT:    v_cndmask_b32_e64 v25, 0, 1, vcc
2748; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, v22, v24
2749; GFX9-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2750; GFX9-NEXT:    v_add3_u32 v23, v23, v25, v24
2751; GFX9-NEXT:    v_mul_hi_u32 v24, v2, v11
2752; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, v22, v24
2753; GFX9-NEXT:    v_mul_hi_u32 v24, v1, v12
2754; GFX9-NEXT:    v_cndmask_b32_e64 v25, 0, 1, vcc
2755; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, v22, v24
2756; GFX9-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2757; GFX9-NEXT:    v_add3_u32 v23, v23, v25, v24
2758; GFX9-NEXT:    v_mul_hi_u32 v24, v0, v13
2759; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, v22, v24
2760; GFX9-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2761; GFX9-NEXT:    v_add_co_u32_e32 v21, vcc, v22, v21
2762; GFX9-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2763; GFX9-NEXT:    v_add3_u32 v22, v23, v24, v22
2764; GFX9-NEXT:    v_mul_lo_u32 v23, v6, v9
2765; GFX9-NEXT:    v_mul_lo_u32 v24, v4, v11
2766; GFX9-NEXT:    v_mul_hi_u32 v4, v4, v10
2767; GFX9-NEXT:    v_mul_hi_u32 v6, v6, v8
2768; GFX9-NEXT:    v_add_u32_e32 v7, v7, v23
2769; GFX9-NEXT:    v_mul_lo_u32 v23, v5, v10
2770; GFX9-NEXT:    v_mul_hi_u32 v5, v5, v9
2771; GFX9-NEXT:    v_mul_hi_u32 v9, v3, v11
2772; GFX9-NEXT:    v_mul_lo_u32 v3, v3, v12
2773; GFX9-NEXT:    v_mul_hi_u32 v10, v2, v12
2774; GFX9-NEXT:    v_mul_lo_u32 v2, v2, v13
2775; GFX9-NEXT:    v_mul_hi_u32 v11, v1, v13
2776; GFX9-NEXT:    v_mul_lo_u32 v12, v1, v14
2777; GFX9-NEXT:    v_mul_lo_u32 v13, v0, v15
2778; GFX9-NEXT:    v_add3_u32 v7, v7, v23, v24
2779; GFX9-NEXT:    v_add3_u32 v2, v7, v3, v2
2780; GFX9-NEXT:    v_mul_lo_u32 v1, v0, v8
2781; GFX9-NEXT:    v_mul_hi_u32 v0, v0, v14
2782; GFX9-NEXT:    v_add3_u32 v2, v2, v12, v13
2783; GFX9-NEXT:    v_add3_u32 v2, v2, v6, v5
2784; GFX9-NEXT:    v_add3_u32 v2, v2, v4, v9
2785; GFX9-NEXT:    v_add3_u32 v2, v2, v10, v11
2786; GFX9-NEXT:    v_add3_u32 v7, v2, v0, v22
2787; GFX9-NEXT:    v_mov_b32_e32 v0, v1
2788; GFX9-NEXT:    v_mov_b32_e32 v1, v16
2789; GFX9-NEXT:    v_mov_b32_e32 v2, v17
2790; GFX9-NEXT:    v_mov_b32_e32 v3, v18
2791; GFX9-NEXT:    v_mov_b32_e32 v4, v19
2792; GFX9-NEXT:    v_mov_b32_e32 v5, v20
2793; GFX9-NEXT:    v_mov_b32_e32 v6, v21
2794; GFX9-NEXT:    s_setpc_b64 s[30:31]
2795;
2796; GFX10-LABEL: v_mul_i256:
2797; GFX10:       ; %bb.0:
2798; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2799; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
2800; GFX10-NEXT:    v_mul_lo_u32 v16, v1, v8
2801; GFX10-NEXT:    v_mul_lo_u32 v17, v0, v9
2802; GFX10-NEXT:    v_mul_hi_u32 v18, v0, v8
2803; GFX10-NEXT:    v_mul_lo_u32 v19, v2, v8
2804; GFX10-NEXT:    v_mul_lo_u32 v20, v1, v9
2805; GFX10-NEXT:    v_mul_hi_u32 v21, v1, v8
2806; GFX10-NEXT:    v_mul_lo_u32 v22, v3, v8
2807; GFX10-NEXT:    v_mul_lo_u32 v25, v1, v10
2808; GFX10-NEXT:    v_mul_hi_u32 v23, v0, v9
2809; GFX10-NEXT:    v_add_co_u32 v16, s4, v16, v17
2810; GFX10-NEXT:    v_cndmask_b32_e64 v17, 0, 1, s4
2811; GFX10-NEXT:    v_mul_hi_u32 v27, v0, v10
2812; GFX10-NEXT:    v_mul_hi_u32 v29, v3, v9
2813; GFX10-NEXT:    v_add_co_u32 v16, s4, v16, v18
2814; GFX10-NEXT:    v_cndmask_b32_e64 v18, 0, 1, s4
2815; GFX10-NEXT:    v_add_co_u32 v19, s4, v19, v20
2816; GFX10-NEXT:    v_mul_lo_u32 v20, v2, v9
2817; GFX10-NEXT:    v_cndmask_b32_e64 v24, 0, 1, s4
2818; GFX10-NEXT:    v_add_nc_u32_e32 v17, v17, v18
2819; GFX10-NEXT:    v_mul_lo_u32 v18, v0, v10
2820; GFX10-NEXT:    v_mul_hi_u32 v31, v4, v9
2821; GFX10-NEXT:    v_mul_lo_u32 v7, v7, v8
2822; GFX10-NEXT:    v_mul_lo_u32 v15, v0, v15
2823; GFX10-NEXT:    v_add_co_u32 v18, s4, v19, v18
2824; GFX10-NEXT:    v_cndmask_b32_e64 v19, 0, 1, s4
2825; GFX10-NEXT:    v_add_co_u32 v20, s4, v22, v20
2826; GFX10-NEXT:    v_add_co_u32 v18, s5, v18, v21
2827; GFX10-NEXT:    v_cndmask_b32_e64 v21, 0, 1, s5
2828; GFX10-NEXT:    v_mul_lo_u32 v22, v0, v11
2829; GFX10-NEXT:    v_cndmask_b32_e64 v26, 0, 1, s4
2830; GFX10-NEXT:    v_add_co_u32 v20, s4, v20, v25
2831; GFX10-NEXT:    v_add3_u32 v19, v24, v19, v21
2832; GFX10-NEXT:    v_mul_hi_u32 v21, v2, v8
2833; GFX10-NEXT:    v_cndmask_b32_e64 v24, 0, 1, s4
2834; GFX10-NEXT:    v_add_co_u32 v18, s5, v18, v23
2835; GFX10-NEXT:    v_add_co_u32 v20, s4, v20, v22
2836; GFX10-NEXT:    v_mul_hi_u32 v23, v1, v9
2837; GFX10-NEXT:    v_cndmask_b32_e64 v25, 0, 1, s4
2838; GFX10-NEXT:    v_cndmask_b32_e64 v22, 0, 1, s5
2839; GFX10-NEXT:    v_add_co_u32 v20, s4, v20, v21
2840; GFX10-NEXT:    v_add_co_u32 v17, s5, v18, v17
2841; GFX10-NEXT:    v_add3_u32 v21, v26, v24, v25
2842; GFX10-NEXT:    v_cndmask_b32_e64 v24, 0, 1, s4
2843; GFX10-NEXT:    v_add_co_u32 v20, s4, v20, v23
2844; GFX10-NEXT:    v_cndmask_b32_e64 v18, 0, 1, s5
2845; GFX10-NEXT:    v_cndmask_b32_e64 v23, 0, 1, s4
2846; GFX10-NEXT:    v_mul_lo_u32 v25, v4, v8
2847; GFX10-NEXT:    v_mul_lo_u32 v26, v3, v9
2848; GFX10-NEXT:    v_add_co_u32 v20, s5, v20, v27
2849; GFX10-NEXT:    v_add3_u32 v18, v19, v22, v18
2850; GFX10-NEXT:    v_add3_u32 v19, v21, v24, v23
2851; GFX10-NEXT:    v_mul_lo_u32 v21, v2, v10
2852; GFX10-NEXT:    v_mul_lo_u32 v24, v1, v11
2853; GFX10-NEXT:    v_cndmask_b32_e64 v23, 0, 1, s5
2854; GFX10-NEXT:    v_add_co_u32 v22, s4, v25, v26
2855; GFX10-NEXT:    v_cndmask_b32_e64 v25, 0, 1, s4
2856; GFX10-NEXT:    v_mul_hi_u32 v26, v3, v8
2857; GFX10-NEXT:    v_add_co_u32 v21, s4, v22, v21
2858; GFX10-NEXT:    v_mul_lo_u32 v22, v0, v12
2859; GFX10-NEXT:    v_cndmask_b32_e64 v27, 0, 1, s4
2860; GFX10-NEXT:    v_add_co_u32 v21, s4, v21, v24
2861; GFX10-NEXT:    v_cndmask_b32_e64 v24, 0, 1, s4
2862; GFX10-NEXT:    v_add_co_u32 v18, s4, v20, v18
2863; GFX10-NEXT:    v_cndmask_b32_e64 v20, 0, 1, s4
2864; GFX10-NEXT:    v_add_co_u32 v21, s4, v21, v22
2865; GFX10-NEXT:    v_mul_hi_u32 v22, v2, v9
2866; GFX10-NEXT:    v_add3_u32 v24, v25, v27, v24
2867; GFX10-NEXT:    v_cndmask_b32_e64 v25, 0, 1, s4
2868; GFX10-NEXT:    v_add_co_u32 v21, s4, v21, v26
2869; GFX10-NEXT:    v_cndmask_b32_e64 v26, 0, 1, s4
2870; GFX10-NEXT:    v_add3_u32 v19, v19, v23, v20
2871; GFX10-NEXT:    v_mul_hi_u32 v20, v1, v10
2872; GFX10-NEXT:    v_add_co_u32 v21, s4, v21, v22
2873; GFX10-NEXT:    v_add3_u32 v23, v24, v25, v26
2874; GFX10-NEXT:    v_mul_lo_u32 v22, v5, v8
2875; GFX10-NEXT:    v_mul_lo_u32 v24, v4, v9
2876; GFX10-NEXT:    v_cndmask_b32_e64 v25, 0, 1, s4
2877; GFX10-NEXT:    v_mul_lo_u32 v26, v3, v10
2878; GFX10-NEXT:    v_add_co_u32 v20, s4, v21, v20
2879; GFX10-NEXT:    v_cndmask_b32_e64 v21, 0, 1, s4
2880; GFX10-NEXT:    v_mul_hi_u32 v27, v0, v11
2881; GFX10-NEXT:    v_add_co_u32 v22, s4, v22, v24
2882; GFX10-NEXT:    v_add3_u32 v21, v23, v25, v21
2883; GFX10-NEXT:    v_mul_lo_u32 v23, v2, v11
2884; GFX10-NEXT:    v_cndmask_b32_e64 v24, 0, 1, s4
2885; GFX10-NEXT:    v_add_co_u32 v22, s4, v22, v26
2886; GFX10-NEXT:    v_mul_lo_u32 v26, v1, v12
2887; GFX10-NEXT:    v_add_co_u32 v20, s5, v20, v27
2888; GFX10-NEXT:    v_cndmask_b32_e64 v27, 0, 1, s4
2889; GFX10-NEXT:    v_add_co_u32 v22, s4, v22, v23
2890; GFX10-NEXT:    v_mul_lo_u32 v23, v0, v13
2891; GFX10-NEXT:    v_cndmask_b32_e64 v28, 0, 1, s4
2892; GFX10-NEXT:    v_cndmask_b32_e64 v25, 0, 1, s5
2893; GFX10-NEXT:    v_add_co_u32 v22, s4, v22, v26
2894; GFX10-NEXT:    v_mul_hi_u32 v26, v4, v8
2895; GFX10-NEXT:    v_add_co_u32 v19, s5, v20, v19
2896; GFX10-NEXT:    v_cndmask_b32_e64 v20, 0, 1, s5
2897; GFX10-NEXT:    v_cndmask_b32_e64 v30, 0, 1, s4
2898; GFX10-NEXT:    v_add_co_u32 v22, s4, v22, v23
2899; GFX10-NEXT:    v_add3_u32 v23, v24, v27, v28
2900; GFX10-NEXT:    v_cndmask_b32_e64 v24, 0, 1, s4
2901; GFX10-NEXT:    v_add3_u32 v20, v21, v25, v20
2902; GFX10-NEXT:    v_add_co_u32 v21, s4, v22, v26
2903; GFX10-NEXT:    v_mul_hi_u32 v22, v2, v10
2904; GFX10-NEXT:    v_add3_u32 v23, v23, v30, v24
2905; GFX10-NEXT:    v_cndmask_b32_e64 v24, 0, 1, s4
2906; GFX10-NEXT:    v_add_co_u32 v21, s4, v21, v29
2907; GFX10-NEXT:    v_cndmask_b32_e64 v25, 0, 1, s4
2908; GFX10-NEXT:    v_mul_hi_u32 v26, v1, v11
2909; GFX10-NEXT:    v_mul_lo_u32 v27, v6, v8
2910; GFX10-NEXT:    v_mul_lo_u32 v28, v5, v9
2911; GFX10-NEXT:    v_add_co_u32 v21, s4, v21, v22
2912; GFX10-NEXT:    v_add3_u32 v23, v23, v24, v25
2913; GFX10-NEXT:    v_mul_lo_u32 v24, v4, v10
2914; GFX10-NEXT:    v_cndmask_b32_e64 v22, 0, 1, s4
2915; GFX10-NEXT:    v_add_co_u32 v21, s5, v21, v26
2916; GFX10-NEXT:    v_add_co_u32 v25, s4, v27, v28
2917; GFX10-NEXT:    v_cndmask_b32_e64 v26, 0, 1, s5
2918; GFX10-NEXT:    v_mul_lo_u32 v27, v3, v11
2919; GFX10-NEXT:    v_cndmask_b32_e64 v28, 0, 1, s4
2920; GFX10-NEXT:    v_add_co_u32 v24, s4, v25, v24
2921; GFX10-NEXT:    v_mul_hi_u32 v29, v0, v12
2922; GFX10-NEXT:    v_add3_u32 v22, v23, v22, v26
2923; GFX10-NEXT:    v_mul_lo_u32 v23, v2, v12
2924; GFX10-NEXT:    v_cndmask_b32_e64 v25, 0, 1, s4
2925; GFX10-NEXT:    v_add_co_u32 v24, s4, v24, v27
2926; GFX10-NEXT:    v_mul_lo_u32 v27, v1, v13
2927; GFX10-NEXT:    v_add_co_u32 v21, s5, v21, v29
2928; GFX10-NEXT:    v_cndmask_b32_e64 v29, 0, 1, s4
2929; GFX10-NEXT:    v_add_co_u32 v23, s4, v24, v23
2930; GFX10-NEXT:    v_mul_lo_u32 v24, v0, v14
2931; GFX10-NEXT:    v_cndmask_b32_e64 v30, 0, 1, s4
2932; GFX10-NEXT:    v_cndmask_b32_e64 v26, 0, 1, s5
2933; GFX10-NEXT:    v_add_co_u32 v23, s4, v23, v27
2934; GFX10-NEXT:    v_mul_hi_u32 v27, v5, v8
2935; GFX10-NEXT:    v_cndmask_b32_e64 v32, 0, 1, s4
2936; GFX10-NEXT:    v_add_co_u32 v20, s5, v21, v20
2937; GFX10-NEXT:    v_add_co_u32 v23, s4, v23, v24
2938; GFX10-NEXT:    v_add3_u32 v24, v28, v25, v29
2939; GFX10-NEXT:    v_cndmask_b32_e64 v28, 0, 1, s4
2940; GFX10-NEXT:    v_mul_hi_u32 v25, v3, v10
2941; GFX10-NEXT:    v_add_co_u32 v23, s4, v23, v27
2942; GFX10-NEXT:    v_cndmask_b32_e64 v21, 0, 1, s5
2943; GFX10-NEXT:    v_add3_u32 v24, v24, v30, v32
2944; GFX10-NEXT:    v_cndmask_b32_e64 v27, 0, 1, s4
2945; GFX10-NEXT:    v_add_co_u32 v23, s4, v23, v31
2946; GFX10-NEXT:    v_add3_u32 v21, v22, v26, v21
2947; GFX10-NEXT:    v_mul_hi_u32 v26, v2, v11
2948; GFX10-NEXT:    v_add3_u32 v22, v24, v28, v27
2949; GFX10-NEXT:    v_cndmask_b32_e64 v24, 0, 1, s4
2950; GFX10-NEXT:    v_add_co_u32 v23, s4, v23, v25
2951; GFX10-NEXT:    v_cndmask_b32_e64 v25, 0, 1, s4
2952; GFX10-NEXT:    v_mul_lo_u32 v28, v6, v9
2953; GFX10-NEXT:    v_mul_lo_u32 v29, v3, v12
2954; GFX10-NEXT:    v_mul_hi_u32 v27, v1, v12
2955; GFX10-NEXT:    v_add_co_u32 v23, s4, v23, v26
2956; GFX10-NEXT:    v_add3_u32 v22, v22, v24, v25
2957; GFX10-NEXT:    v_mul_lo_u32 v24, v5, v10
2958; GFX10-NEXT:    v_mul_lo_u32 v25, v4, v11
2959; GFX10-NEXT:    v_add_nc_u32_e32 v7, v7, v28
2960; GFX10-NEXT:    v_mul_lo_u32 v28, v2, v13
2961; GFX10-NEXT:    v_mul_hi_u32 v6, v6, v8
2962; GFX10-NEXT:    v_mul_hi_u32 v5, v5, v9
2963; GFX10-NEXT:    v_cndmask_b32_e64 v26, 0, 1, s4
2964; GFX10-NEXT:    v_add_co_u32 v23, s4, v23, v27
2965; GFX10-NEXT:    v_add3_u32 v7, v7, v24, v25
2966; GFX10-NEXT:    v_mul_lo_u32 v24, v1, v14
2967; GFX10-NEXT:    v_mul_hi_u32 v25, v0, v13
2968; GFX10-NEXT:    v_mul_hi_u32 v4, v4, v10
2969; GFX10-NEXT:    v_mul_hi_u32 v3, v3, v11
2970; GFX10-NEXT:    v_add3_u32 v7, v7, v29, v28
2971; GFX10-NEXT:    v_cndmask_b32_e64 v27, 0, 1, s4
2972; GFX10-NEXT:    v_mul_hi_u32 v2, v2, v12
2973; GFX10-NEXT:    v_mul_hi_u32 v1, v1, v13
2974; GFX10-NEXT:    v_add3_u32 v7, v7, v24, v15
2975; GFX10-NEXT:    v_add_co_u32 v9, s4, v23, v25
2976; GFX10-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s4
2977; GFX10-NEXT:    v_add3_u32 v22, v22, v26, v27
2978; GFX10-NEXT:    v_add3_u32 v5, v7, v6, v5
2979; GFX10-NEXT:    v_add_co_u32 v6, s4, v9, v21
2980; GFX10-NEXT:    v_cndmask_b32_e64 v7, 0, 1, s4
2981; GFX10-NEXT:    v_add3_u32 v3, v5, v4, v3
2982; GFX10-NEXT:    v_mul_hi_u32 v4, v0, v14
2983; GFX10-NEXT:    v_mul_lo_u32 v0, v0, v8
2984; GFX10-NEXT:    v_add3_u32 v5, v22, v10, v7
2985; GFX10-NEXT:    v_add3_u32 v1, v3, v2, v1
2986; GFX10-NEXT:    v_mov_b32_e32 v2, v17
2987; GFX10-NEXT:    v_mov_b32_e32 v3, v18
2988; GFX10-NEXT:    v_add3_u32 v7, v1, v4, v5
2989; GFX10-NEXT:    v_mov_b32_e32 v1, v16
2990; GFX10-NEXT:    v_mov_b32_e32 v4, v19
2991; GFX10-NEXT:    v_mov_b32_e32 v5, v20
2992; GFX10-NEXT:    s_setpc_b64 s[30:31]
2993  %result = mul i256 %num, %den
2994  ret i256 %result
2995}
2996