1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -global-isel -march=amdgcn -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX7 %s
3; RUN: llc -global-isel -march=amdgcn -mcpu=gfx801 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX8 %s
4; RUN: llc -global-isel -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9 %s
5; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10 %s
6
7define amdgpu_ps i16 @s_mul_i16(i16 inreg %num, i16 inreg %den) {
8; GFX7-LABEL: s_mul_i16:
9; GFX7:       ; %bb.0:
10; GFX7-NEXT:    s_mul_i32 s0, s0, s1
11; GFX7-NEXT:    ; return to shader part epilog
12;
13; GFX8-LABEL: s_mul_i16:
14; GFX8:       ; %bb.0:
15; GFX8-NEXT:    s_mov_b32 s2, 0xffff
16; GFX8-NEXT:    s_and_b32 s0, s0, s2
17; GFX8-NEXT:    s_and_b32 s1, s1, s2
18; GFX8-NEXT:    s_mul_i32 s0, s0, s1
19; GFX8-NEXT:    ; return to shader part epilog
20;
21; GFX9-LABEL: s_mul_i16:
22; GFX9:       ; %bb.0:
23; GFX9-NEXT:    s_mov_b32 s2, 0xffff
24; GFX9-NEXT:    s_and_b32 s0, s0, s2
25; GFX9-NEXT:    s_and_b32 s1, s1, s2
26; GFX9-NEXT:    s_mul_i32 s0, s0, s1
27; GFX9-NEXT:    ; return to shader part epilog
28;
29; GFX10-LABEL: s_mul_i16:
30; GFX10:       ; %bb.0:
31; GFX10-NEXT:    s_mov_b32 s2, 0xffff
32; GFX10-NEXT:    s_and_b32 s0, s0, s2
33; GFX10-NEXT:    s_and_b32 s1, s1, s2
34; GFX10-NEXT:    s_mul_i32 s0, s0, s1
35; GFX10-NEXT:    ; return to shader part epilog
36  %result = mul i16 %num, %den
37  ret i16 %result
38}
39
40define i16 @v_mul_i16(i16 %num, i16 %den) {
41; GFX7-LABEL: v_mul_i16:
42; GFX7:       ; %bb.0:
43; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
44; GFX7-NEXT:    s_mov_b32 s4, 0xffff
45; GFX7-NEXT:    v_and_b32_e32 v0, s4, v0
46; GFX7-NEXT:    v_and_b32_e32 v1, s4, v1
47; GFX7-NEXT:    v_mul_u32_u24_e32 v0, v0, v1
48; GFX7-NEXT:    s_setpc_b64 s[30:31]
49;
50; GFX8-LABEL: v_mul_i16:
51; GFX8:       ; %bb.0:
52; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
53; GFX8-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
54; GFX8-NEXT:    s_setpc_b64 s[30:31]
55;
56; GFX9-LABEL: v_mul_i16:
57; GFX9:       ; %bb.0:
58; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
59; GFX9-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
60; GFX9-NEXT:    s_setpc_b64 s[30:31]
61;
62; GFX10-LABEL: v_mul_i16:
63; GFX10:       ; %bb.0:
64; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
65; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
66; GFX10-NEXT:    v_mul_lo_u16 v0, v0, v1
67; GFX10-NEXT:    s_setpc_b64 s[30:31]
68  %result = mul i16 %num, %den
69  ret i16 %result
70}
71
72define amdgpu_ps zeroext i16 @s_mul_i16_zeroext(i16 inreg zeroext %num, i16 inreg zeroext %den) {
73; GFX7-LABEL: s_mul_i16_zeroext:
74; GFX7:       ; %bb.0:
75; GFX7-NEXT:    s_mul_i32 s0, s0, s1
76; GFX7-NEXT:    s_and_b32 s0, s0, 0xffff
77; GFX7-NEXT:    ; return to shader part epilog
78;
79; GFX8-LABEL: s_mul_i16_zeroext:
80; GFX8:       ; %bb.0:
81; GFX8-NEXT:    s_mov_b32 s2, 0xffff
82; GFX8-NEXT:    s_and_b32 s0, s0, s2
83; GFX8-NEXT:    s_and_b32 s1, s1, s2
84; GFX8-NEXT:    s_mul_i32 s0, s0, s1
85; GFX8-NEXT:    s_and_b32 s0, s0, s2
86; GFX8-NEXT:    ; return to shader part epilog
87;
88; GFX9-LABEL: s_mul_i16_zeroext:
89; GFX9:       ; %bb.0:
90; GFX9-NEXT:    s_mov_b32 s2, 0xffff
91; GFX9-NEXT:    s_and_b32 s0, s0, s2
92; GFX9-NEXT:    s_and_b32 s1, s1, s2
93; GFX9-NEXT:    s_mul_i32 s0, s0, s1
94; GFX9-NEXT:    s_and_b32 s0, s0, s2
95; GFX9-NEXT:    ; return to shader part epilog
96;
97; GFX10-LABEL: s_mul_i16_zeroext:
98; GFX10:       ; %bb.0:
99; GFX10-NEXT:    s_mov_b32 s2, 0xffff
100; GFX10-NEXT:    s_and_b32 s0, s0, s2
101; GFX10-NEXT:    s_and_b32 s1, s1, s2
102; GFX10-NEXT:    s_mul_i32 s0, s0, s1
103; GFX10-NEXT:    s_and_b32 s0, s0, s2
104; GFX10-NEXT:    ; return to shader part epilog
105  %result = mul i16 %num, %den
106  ret i16 %result
107}
108
109define zeroext i16 @v_mul_i16_zeroext(i16 zeroext %num, i16 zeroext %den) {
110; GFX7-LABEL: v_mul_i16_zeroext:
111; GFX7:       ; %bb.0:
112; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
113; GFX7-NEXT:    v_mul_u32_u24_e32 v0, v0, v1
114; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff, v0
115; GFX7-NEXT:    s_setpc_b64 s[30:31]
116;
117; GFX8-LABEL: v_mul_i16_zeroext:
118; GFX8:       ; %bb.0:
119; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
120; GFX8-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
121; GFX8-NEXT:    s_setpc_b64 s[30:31]
122;
123; GFX9-LABEL: v_mul_i16_zeroext:
124; GFX9:       ; %bb.0:
125; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
126; GFX9-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
127; GFX9-NEXT:    s_setpc_b64 s[30:31]
128;
129; GFX10-LABEL: v_mul_i16_zeroext:
130; GFX10:       ; %bb.0:
131; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
132; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
133; GFX10-NEXT:    v_mul_lo_u16 v0, v0, v1
134; GFX10-NEXT:    v_bfe_u32 v0, v0, 0, 16
135; GFX10-NEXT:    s_setpc_b64 s[30:31]
136  %result = mul i16 %num, %den
137  ret i16 %result
138}
139
140define amdgpu_ps signext i16 @s_mul_i16_signext(i16 inreg signext %num, i16 inreg signext %den) {
141; GFX7-LABEL: s_mul_i16_signext:
142; GFX7:       ; %bb.0:
143; GFX7-NEXT:    s_mul_i32 s0, s0, s1
144; GFX7-NEXT:    s_sext_i32_i16 s0, s0
145; GFX7-NEXT:    ; return to shader part epilog
146;
147; GFX8-LABEL: s_mul_i16_signext:
148; GFX8:       ; %bb.0:
149; GFX8-NEXT:    s_mov_b32 s2, 0xffff
150; GFX8-NEXT:    s_and_b32 s0, s0, s2
151; GFX8-NEXT:    s_and_b32 s1, s1, s2
152; GFX8-NEXT:    s_mul_i32 s0, s0, s1
153; GFX8-NEXT:    s_sext_i32_i16 s0, s0
154; GFX8-NEXT:    ; return to shader part epilog
155;
156; GFX9-LABEL: s_mul_i16_signext:
157; GFX9:       ; %bb.0:
158; GFX9-NEXT:    s_mov_b32 s2, 0xffff
159; GFX9-NEXT:    s_and_b32 s0, s0, s2
160; GFX9-NEXT:    s_and_b32 s1, s1, s2
161; GFX9-NEXT:    s_mul_i32 s0, s0, s1
162; GFX9-NEXT:    s_sext_i32_i16 s0, s0
163; GFX9-NEXT:    ; return to shader part epilog
164;
165; GFX10-LABEL: s_mul_i16_signext:
166; GFX10:       ; %bb.0:
167; GFX10-NEXT:    s_mov_b32 s2, 0xffff
168; GFX10-NEXT:    s_and_b32 s0, s0, s2
169; GFX10-NEXT:    s_and_b32 s1, s1, s2
170; GFX10-NEXT:    s_mul_i32 s0, s0, s1
171; GFX10-NEXT:    s_sext_i32_i16 s0, s0
172; GFX10-NEXT:    ; return to shader part epilog
173  %result = mul i16 %num, %den
174  ret i16 %result
175}
176
177define signext i16 @v_mul_i16_signext(i16 signext %num, i16 signext %den) {
178; GFX7-LABEL: v_mul_i16_signext:
179; GFX7:       ; %bb.0:
180; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
181; GFX7-NEXT:    s_mov_b32 s4, 0xffff
182; GFX7-NEXT:    v_and_b32_e32 v0, s4, v0
183; GFX7-NEXT:    v_and_b32_e32 v1, s4, v1
184; GFX7-NEXT:    v_mul_u32_u24_e32 v0, v0, v1
185; GFX7-NEXT:    v_bfe_i32 v0, v0, 0, 16
186; GFX7-NEXT:    s_setpc_b64 s[30:31]
187;
188; GFX8-LABEL: v_mul_i16_signext:
189; GFX8:       ; %bb.0:
190; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
191; GFX8-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
192; GFX8-NEXT:    v_bfe_i32 v0, v0, 0, 16
193; GFX8-NEXT:    s_setpc_b64 s[30:31]
194;
195; GFX9-LABEL: v_mul_i16_signext:
196; GFX9:       ; %bb.0:
197; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
198; GFX9-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
199; GFX9-NEXT:    v_bfe_i32 v0, v0, 0, 16
200; GFX9-NEXT:    s_setpc_b64 s[30:31]
201;
202; GFX10-LABEL: v_mul_i16_signext:
203; GFX10:       ; %bb.0:
204; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
205; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
206; GFX10-NEXT:    v_mul_lo_u16 v0, v0, v1
207; GFX10-NEXT:    v_bfe_i32 v0, v0, 0, 16
208; GFX10-NEXT:    s_setpc_b64 s[30:31]
209  %result = mul i16 %num, %den
210  ret i16 %result
211}
212
213define amdgpu_ps i32 @s_mul_i32(i32 inreg %num, i32 inreg %den) {
214; GCN-LABEL: s_mul_i32:
215; GCN:       ; %bb.0:
216; GCN-NEXT:    s_mul_i32 s0, s0, s1
217; GCN-NEXT:    ; return to shader part epilog
218;
219; GFX10-LABEL: s_mul_i32:
220; GFX10:       ; %bb.0:
221; GFX10-NEXT:    s_mul_i32 s0, s0, s1
222; GFX10-NEXT:    ; return to shader part epilog
223  %result = mul i32 %num, %den
224  ret i32 %result
225}
226
227define i32 @v_mul_i32(i32 %num, i32 %den) {
228; GCN-LABEL: v_mul_i32:
229; GCN:       ; %bb.0:
230; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
231; GCN-NEXT:    v_mul_lo_u32 v0, v0, v1
232; GCN-NEXT:    s_setpc_b64 s[30:31]
233;
234; GFX10-LABEL: v_mul_i32:
235; GFX10:       ; %bb.0:
236; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
237; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
238; GFX10-NEXT:    v_mul_lo_u32 v0, v0, v1
239; GFX10-NEXT:    s_setpc_b64 s[30:31]
240  %result = mul i32 %num, %den
241  ret i32 %result
242}
243
244define amdgpu_ps <2 x i32> @s_mul_v2i32(<2 x i32> inreg %num, <2 x i32> inreg %den) {
245; GCN-LABEL: s_mul_v2i32:
246; GCN:       ; %bb.0:
247; GCN-NEXT:    s_mul_i32 s0, s0, s2
248; GCN-NEXT:    s_mul_i32 s1, s1, s3
249; GCN-NEXT:    ; return to shader part epilog
250;
251; GFX10-LABEL: s_mul_v2i32:
252; GFX10:       ; %bb.0:
253; GFX10-NEXT:    s_mul_i32 s0, s0, s2
254; GFX10-NEXT:    s_mul_i32 s1, s1, s3
255; GFX10-NEXT:    ; return to shader part epilog
256  %result = mul <2 x i32> %num, %den
257  ret <2 x i32> %result
258}
259
260define <2 x i32> @v_mul_v2i32(<2 x i32> %num, <2 x i32> %den) {
261; GCN-LABEL: v_mul_v2i32:
262; GCN:       ; %bb.0:
263; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
264; GCN-NEXT:    v_mul_lo_u32 v0, v0, v2
265; GCN-NEXT:    v_mul_lo_u32 v1, v1, v3
266; GCN-NEXT:    s_setpc_b64 s[30:31]
267;
268; GFX10-LABEL: v_mul_v2i32:
269; GFX10:       ; %bb.0:
270; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
271; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
272; GFX10-NEXT:    v_mul_lo_u32 v0, v0, v2
273; GFX10-NEXT:    v_mul_lo_u32 v1, v1, v3
274; GFX10-NEXT:    s_setpc_b64 s[30:31]
275  %result = mul <2 x i32> %num, %den
276  ret <2 x i32> %result
277}
278
279define amdgpu_ps i64 @s_mul_i64(i64 inreg %num, i64 inreg %den) {
280; GFX7-LABEL: s_mul_i64:
281; GFX7:       ; %bb.0:
282; GFX7-NEXT:    v_mov_b32_e32 v0, s2
283; GFX7-NEXT:    v_mul_hi_u32 v0, s0, v0
284; GFX7-NEXT:    s_mul_i32 s4, s0, s2
285; GFX7-NEXT:    s_mul_i32 s1, s1, s2
286; GFX7-NEXT:    s_mul_i32 s0, s0, s3
287; GFX7-NEXT:    s_add_i32 s1, s1, s0
288; GFX7-NEXT:    v_add_i32_e32 v0, vcc, s1, v0
289; GFX7-NEXT:    v_readfirstlane_b32 s1, v0
290; GFX7-NEXT:    s_mov_b32 s0, s4
291; GFX7-NEXT:    ; return to shader part epilog
292;
293; GFX8-LABEL: s_mul_i64:
294; GFX8:       ; %bb.0:
295; GFX8-NEXT:    v_mov_b32_e32 v0, s2
296; GFX8-NEXT:    v_mul_hi_u32 v0, s0, v0
297; GFX8-NEXT:    s_mul_i32 s4, s0, s2
298; GFX8-NEXT:    s_mul_i32 s1, s1, s2
299; GFX8-NEXT:    s_mul_i32 s0, s0, s3
300; GFX8-NEXT:    s_add_i32 s1, s1, s0
301; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s1, v0
302; GFX8-NEXT:    v_readfirstlane_b32 s1, v0
303; GFX8-NEXT:    s_mov_b32 s0, s4
304; GFX8-NEXT:    ; return to shader part epilog
305;
306; GFX9-LABEL: s_mul_i64:
307; GFX9:       ; %bb.0:
308; GFX9-NEXT:    s_mul_i32 s1, s1, s2
309; GFX9-NEXT:    s_mul_i32 s3, s0, s3
310; GFX9-NEXT:    s_mul_i32 s4, s0, s2
311; GFX9-NEXT:    s_mul_hi_u32 s0, s0, s2
312; GFX9-NEXT:    s_add_i32 s1, s1, s3
313; GFX9-NEXT:    s_add_i32 s1, s1, s0
314; GFX9-NEXT:    s_mov_b32 s0, s4
315; GFX9-NEXT:    ; return to shader part epilog
316;
317; GFX10-LABEL: s_mul_i64:
318; GFX10:       ; %bb.0:
319; GFX10-NEXT:    s_mul_i32 s1, s1, s2
320; GFX10-NEXT:    s_mul_i32 s3, s0, s3
321; GFX10-NEXT:    s_mul_hi_u32 s4, s0, s2
322; GFX10-NEXT:    s_add_i32 s1, s1, s3
323; GFX10-NEXT:    s_mul_i32 s0, s0, s2
324; GFX10-NEXT:    s_add_i32 s1, s1, s4
325; GFX10-NEXT:    ; return to shader part epilog
326  %result = mul i64 %num, %den
327  ret i64 %result
328}
329
330define i64 @v_mul_i64(i64 %num, i64 %den) {
331; GFX7-LABEL: v_mul_i64:
332; GFX7:       ; %bb.0:
333; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
334; GFX7-NEXT:    v_mul_lo_u32 v4, v0, v3
335; GFX7-NEXT:    v_mul_lo_u32 v1, v1, v2
336; GFX7-NEXT:    v_mul_lo_u32 v3, v0, v2
337; GFX7-NEXT:    v_mul_hi_u32 v0, v0, v2
338; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v1, v4
339; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v1, v0
340; GFX7-NEXT:    v_mov_b32_e32 v0, v3
341; GFX7-NEXT:    s_setpc_b64 s[30:31]
342;
343; GFX8-LABEL: v_mul_i64:
344; GFX8:       ; %bb.0:
345; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
346; GFX8-NEXT:    v_mul_lo_u32 v4, v0, v3
347; GFX8-NEXT:    v_mul_lo_u32 v1, v1, v2
348; GFX8-NEXT:    v_mul_lo_u32 v3, v0, v2
349; GFX8-NEXT:    v_mul_hi_u32 v0, v0, v2
350; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v4
351; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v0
352; GFX8-NEXT:    v_mov_b32_e32 v0, v3
353; GFX8-NEXT:    s_setpc_b64 s[30:31]
354;
355; GFX9-LABEL: v_mul_i64:
356; GFX9:       ; %bb.0:
357; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
358; GFX9-NEXT:    v_mul_lo_u32 v1, v1, v2
359; GFX9-NEXT:    v_mul_lo_u32 v3, v0, v3
360; GFX9-NEXT:    v_mul_hi_u32 v4, v0, v2
361; GFX9-NEXT:    v_mul_lo_u32 v0, v0, v2
362; GFX9-NEXT:    v_add3_u32 v1, v1, v3, v4
363; GFX9-NEXT:    s_setpc_b64 s[30:31]
364;
365; GFX10-LABEL: v_mul_i64:
366; GFX10:       ; %bb.0:
367; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
368; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
369; GFX10-NEXT:    v_mul_lo_u32 v1, v1, v2
370; GFX10-NEXT:    v_mul_lo_u32 v3, v0, v3
371; GFX10-NEXT:    v_mul_hi_u32 v4, v0, v2
372; GFX10-NEXT:    v_mul_lo_u32 v0, v0, v2
373; GFX10-NEXT:    v_add3_u32 v1, v1, v3, v4
374; GFX10-NEXT:    s_setpc_b64 s[30:31]
375  %result = mul i64 %num, %den
376  ret i64 %result
377}
378
379define amdgpu_ps <3 x i32> @s_mul_i96(i96 inreg %num, i96 inreg %den) {
380; GFX7-LABEL: s_mul_i96:
381; GFX7:       ; %bb.0:
382; GFX7-NEXT:    v_mov_b32_e32 v0, s3
383; GFX7-NEXT:    v_mul_hi_u32 v0, s0, v0
384; GFX7-NEXT:    v_mov_b32_e32 v2, s1
385; GFX7-NEXT:    s_mul_i32 s7, s1, s3
386; GFX7-NEXT:    s_mul_i32 s8, s0, s4
387; GFX7-NEXT:    s_add_u32 s7, s7, s8
388; GFX7-NEXT:    v_mov_b32_e32 v3, s4
389; GFX7-NEXT:    v_mul_hi_u32 v2, v2, s3
390; GFX7-NEXT:    v_add_i32_e32 v0, vcc, s7, v0
391; GFX7-NEXT:    s_mul_i32 s7, s1, s4
392; GFX7-NEXT:    s_mul_i32 s2, s2, s3
393; GFX7-NEXT:    v_mul_hi_u32 v3, s0, v3
394; GFX7-NEXT:    s_cselect_b32 s8, 1, 0
395; GFX7-NEXT:    s_mul_i32 s6, s0, s3
396; GFX7-NEXT:    s_mul_i32 s5, s0, s5
397; GFX7-NEXT:    s_add_i32 s0, s2, s7
398; GFX7-NEXT:    s_add_i32 s0, s0, s5
399; GFX7-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
400; GFX7-NEXT:    v_add_i32_e32 v2, vcc, s0, v2
401; GFX7-NEXT:    s_and_b32 s8, s8, 1
402; GFX7-NEXT:    v_add_i32_e32 v1, vcc, s8, v1
403; GFX7-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
404; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
405; GFX7-NEXT:    v_readfirstlane_b32 s1, v0
406; GFX7-NEXT:    v_readfirstlane_b32 s2, v1
407; GFX7-NEXT:    s_mov_b32 s0, s6
408; GFX7-NEXT:    ; return to shader part epilog
409;
410; GFX8-LABEL: s_mul_i96:
411; GFX8:       ; %bb.0:
412; GFX8-NEXT:    v_mov_b32_e32 v0, s3
413; GFX8-NEXT:    v_mul_hi_u32 v0, s0, v0
414; GFX8-NEXT:    v_mov_b32_e32 v2, s1
415; GFX8-NEXT:    s_mul_i32 s7, s1, s3
416; GFX8-NEXT:    s_mul_i32 s8, s0, s4
417; GFX8-NEXT:    s_add_u32 s7, s7, s8
418; GFX8-NEXT:    v_mov_b32_e32 v3, s4
419; GFX8-NEXT:    v_mul_hi_u32 v2, v2, s3
420; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s7, v0
421; GFX8-NEXT:    s_mul_i32 s7, s1, s4
422; GFX8-NEXT:    s_mul_i32 s2, s2, s3
423; GFX8-NEXT:    v_mul_hi_u32 v3, s0, v3
424; GFX8-NEXT:    s_cselect_b32 s8, 1, 0
425; GFX8-NEXT:    s_mul_i32 s6, s0, s3
426; GFX8-NEXT:    s_mul_i32 s5, s0, s5
427; GFX8-NEXT:    s_add_i32 s0, s2, s7
428; GFX8-NEXT:    s_add_i32 s0, s0, s5
429; GFX8-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
430; GFX8-NEXT:    v_add_u32_e32 v2, vcc, s0, v2
431; GFX8-NEXT:    s_and_b32 s8, s8, 1
432; GFX8-NEXT:    v_add_u32_e32 v1, vcc, s8, v1
433; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v3
434; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v2, v1
435; GFX8-NEXT:    v_readfirstlane_b32 s1, v0
436; GFX8-NEXT:    v_readfirstlane_b32 s2, v1
437; GFX8-NEXT:    s_mov_b32 s0, s6
438; GFX8-NEXT:    ; return to shader part epilog
439;
440; GFX9-LABEL: s_mul_i96:
441; GFX9:       ; %bb.0:
442; GFX9-NEXT:    s_mul_i32 s7, s1, s3
443; GFX9-NEXT:    s_mul_i32 s8, s0, s4
444; GFX9-NEXT:    s_add_u32 s7, s7, s8
445; GFX9-NEXT:    s_cselect_b32 s8, 1, 0
446; GFX9-NEXT:    s_mul_hi_u32 s9, s0, s3
447; GFX9-NEXT:    s_and_b32 s8, s8, 1
448; GFX9-NEXT:    s_add_u32 s7, s7, s9
449; GFX9-NEXT:    s_cselect_b32 s9, 1, 0
450; GFX9-NEXT:    s_and_b32 s9, s9, 1
451; GFX9-NEXT:    s_add_i32 s8, s8, s9
452; GFX9-NEXT:    s_mul_i32 s9, s1, s4
453; GFX9-NEXT:    s_mul_i32 s2, s2, s3
454; GFX9-NEXT:    s_mul_i32 s5, s0, s5
455; GFX9-NEXT:    s_add_i32 s2, s2, s9
456; GFX9-NEXT:    s_mul_hi_u32 s1, s1, s3
457; GFX9-NEXT:    s_add_i32 s2, s2, s5
458; GFX9-NEXT:    s_mul_i32 s6, s0, s3
459; GFX9-NEXT:    s_mul_hi_u32 s0, s0, s4
460; GFX9-NEXT:    s_add_i32 s1, s2, s1
461; GFX9-NEXT:    s_add_i32 s0, s1, s0
462; GFX9-NEXT:    s_add_i32 s2, s0, s8
463; GFX9-NEXT:    s_mov_b32 s0, s6
464; GFX9-NEXT:    s_mov_b32 s1, s7
465; GFX9-NEXT:    ; return to shader part epilog
466;
467; GFX10-LABEL: s_mul_i96:
468; GFX10:       ; %bb.0:
469; GFX10-NEXT:    s_mul_i32 s6, s1, s3
470; GFX10-NEXT:    s_mul_i32 s7, s0, s4
471; GFX10-NEXT:    s_mul_hi_u32 s8, s0, s3
472; GFX10-NEXT:    s_add_u32 s6, s6, s7
473; GFX10-NEXT:    s_cselect_b32 s7, 1, 0
474; GFX10-NEXT:    s_mul_i32 s9, s1, s4
475; GFX10-NEXT:    s_and_b32 s7, s7, 1
476; GFX10-NEXT:    s_mul_i32 s2, s2, s3
477; GFX10-NEXT:    s_add_u32 s6, s6, s8
478; GFX10-NEXT:    s_cselect_b32 s8, 1, 0
479; GFX10-NEXT:    s_mul_i32 s5, s0, s5
480; GFX10-NEXT:    s_add_i32 s2, s2, s9
481; GFX10-NEXT:    s_mul_hi_u32 s1, s1, s3
482; GFX10-NEXT:    s_add_i32 s2, s2, s5
483; GFX10-NEXT:    s_and_b32 s8, s8, 1
484; GFX10-NEXT:    s_mul_hi_u32 s4, s0, s4
485; GFX10-NEXT:    s_add_i32 s1, s2, s1
486; GFX10-NEXT:    s_add_i32 s7, s7, s8
487; GFX10-NEXT:    s_add_i32 s1, s1, s4
488; GFX10-NEXT:    s_mul_i32 s0, s0, s3
489; GFX10-NEXT:    s_add_i32 s2, s1, s7
490; GFX10-NEXT:    s_mov_b32 s1, s6
491; GFX10-NEXT:    ; return to shader part epilog
492  %result = mul i96 %num, %den
493  %cast = bitcast i96 %result to <3 x i32>
494  ret <3 x i32> %cast
495}
496
497define i96 @v_mul_i96(i96 %num, i96 %den) {
498; GFX7-LABEL: v_mul_i96:
499; GFX7:       ; %bb.0:
500; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
501; GFX7-NEXT:    v_mul_lo_u32 v7, v1, v3
502; GFX7-NEXT:    v_mul_lo_u32 v8, v0, v4
503; GFX7-NEXT:    v_mul_hi_u32 v9, v0, v3
504; GFX7-NEXT:    v_mul_lo_u32 v2, v2, v3
505; GFX7-NEXT:    v_mul_lo_u32 v5, v0, v5
506; GFX7-NEXT:    v_add_i32_e32 v7, vcc, v7, v8
507; GFX7-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
508; GFX7-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
509; GFX7-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
510; GFX7-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
511; GFX7-NEXT:    v_mul_lo_u32 v9, v1, v4
512; GFX7-NEXT:    v_mul_hi_u32 v1, v1, v3
513; GFX7-NEXT:    v_mul_lo_u32 v6, v0, v3
514; GFX7-NEXT:    v_mul_hi_u32 v0, v0, v4
515; GFX7-NEXT:    v_add_i32_e32 v2, vcc, v2, v9
516; GFX7-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
517; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
518; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
519; GFX7-NEXT:    v_add_i32_e32 v2, vcc, v0, v8
520; GFX7-NEXT:    v_mov_b32_e32 v0, v6
521; GFX7-NEXT:    v_mov_b32_e32 v1, v7
522; GFX7-NEXT:    s_setpc_b64 s[30:31]
523;
524; GFX8-LABEL: v_mul_i96:
525; GFX8:       ; %bb.0:
526; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
527; GFX8-NEXT:    v_mul_lo_u32 v7, v1, v3
528; GFX8-NEXT:    v_mul_lo_u32 v8, v0, v4
529; GFX8-NEXT:    v_mul_hi_u32 v9, v0, v3
530; GFX8-NEXT:    v_mul_lo_u32 v2, v2, v3
531; GFX8-NEXT:    v_mul_lo_u32 v5, v0, v5
532; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v7, v8
533; GFX8-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
534; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v7, v9
535; GFX8-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
536; GFX8-NEXT:    v_add_u32_e32 v8, vcc, v8, v9
537; GFX8-NEXT:    v_mul_lo_u32 v9, v1, v4
538; GFX8-NEXT:    v_mul_hi_u32 v1, v1, v3
539; GFX8-NEXT:    v_mul_lo_u32 v6, v0, v3
540; GFX8-NEXT:    v_mul_hi_u32 v0, v0, v4
541; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v9
542; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v5
543; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v2, v1
544; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v1, v0
545; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v0, v8
546; GFX8-NEXT:    v_mov_b32_e32 v0, v6
547; GFX8-NEXT:    v_mov_b32_e32 v1, v7
548; GFX8-NEXT:    s_setpc_b64 s[30:31]
549;
550; GFX9-LABEL: v_mul_i96:
551; GFX9:       ; %bb.0:
552; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
553; GFX9-NEXT:    v_mul_lo_u32 v7, v1, v3
554; GFX9-NEXT:    v_mul_lo_u32 v8, v0, v4
555; GFX9-NEXT:    v_mul_hi_u32 v9, v0, v3
556; GFX9-NEXT:    v_mul_lo_u32 v10, v1, v4
557; GFX9-NEXT:    v_mul_lo_u32 v2, v2, v3
558; GFX9-NEXT:    v_mul_lo_u32 v5, v0, v5
559; GFX9-NEXT:    v_mul_hi_u32 v1, v1, v3
560; GFX9-NEXT:    v_add_co_u32_e32 v7, vcc, v7, v8
561; GFX9-NEXT:    v_mul_lo_u32 v6, v0, v3
562; GFX9-NEXT:    v_mul_hi_u32 v0, v0, v4
563; GFX9-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
564; GFX9-NEXT:    v_add_co_u32_e32 v7, vcc, v7, v9
565; GFX9-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
566; GFX9-NEXT:    v_add_u32_e32 v2, v2, v10
567; GFX9-NEXT:    v_add_u32_e32 v3, v8, v9
568; GFX9-NEXT:    v_add3_u32 v1, v2, v5, v1
569; GFX9-NEXT:    v_add3_u32 v2, v1, v0, v3
570; GFX9-NEXT:    v_mov_b32_e32 v0, v6
571; GFX9-NEXT:    v_mov_b32_e32 v1, v7
572; GFX9-NEXT:    s_setpc_b64 s[30:31]
573;
574; GFX10-LABEL: v_mul_i96:
575; GFX10:       ; %bb.0:
576; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
577; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
578; GFX10-NEXT:    v_mul_lo_u32 v6, v1, v3
579; GFX10-NEXT:    v_mul_lo_u32 v7, v0, v4
580; GFX10-NEXT:    v_mul_hi_u32 v8, v0, v3
581; GFX10-NEXT:    v_mul_lo_u32 v9, v1, v4
582; GFX10-NEXT:    v_mul_lo_u32 v2, v2, v3
583; GFX10-NEXT:    v_mul_lo_u32 v5, v0, v5
584; GFX10-NEXT:    v_mul_hi_u32 v4, v0, v4
585; GFX10-NEXT:    v_mul_lo_u32 v0, v0, v3
586; GFX10-NEXT:    v_add_co_u32 v6, s4, v6, v7
587; GFX10-NEXT:    v_mul_hi_u32 v7, v1, v3
588; GFX10-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s4
589; GFX10-NEXT:    v_add_nc_u32_e32 v2, v2, v9
590; GFX10-NEXT:    v_add_co_u32 v1, s4, v6, v8
591; GFX10-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s4
592; GFX10-NEXT:    v_add3_u32 v2, v2, v5, v7
593; GFX10-NEXT:    v_add_nc_u32_e32 v3, v10, v6
594; GFX10-NEXT:    v_add3_u32 v2, v2, v4, v3
595; GFX10-NEXT:    s_setpc_b64 s[30:31]
596  %result = mul i96 %num, %den
597  ret i96 %result
598}
599
600define amdgpu_ps <4 x i32> @s_mul_i128(i128 inreg %num, i128 inreg %den) {
601; GFX7-LABEL: s_mul_i128:
602; GFX7:       ; %bb.0:
603; GFX7-NEXT:    v_mov_b32_e32 v0, s4
604; GFX7-NEXT:    v_mul_hi_u32 v0, s0, v0
605; GFX7-NEXT:    s_mul_i32 s9, s1, s4
606; GFX7-NEXT:    s_mul_i32 s10, s0, s5
607; GFX7-NEXT:    s_add_u32 s9, s9, s10
608; GFX7-NEXT:    s_cselect_b32 s10, 1, 0
609; GFX7-NEXT:    v_add_i32_e32 v0, vcc, s9, v0
610; GFX7-NEXT:    s_and_b32 s10, s10, 1
611; GFX7-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
612; GFX7-NEXT:    v_add_i32_e32 v1, vcc, s10, v1
613; GFX7-NEXT:    s_mul_i32 s9, s2, s4
614; GFX7-NEXT:    s_mul_i32 s10, s1, s5
615; GFX7-NEXT:    v_mov_b32_e32 v2, s1
616; GFX7-NEXT:    s_add_u32 s9, s9, s10
617; GFX7-NEXT:    s_cselect_b32 s10, 1, 0
618; GFX7-NEXT:    v_mul_hi_u32 v2, v2, s4
619; GFX7-NEXT:    s_mul_i32 s11, s0, s6
620; GFX7-NEXT:    s_and_b32 s10, s10, 1
621; GFX7-NEXT:    s_add_u32 s9, s9, s11
622; GFX7-NEXT:    v_mov_b32_e32 v3, s5
623; GFX7-NEXT:    s_cselect_b32 s11, 1, 0
624; GFX7-NEXT:    v_mul_hi_u32 v4, s0, v3
625; GFX7-NEXT:    v_add_i32_e32 v2, vcc, s9, v2
626; GFX7-NEXT:    s_and_b32 s11, s11, 1
627; GFX7-NEXT:    s_add_i32 s10, s10, s11
628; GFX7-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
629; GFX7-NEXT:    v_add_i32_e32 v5, vcc, s10, v5
630; GFX7-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
631; GFX7-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
632; GFX7-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
633; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
634; GFX7-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
635; GFX7-NEXT:    v_add_i32_e32 v2, vcc, v4, v2
636; GFX7-NEXT:    v_mov_b32_e32 v4, s2
637; GFX7-NEXT:    v_mov_b32_e32 v5, s6
638; GFX7-NEXT:    s_mul_i32 s5, s2, s5
639; GFX7-NEXT:    s_mul_i32 s3, s3, s4
640; GFX7-NEXT:    v_mul_hi_u32 v4, v4, s4
641; GFX7-NEXT:    s_mul_i32 s8, s0, s4
642; GFX7-NEXT:    s_mul_i32 s9, s1, s6
643; GFX7-NEXT:    v_mul_hi_u32 v3, s1, v3
644; GFX7-NEXT:    s_mul_i32 s7, s0, s7
645; GFX7-NEXT:    v_mul_hi_u32 v5, s0, v5
646; GFX7-NEXT:    s_add_i32 s0, s3, s5
647; GFX7-NEXT:    s_add_i32 s0, s0, s9
648; GFX7-NEXT:    s_add_i32 s0, s0, s7
649; GFX7-NEXT:    v_add_i32_e32 v4, vcc, s0, v4
650; GFX7-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
651; GFX7-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
652; GFX7-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
653; GFX7-NEXT:    v_readfirstlane_b32 s1, v0
654; GFX7-NEXT:    v_readfirstlane_b32 s2, v1
655; GFX7-NEXT:    v_readfirstlane_b32 s3, v2
656; GFX7-NEXT:    s_mov_b32 s0, s8
657; GFX7-NEXT:    ; return to shader part epilog
658;
659; GFX8-LABEL: s_mul_i128:
660; GFX8:       ; %bb.0:
661; GFX8-NEXT:    v_mov_b32_e32 v0, s4
662; GFX8-NEXT:    v_mul_hi_u32 v0, s0, v0
663; GFX8-NEXT:    s_mul_i32 s9, s1, s4
664; GFX8-NEXT:    s_mul_i32 s10, s0, s5
665; GFX8-NEXT:    s_add_u32 s9, s9, s10
666; GFX8-NEXT:    s_cselect_b32 s10, 1, 0
667; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s9, v0
668; GFX8-NEXT:    s_and_b32 s10, s10, 1
669; GFX8-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
670; GFX8-NEXT:    v_add_u32_e32 v1, vcc, s10, v1
671; GFX8-NEXT:    s_mul_i32 s9, s2, s4
672; GFX8-NEXT:    s_mul_i32 s10, s1, s5
673; GFX8-NEXT:    v_mov_b32_e32 v2, s1
674; GFX8-NEXT:    s_add_u32 s9, s9, s10
675; GFX8-NEXT:    s_cselect_b32 s10, 1, 0
676; GFX8-NEXT:    v_mul_hi_u32 v2, v2, s4
677; GFX8-NEXT:    s_mul_i32 s11, s0, s6
678; GFX8-NEXT:    s_and_b32 s10, s10, 1
679; GFX8-NEXT:    s_add_u32 s9, s9, s11
680; GFX8-NEXT:    v_mov_b32_e32 v3, s5
681; GFX8-NEXT:    s_cselect_b32 s11, 1, 0
682; GFX8-NEXT:    v_mul_hi_u32 v4, s0, v3
683; GFX8-NEXT:    v_add_u32_e32 v2, vcc, s9, v2
684; GFX8-NEXT:    s_and_b32 s11, s11, 1
685; GFX8-NEXT:    s_add_i32 s10, s10, s11
686; GFX8-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
687; GFX8-NEXT:    v_add_u32_e32 v5, vcc, s10, v5
688; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v4
689; GFX8-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
690; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v5, v4
691; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v2, v1
692; GFX8-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
693; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v4, v2
694; GFX8-NEXT:    v_mov_b32_e32 v4, s2
695; GFX8-NEXT:    v_mov_b32_e32 v5, s6
696; GFX8-NEXT:    s_mul_i32 s5, s2, s5
697; GFX8-NEXT:    s_mul_i32 s3, s3, s4
698; GFX8-NEXT:    v_mul_hi_u32 v4, v4, s4
699; GFX8-NEXT:    s_mul_i32 s8, s0, s4
700; GFX8-NEXT:    s_mul_i32 s9, s1, s6
701; GFX8-NEXT:    v_mul_hi_u32 v3, s1, v3
702; GFX8-NEXT:    s_mul_i32 s7, s0, s7
703; GFX8-NEXT:    v_mul_hi_u32 v5, s0, v5
704; GFX8-NEXT:    s_add_i32 s0, s3, s5
705; GFX8-NEXT:    s_add_i32 s0, s0, s9
706; GFX8-NEXT:    s_add_i32 s0, s0, s7
707; GFX8-NEXT:    v_add_u32_e32 v4, vcc, s0, v4
708; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v4, v3
709; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v3, v5
710; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v3, v2
711; GFX8-NEXT:    v_readfirstlane_b32 s1, v0
712; GFX8-NEXT:    v_readfirstlane_b32 s2, v1
713; GFX8-NEXT:    v_readfirstlane_b32 s3, v2
714; GFX8-NEXT:    s_mov_b32 s0, s8
715; GFX8-NEXT:    ; return to shader part epilog
716;
717; GFX9-LABEL: s_mul_i128:
718; GFX9:       ; %bb.0:
719; GFX9-NEXT:    s_mul_i32 s9, s1, s4
720; GFX9-NEXT:    s_mul_i32 s10, s0, s5
721; GFX9-NEXT:    s_add_u32 s9, s9, s10
722; GFX9-NEXT:    s_cselect_b32 s10, 1, 0
723; GFX9-NEXT:    s_mul_hi_u32 s11, s0, s4
724; GFX9-NEXT:    s_and_b32 s10, s10, 1
725; GFX9-NEXT:    s_add_u32 s9, s9, s11
726; GFX9-NEXT:    s_cselect_b32 s11, 1, 0
727; GFX9-NEXT:    s_and_b32 s11, s11, 1
728; GFX9-NEXT:    s_add_i32 s10, s10, s11
729; GFX9-NEXT:    s_mul_i32 s11, s2, s4
730; GFX9-NEXT:    s_mul_i32 s12, s1, s5
731; GFX9-NEXT:    s_add_u32 s11, s11, s12
732; GFX9-NEXT:    s_cselect_b32 s12, 1, 0
733; GFX9-NEXT:    s_mul_i32 s13, s0, s6
734; GFX9-NEXT:    s_and_b32 s12, s12, 1
735; GFX9-NEXT:    s_add_u32 s11, s11, s13
736; GFX9-NEXT:    s_cselect_b32 s13, 1, 0
737; GFX9-NEXT:    s_and_b32 s13, s13, 1
738; GFX9-NEXT:    s_mul_hi_u32 s14, s1, s4
739; GFX9-NEXT:    s_add_i32 s12, s12, s13
740; GFX9-NEXT:    s_add_u32 s11, s11, s14
741; GFX9-NEXT:    s_cselect_b32 s13, 1, 0
742; GFX9-NEXT:    s_and_b32 s13, s13, 1
743; GFX9-NEXT:    s_mul_hi_u32 s15, s0, s5
744; GFX9-NEXT:    s_add_i32 s12, s12, s13
745; GFX9-NEXT:    s_add_u32 s11, s11, s15
746; GFX9-NEXT:    s_cselect_b32 s13, 1, 0
747; GFX9-NEXT:    s_and_b32 s13, s13, 1
748; GFX9-NEXT:    s_add_i32 s12, s12, s13
749; GFX9-NEXT:    s_add_u32 s10, s11, s10
750; GFX9-NEXT:    s_cselect_b32 s11, 1, 0
751; GFX9-NEXT:    s_and_b32 s11, s11, 1
752; GFX9-NEXT:    s_add_i32 s12, s12, s11
753; GFX9-NEXT:    s_mul_i32 s11, s2, s5
754; GFX9-NEXT:    s_mul_i32 s3, s3, s4
755; GFX9-NEXT:    s_mul_i32 s13, s1, s6
756; GFX9-NEXT:    s_add_i32 s3, s3, s11
757; GFX9-NEXT:    s_mul_i32 s7, s0, s7
758; GFX9-NEXT:    s_add_i32 s3, s3, s13
759; GFX9-NEXT:    s_mul_hi_u32 s2, s2, s4
760; GFX9-NEXT:    s_add_i32 s3, s3, s7
761; GFX9-NEXT:    s_mul_hi_u32 s1, s1, s5
762; GFX9-NEXT:    s_add_i32 s2, s3, s2
763; GFX9-NEXT:    s_mul_i32 s8, s0, s4
764; GFX9-NEXT:    s_add_i32 s1, s2, s1
765; GFX9-NEXT:    s_mul_hi_u32 s0, s0, s6
766; GFX9-NEXT:    s_add_i32 s0, s1, s0
767; GFX9-NEXT:    s_add_i32 s3, s0, s12
768; GFX9-NEXT:    s_mov_b32 s0, s8
769; GFX9-NEXT:    s_mov_b32 s1, s9
770; GFX9-NEXT:    s_mov_b32 s2, s10
771; GFX9-NEXT:    ; return to shader part epilog
772;
773; GFX10-LABEL: s_mul_i128:
774; GFX10:       ; %bb.0:
775; GFX10-NEXT:    s_mul_i32 s8, s1, s4
776; GFX10-NEXT:    s_mul_i32 s9, s0, s5
777; GFX10-NEXT:    s_mul_hi_u32 s10, s0, s4
778; GFX10-NEXT:    s_add_u32 s8, s8, s9
779; GFX10-NEXT:    s_cselect_b32 s9, 1, 0
780; GFX10-NEXT:    s_mul_i32 s11, s1, s5
781; GFX10-NEXT:    s_and_b32 s9, s9, 1
782; GFX10-NEXT:    s_add_u32 s8, s8, s10
783; GFX10-NEXT:    s_cselect_b32 s10, 1, 0
784; GFX10-NEXT:    s_mul_i32 s12, s0, s6
785; GFX10-NEXT:    s_and_b32 s10, s10, 1
786; GFX10-NEXT:    s_mul_hi_u32 s13, s1, s4
787; GFX10-NEXT:    s_add_i32 s9, s9, s10
788; GFX10-NEXT:    s_mul_i32 s10, s2, s4
789; GFX10-NEXT:    s_mul_i32 s3, s3, s4
790; GFX10-NEXT:    s_add_u32 s10, s10, s11
791; GFX10-NEXT:    s_cselect_b32 s11, 1, 0
792; GFX10-NEXT:    s_mul_i32 s7, s0, s7
793; GFX10-NEXT:    s_and_b32 s11, s11, 1
794; GFX10-NEXT:    s_add_u32 s10, s10, s12
795; GFX10-NEXT:    s_cselect_b32 s12, 1, 0
796; GFX10-NEXT:    s_and_b32 s12, s12, 1
797; GFX10-NEXT:    s_add_i32 s11, s11, s12
798; GFX10-NEXT:    s_add_u32 s10, s10, s13
799; GFX10-NEXT:    s_cselect_b32 s12, 1, 0
800; GFX10-NEXT:    s_mul_hi_u32 s13, s0, s5
801; GFX10-NEXT:    s_and_b32 s12, s12, 1
802; GFX10-NEXT:    s_add_i32 s11, s11, s12
803; GFX10-NEXT:    s_add_u32 s10, s10, s13
804; GFX10-NEXT:    s_cselect_b32 s12, 1, 0
805; GFX10-NEXT:    s_mul_i32 s13, s1, s6
806; GFX10-NEXT:    s_and_b32 s12, s12, 1
807; GFX10-NEXT:    s_mul_hi_u32 s1, s1, s5
808; GFX10-NEXT:    s_add_i32 s11, s11, s12
809; GFX10-NEXT:    s_mul_i32 s12, s2, s5
810; GFX10-NEXT:    s_add_u32 s9, s10, s9
811; GFX10-NEXT:    s_cselect_b32 s10, 1, 0
812; GFX10-NEXT:    s_add_i32 s3, s3, s12
813; GFX10-NEXT:    s_mul_hi_u32 s2, s2, s4
814; GFX10-NEXT:    s_add_i32 s3, s3, s13
815; GFX10-NEXT:    s_and_b32 s10, s10, 1
816; GFX10-NEXT:    s_add_i32 s3, s3, s7
817; GFX10-NEXT:    s_add_i32 s11, s11, s10
818; GFX10-NEXT:    s_add_i32 s2, s3, s2
819; GFX10-NEXT:    s_mul_hi_u32 s3, s0, s6
820; GFX10-NEXT:    s_add_i32 s1, s2, s1
821; GFX10-NEXT:    s_mul_i32 s0, s0, s4
822; GFX10-NEXT:    s_add_i32 s1, s1, s3
823; GFX10-NEXT:    s_mov_b32 s2, s9
824; GFX10-NEXT:    s_add_i32 s3, s1, s11
825; GFX10-NEXT:    s_mov_b32 s1, s8
826; GFX10-NEXT:    ; return to shader part epilog
827  %result = mul i128 %num, %den
828  %cast = bitcast i128 %result to <4 x i32>
829  ret <4 x i32> %cast
830}
831
832define i128 @v_mul_i128(i128 %num, i128 %den) {
833; GFX7-LABEL: v_mul_i128:
834; GFX7:       ; %bb.0:
835; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
836; GFX7-NEXT:    v_mul_lo_u32 v9, v1, v4
837; GFX7-NEXT:    v_mul_lo_u32 v10, v0, v5
838; GFX7-NEXT:    v_mul_hi_u32 v11, v0, v4
839; GFX7-NEXT:    v_mul_lo_u32 v12, v1, v5
840; GFX7-NEXT:    v_mul_lo_u32 v13, v0, v6
841; GFX7-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
842; GFX7-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
843; GFX7-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
844; GFX7-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
845; GFX7-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
846; GFX7-NEXT:    v_mul_lo_u32 v11, v2, v4
847; GFX7-NEXT:    v_mul_hi_u32 v14, v1, v4
848; GFX7-NEXT:    v_mul_hi_u32 v15, v0, v5
849; GFX7-NEXT:    v_mul_lo_u32 v3, v3, v4
850; GFX7-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
851; GFX7-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
852; GFX7-NEXT:    v_add_i32_e32 v11, vcc, v11, v13
853; GFX7-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
854; GFX7-NEXT:    v_add_i32_e32 v12, vcc, v12, v13
855; GFX7-NEXT:    v_add_i32_e32 v11, vcc, v11, v14
856; GFX7-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
857; GFX7-NEXT:    v_add_i32_e32 v12, vcc, v12, v13
858; GFX7-NEXT:    v_add_i32_e32 v11, vcc, v11, v15
859; GFX7-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
860; GFX7-NEXT:    v_add_i32_e32 v12, vcc, v12, v13
861; GFX7-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
862; GFX7-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
863; GFX7-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
864; GFX7-NEXT:    v_mul_lo_u32 v12, v2, v5
865; GFX7-NEXT:    v_mul_lo_u32 v13, v1, v6
866; GFX7-NEXT:    v_mul_lo_u32 v7, v0, v7
867; GFX7-NEXT:    v_mul_hi_u32 v2, v2, v4
868; GFX7-NEXT:    v_add_i32_e32 v3, vcc, v3, v12
869; GFX7-NEXT:    v_mul_hi_u32 v1, v1, v5
870; GFX7-NEXT:    v_add_i32_e32 v3, vcc, v3, v13
871; GFX7-NEXT:    v_mul_lo_u32 v8, v0, v4
872; GFX7-NEXT:    v_mul_hi_u32 v0, v0, v6
873; GFX7-NEXT:    v_add_i32_e32 v3, vcc, v3, v7
874; GFX7-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
875; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
876; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
877; GFX7-NEXT:    v_add_i32_e32 v3, vcc, v0, v11
878; GFX7-NEXT:    v_mov_b32_e32 v0, v8
879; GFX7-NEXT:    v_mov_b32_e32 v1, v9
880; GFX7-NEXT:    v_mov_b32_e32 v2, v10
881; GFX7-NEXT:    s_setpc_b64 s[30:31]
882;
883; GFX8-LABEL: v_mul_i128:
884; GFX8:       ; %bb.0:
885; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
886; GFX8-NEXT:    v_mul_lo_u32 v9, v1, v4
887; GFX8-NEXT:    v_mul_lo_u32 v10, v0, v5
888; GFX8-NEXT:    v_mul_hi_u32 v11, v0, v4
889; GFX8-NEXT:    v_mul_lo_u32 v12, v1, v5
890; GFX8-NEXT:    v_mul_lo_u32 v13, v0, v6
891; GFX8-NEXT:    v_add_u32_e32 v9, vcc, v9, v10
892; GFX8-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
893; GFX8-NEXT:    v_add_u32_e32 v9, vcc, v9, v11
894; GFX8-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
895; GFX8-NEXT:    v_add_u32_e32 v10, vcc, v10, v11
896; GFX8-NEXT:    v_mul_lo_u32 v11, v2, v4
897; GFX8-NEXT:    v_mul_hi_u32 v14, v1, v4
898; GFX8-NEXT:    v_mul_hi_u32 v15, v0, v5
899; GFX8-NEXT:    v_mul_lo_u32 v3, v3, v4
900; GFX8-NEXT:    v_add_u32_e32 v11, vcc, v11, v12
901; GFX8-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
902; GFX8-NEXT:    v_add_u32_e32 v11, vcc, v11, v13
903; GFX8-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
904; GFX8-NEXT:    v_add_u32_e32 v12, vcc, v12, v13
905; GFX8-NEXT:    v_add_u32_e32 v11, vcc, v11, v14
906; GFX8-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
907; GFX8-NEXT:    v_add_u32_e32 v12, vcc, v12, v13
908; GFX8-NEXT:    v_add_u32_e32 v11, vcc, v11, v15
909; GFX8-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
910; GFX8-NEXT:    v_add_u32_e32 v12, vcc, v12, v13
911; GFX8-NEXT:    v_add_u32_e32 v10, vcc, v11, v10
912; GFX8-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
913; GFX8-NEXT:    v_add_u32_e32 v11, vcc, v12, v11
914; GFX8-NEXT:    v_mul_lo_u32 v12, v2, v5
915; GFX8-NEXT:    v_mul_lo_u32 v13, v1, v6
916; GFX8-NEXT:    v_mul_lo_u32 v7, v0, v7
917; GFX8-NEXT:    v_mul_hi_u32 v2, v2, v4
918; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v3, v12
919; GFX8-NEXT:    v_mul_hi_u32 v1, v1, v5
920; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v3, v13
921; GFX8-NEXT:    v_mul_lo_u32 v8, v0, v4
922; GFX8-NEXT:    v_mul_hi_u32 v0, v0, v6
923; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v3, v7
924; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v3, v2
925; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v2, v1
926; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v1, v0
927; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v0, v11
928; GFX8-NEXT:    v_mov_b32_e32 v0, v8
929; GFX8-NEXT:    v_mov_b32_e32 v1, v9
930; GFX8-NEXT:    v_mov_b32_e32 v2, v10
931; GFX8-NEXT:    s_setpc_b64 s[30:31]
932;
933; GFX9-LABEL: v_mul_i128:
934; GFX9:       ; %bb.0:
935; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
936; GFX9-NEXT:    v_mul_lo_u32 v9, v1, v4
937; GFX9-NEXT:    v_mul_lo_u32 v10, v0, v5
938; GFX9-NEXT:    v_mul_hi_u32 v11, v0, v4
939; GFX9-NEXT:    v_mul_lo_u32 v12, v1, v5
940; GFX9-NEXT:    v_mul_lo_u32 v13, v0, v6
941; GFX9-NEXT:    v_add_co_u32_e32 v9, vcc, v9, v10
942; GFX9-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
943; GFX9-NEXT:    v_add_co_u32_e32 v9, vcc, v9, v11
944; GFX9-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
945; GFX9-NEXT:    v_add_u32_e32 v10, v10, v11
946; GFX9-NEXT:    v_mul_lo_u32 v11, v2, v4
947; GFX9-NEXT:    v_mul_hi_u32 v14, v1, v4
948; GFX9-NEXT:    v_mul_hi_u32 v15, v0, v5
949; GFX9-NEXT:    v_mul_lo_u32 v3, v3, v4
950; GFX9-NEXT:    v_add_co_u32_e32 v11, vcc, v11, v12
951; GFX9-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
952; GFX9-NEXT:    v_add_co_u32_e32 v11, vcc, v11, v13
953; GFX9-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
954; GFX9-NEXT:    v_add_co_u32_e32 v11, vcc, v11, v14
955; GFX9-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
956; GFX9-NEXT:    v_add_co_u32_e32 v11, vcc, v11, v15
957; GFX9-NEXT:    v_add3_u32 v12, v12, v13, v14
958; GFX9-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
959; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, v11, v10
960; GFX9-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
961; GFX9-NEXT:    v_add3_u32 v11, v12, v13, v11
962; GFX9-NEXT:    v_mul_lo_u32 v12, v2, v5
963; GFX9-NEXT:    v_mul_lo_u32 v13, v1, v6
964; GFX9-NEXT:    v_mul_lo_u32 v7, v0, v7
965; GFX9-NEXT:    v_mul_hi_u32 v2, v2, v4
966; GFX9-NEXT:    v_mul_hi_u32 v1, v1, v5
967; GFX9-NEXT:    v_mul_lo_u32 v8, v0, v4
968; GFX9-NEXT:    v_mul_hi_u32 v0, v0, v6
969; GFX9-NEXT:    v_add_u32_e32 v3, v3, v12
970; GFX9-NEXT:    v_add3_u32 v3, v3, v13, v7
971; GFX9-NEXT:    v_add3_u32 v1, v3, v2, v1
972; GFX9-NEXT:    v_add3_u32 v3, v1, v0, v11
973; GFX9-NEXT:    v_mov_b32_e32 v0, v8
974; GFX9-NEXT:    v_mov_b32_e32 v1, v9
975; GFX9-NEXT:    v_mov_b32_e32 v2, v10
976; GFX9-NEXT:    s_setpc_b64 s[30:31]
977;
978; GFX10-LABEL: v_mul_i128:
979; GFX10:       ; %bb.0:
980; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
981; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
982; GFX10-NEXT:    v_mul_lo_u32 v8, v2, v4
983; GFX10-NEXT:    v_mul_lo_u32 v9, v1, v5
984; GFX10-NEXT:    v_mul_lo_u32 v10, v1, v4
985; GFX10-NEXT:    v_mul_lo_u32 v11, v0, v5
986; GFX10-NEXT:    v_mul_hi_u32 v12, v0, v4
987; GFX10-NEXT:    v_mul_lo_u32 v13, v0, v6
988; GFX10-NEXT:    v_mul_hi_u32 v15, v0, v5
989; GFX10-NEXT:    v_mul_lo_u32 v3, v3, v4
990; GFX10-NEXT:    v_mul_lo_u32 v7, v0, v7
991; GFX10-NEXT:    v_add_co_u32 v8, s4, v8, v9
992; GFX10-NEXT:    v_add_co_u32 v9, s5, v10, v11
993; GFX10-NEXT:    v_mul_hi_u32 v11, v1, v4
994; GFX10-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s4
995; GFX10-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s5
996; GFX10-NEXT:    v_add_co_u32 v13, s4, v8, v13
997; GFX10-NEXT:    v_add_co_u32 v8, s5, v9, v12
998; GFX10-NEXT:    v_cndmask_b32_e64 v12, 0, 1, s4
999; GFX10-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s5
1000; GFX10-NEXT:    v_add_co_u32 v11, s4, v13, v11
1001; GFX10-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s4
1002; GFX10-NEXT:    v_add_nc_u32_e32 v9, v10, v9
1003; GFX10-NEXT:    v_mul_lo_u32 v10, v2, v5
1004; GFX10-NEXT:    v_add_co_u32 v11, s4, v11, v15
1005; GFX10-NEXT:    v_mul_hi_u32 v15, v2, v4
1006; GFX10-NEXT:    v_add3_u32 v12, v14, v12, v13
1007; GFX10-NEXT:    v_mul_lo_u32 v13, v1, v6
1008; GFX10-NEXT:    v_mul_hi_u32 v1, v1, v5
1009; GFX10-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s4
1010; GFX10-NEXT:    v_add_co_u32 v2, s4, v11, v9
1011; GFX10-NEXT:    v_add_nc_u32_e32 v3, v3, v10
1012; GFX10-NEXT:    v_cndmask_b32_e64 v5, 0, 1, s4
1013; GFX10-NEXT:    v_mul_hi_u32 v6, v0, v6
1014; GFX10-NEXT:    v_mul_lo_u32 v0, v0, v4
1015; GFX10-NEXT:    v_add3_u32 v3, v3, v13, v7
1016; GFX10-NEXT:    v_add3_u32 v4, v12, v14, v5
1017; GFX10-NEXT:    v_add3_u32 v1, v3, v15, v1
1018; GFX10-NEXT:    v_add3_u32 v3, v1, v6, v4
1019; GFX10-NEXT:    v_mov_b32_e32 v1, v8
1020; GFX10-NEXT:    s_setpc_b64 s[30:31]
1021  %result = mul i128 %num, %den
1022  ret i128 %result
1023}
1024
1025define amdgpu_ps <8 x i32> @s_mul_i256(i256 inreg %num, i256 inreg %den) {
1026; GFX7-LABEL: s_mul_i256:
1027; GFX7:       ; %bb.0:
1028; GFX7-NEXT:    v_mov_b32_e32 v0, s8
1029; GFX7-NEXT:    v_mul_hi_u32 v0, s0, v0
1030; GFX7-NEXT:    s_mul_i32 s17, s1, s8
1031; GFX7-NEXT:    s_mul_i32 s18, s0, s9
1032; GFX7-NEXT:    s_add_u32 s17, s17, s18
1033; GFX7-NEXT:    s_cselect_b32 s18, 1, 0
1034; GFX7-NEXT:    v_add_i32_e32 v0, vcc, s17, v0
1035; GFX7-NEXT:    s_and_b32 s18, s18, 1
1036; GFX7-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
1037; GFX7-NEXT:    v_add_i32_e32 v1, vcc, s18, v1
1038; GFX7-NEXT:    s_mul_i32 s17, s2, s8
1039; GFX7-NEXT:    s_mul_i32 s18, s1, s9
1040; GFX7-NEXT:    v_mov_b32_e32 v2, s1
1041; GFX7-NEXT:    s_add_u32 s17, s17, s18
1042; GFX7-NEXT:    s_cselect_b32 s18, 1, 0
1043; GFX7-NEXT:    v_mul_hi_u32 v2, v2, s8
1044; GFX7-NEXT:    s_mul_i32 s19, s0, s10
1045; GFX7-NEXT:    s_and_b32 s18, s18, 1
1046; GFX7-NEXT:    s_add_u32 s17, s17, s19
1047; GFX7-NEXT:    v_mov_b32_e32 v3, s9
1048; GFX7-NEXT:    s_cselect_b32 s19, 1, 0
1049; GFX7-NEXT:    v_mul_hi_u32 v4, s0, v3
1050; GFX7-NEXT:    v_add_i32_e32 v2, vcc, s17, v2
1051; GFX7-NEXT:    s_and_b32 s19, s19, 1
1052; GFX7-NEXT:    s_add_i32 s18, s18, s19
1053; GFX7-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
1054; GFX7-NEXT:    v_add_i32_e32 v5, vcc, s18, v5
1055; GFX7-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
1056; GFX7-NEXT:    s_mul_i32 s17, s3, s8
1057; GFX7-NEXT:    s_mul_i32 s18, s2, s9
1058; GFX7-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
1059; GFX7-NEXT:    s_add_u32 s17, s17, s18
1060; GFX7-NEXT:    s_cselect_b32 s18, 1, 0
1061; GFX7-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
1062; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
1063; GFX7-NEXT:    s_mul_i32 s19, s1, s10
1064; GFX7-NEXT:    s_and_b32 s18, s18, 1
1065; GFX7-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
1066; GFX7-NEXT:    s_add_u32 s17, s17, s19
1067; GFX7-NEXT:    s_cselect_b32 s19, 1, 0
1068; GFX7-NEXT:    v_add_i32_e32 v2, vcc, v4, v2
1069; GFX7-NEXT:    v_mov_b32_e32 v4, s2
1070; GFX7-NEXT:    v_mul_hi_u32 v5, v4, s8
1071; GFX7-NEXT:    s_and_b32 s19, s19, 1
1072; GFX7-NEXT:    s_mul_i32 s20, s0, s11
1073; GFX7-NEXT:    s_add_i32 s18, s18, s19
1074; GFX7-NEXT:    s_add_u32 s17, s17, s20
1075; GFX7-NEXT:    s_cselect_b32 s19, 1, 0
1076; GFX7-NEXT:    v_mul_hi_u32 v3, s1, v3
1077; GFX7-NEXT:    v_add_i32_e32 v5, vcc, s17, v5
1078; GFX7-NEXT:    s_and_b32 s19, s19, 1
1079; GFX7-NEXT:    v_mov_b32_e32 v6, s10
1080; GFX7-NEXT:    s_add_i32 s18, s18, s19
1081; GFX7-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
1082; GFX7-NEXT:    v_add_i32_e32 v8, vcc, s18, v8
1083; GFX7-NEXT:    v_mul_hi_u32 v7, s0, v6
1084; GFX7-NEXT:    s_mul_i32 s17, s4, s8
1085; GFX7-NEXT:    s_mul_i32 s18, s3, s9
1086; GFX7-NEXT:    v_add_i32_e32 v3, vcc, v5, v3
1087; GFX7-NEXT:    s_add_u32 s17, s17, s18
1088; GFX7-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
1089; GFX7-NEXT:    s_cselect_b32 s18, 1, 0
1090; GFX7-NEXT:    v_add_i32_e32 v5, vcc, v8, v5
1091; GFX7-NEXT:    s_mul_i32 s19, s2, s10
1092; GFX7-NEXT:    s_and_b32 s18, s18, 1
1093; GFX7-NEXT:    v_add_i32_e32 v3, vcc, v3, v7
1094; GFX7-NEXT:    s_add_u32 s17, s17, s19
1095; GFX7-NEXT:    s_cselect_b32 s19, 1, 0
1096; GFX7-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
1097; GFX7-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
1098; GFX7-NEXT:    s_and_b32 s19, s19, 1
1099; GFX7-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
1100; GFX7-NEXT:    s_mul_i32 s20, s1, s11
1101; GFX7-NEXT:    s_add_i32 s18, s18, s19
1102; GFX7-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
1103; GFX7-NEXT:    s_add_u32 s17, s17, s20
1104; GFX7-NEXT:    s_cselect_b32 s19, 1, 0
1105; GFX7-NEXT:    v_add_i32_e32 v3, vcc, v5, v3
1106; GFX7-NEXT:    v_mov_b32_e32 v5, s3
1107; GFX7-NEXT:    s_and_b32 s19, s19, 1
1108; GFX7-NEXT:    v_mul_hi_u32 v7, v5, s8
1109; GFX7-NEXT:    s_mul_i32 s21, s0, s12
1110; GFX7-NEXT:    s_add_i32 s18, s18, s19
1111; GFX7-NEXT:    s_add_u32 s17, s17, s21
1112; GFX7-NEXT:    s_cselect_b32 s19, 1, 0
1113; GFX7-NEXT:    v_add_i32_e32 v7, vcc, s17, v7
1114; GFX7-NEXT:    s_and_b32 s19, s19, 1
1115; GFX7-NEXT:    v_mul_hi_u32 v4, v4, s9
1116; GFX7-NEXT:    s_add_i32 s18, s18, s19
1117; GFX7-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
1118; GFX7-NEXT:    v_add_i32_e32 v11, vcc, s18, v11
1119; GFX7-NEXT:    s_mul_i32 s17, s5, s8
1120; GFX7-NEXT:    s_mul_i32 s18, s4, s9
1121; GFX7-NEXT:    s_add_u32 s17, s17, s18
1122; GFX7-NEXT:    v_mul_hi_u32 v8, s1, v6
1123; GFX7-NEXT:    s_cselect_b32 s18, 1, 0
1124; GFX7-NEXT:    v_add_i32_e32 v4, vcc, v7, v4
1125; GFX7-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
1126; GFX7-NEXT:    v_mov_b32_e32 v9, s11
1127; GFX7-NEXT:    s_mul_i32 s19, s3, s10
1128; GFX7-NEXT:    s_and_b32 s18, s18, 1
1129; GFX7-NEXT:    v_add_i32_e32 v7, vcc, v11, v7
1130; GFX7-NEXT:    s_add_u32 s17, s17, s19
1131; GFX7-NEXT:    v_mul_hi_u32 v10, s0, v9
1132; GFX7-NEXT:    s_cselect_b32 s19, 1, 0
1133; GFX7-NEXT:    v_add_i32_e32 v4, vcc, v4, v8
1134; GFX7-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
1135; GFX7-NEXT:    s_and_b32 s19, s19, 1
1136; GFX7-NEXT:    v_add_i32_e32 v7, vcc, v7, v8
1137; GFX7-NEXT:    s_mul_i32 s20, s2, s11
1138; GFX7-NEXT:    s_add_i32 s18, s18, s19
1139; GFX7-NEXT:    v_add_i32_e32 v4, vcc, v4, v10
1140; GFX7-NEXT:    s_add_u32 s17, s17, s20
1141; GFX7-NEXT:    s_cselect_b32 s19, 1, 0
1142; GFX7-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
1143; GFX7-NEXT:    v_add_i32_e32 v7, vcc, v7, v8
1144; GFX7-NEXT:    s_and_b32 s19, s19, 1
1145; GFX7-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
1146; GFX7-NEXT:    s_mul_i32 s21, s1, s12
1147; GFX7-NEXT:    s_add_i32 s18, s18, s19
1148; GFX7-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
1149; GFX7-NEXT:    s_add_u32 s17, s17, s21
1150; GFX7-NEXT:    s_cselect_b32 s19, 1, 0
1151; GFX7-NEXT:    v_add_i32_e32 v4, vcc, v7, v4
1152; GFX7-NEXT:    v_mov_b32_e32 v7, s4
1153; GFX7-NEXT:    s_and_b32 s19, s19, 1
1154; GFX7-NEXT:    v_mul_hi_u32 v8, v7, s8
1155; GFX7-NEXT:    s_mul_i32 s22, s0, s13
1156; GFX7-NEXT:    s_add_i32 s18, s18, s19
1157; GFX7-NEXT:    s_add_u32 s17, s17, s22
1158; GFX7-NEXT:    s_cselect_b32 s19, 1, 0
1159; GFX7-NEXT:    v_add_i32_e32 v8, vcc, s17, v8
1160; GFX7-NEXT:    s_and_b32 s19, s19, 1
1161; GFX7-NEXT:    v_mul_hi_u32 v10, v5, s9
1162; GFX7-NEXT:    s_add_i32 s18, s18, s19
1163; GFX7-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
1164; GFX7-NEXT:    v_add_i32_e32 v14, vcc, s18, v14
1165; GFX7-NEXT:    s_mul_i32 s17, s6, s8
1166; GFX7-NEXT:    s_mul_i32 s18, s5, s9
1167; GFX7-NEXT:    s_add_u32 s17, s17, s18
1168; GFX7-NEXT:    s_cselect_b32 s18, 1, 0
1169; GFX7-NEXT:    v_mul_hi_u32 v6, s2, v6
1170; GFX7-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
1171; GFX7-NEXT:    s_mul_i32 s19, s4, s10
1172; GFX7-NEXT:    s_and_b32 s18, s18, 1
1173; GFX7-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
1174; GFX7-NEXT:    s_add_u32 s17, s17, s19
1175; GFX7-NEXT:    s_cselect_b32 s19, 1, 0
1176; GFX7-NEXT:    v_add_i32_e32 v10, vcc, v14, v10
1177; GFX7-NEXT:    v_mul_hi_u32 v11, s1, v9
1178; GFX7-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
1179; GFX7-NEXT:    s_and_b32 s19, s19, 1
1180; GFX7-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
1181; GFX7-NEXT:    v_mov_b32_e32 v12, s12
1182; GFX7-NEXT:    s_mul_i32 s20, s3, s11
1183; GFX7-NEXT:    s_add_i32 s18, s18, s19
1184; GFX7-NEXT:    v_add_i32_e32 v8, vcc, v10, v8
1185; GFX7-NEXT:    s_add_u32 s17, s17, s20
1186; GFX7-NEXT:    v_mul_hi_u32 v13, s0, v12
1187; GFX7-NEXT:    s_cselect_b32 s19, 1, 0
1188; GFX7-NEXT:    v_add_i32_e32 v6, vcc, v6, v11
1189; GFX7-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
1190; GFX7-NEXT:    s_and_b32 s19, s19, 1
1191; GFX7-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
1192; GFX7-NEXT:    s_mul_i32 s21, s2, s12
1193; GFX7-NEXT:    s_add_i32 s18, s18, s19
1194; GFX7-NEXT:    v_add_i32_e32 v6, vcc, v6, v13
1195; GFX7-NEXT:    s_add_u32 s17, s17, s21
1196; GFX7-NEXT:    s_cselect_b32 s19, 1, 0
1197; GFX7-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
1198; GFX7-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
1199; GFX7-NEXT:    s_and_b32 s19, s19, 1
1200; GFX7-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
1201; GFX7-NEXT:    s_mul_i32 s22, s1, s13
1202; GFX7-NEXT:    s_add_i32 s18, s18, s19
1203; GFX7-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
1204; GFX7-NEXT:    s_add_u32 s17, s17, s22
1205; GFX7-NEXT:    s_cselect_b32 s19, 1, 0
1206; GFX7-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
1207; GFX7-NEXT:    v_mov_b32_e32 v8, s5
1208; GFX7-NEXT:    v_mul_hi_u32 v10, v8, s8
1209; GFX7-NEXT:    s_and_b32 s19, s19, 1
1210; GFX7-NEXT:    s_mul_i32 s23, s0, s14
1211; GFX7-NEXT:    s_add_i32 s18, s18, s19
1212; GFX7-NEXT:    s_add_u32 s17, s17, s23
1213; GFX7-NEXT:    s_cselect_b32 s19, 1, 0
1214; GFX7-NEXT:    v_mul_hi_u32 v11, v7, s9
1215; GFX7-NEXT:    v_add_i32_e32 v10, vcc, s17, v10
1216; GFX7-NEXT:    s_and_b32 s19, s19, 1
1217; GFX7-NEXT:    s_add_i32 s18, s18, s19
1218; GFX7-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
1219; GFX7-NEXT:    v_add_i32_e32 v17, vcc, s18, v17
1220; GFX7-NEXT:    v_mul_hi_u32 v5, v5, s10
1221; GFX7-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
1222; GFX7-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
1223; GFX7-NEXT:    v_mul_hi_u32 v13, s2, v9
1224; GFX7-NEXT:    v_add_i32_e32 v11, vcc, v17, v11
1225; GFX7-NEXT:    v_add_i32_e32 v5, vcc, v10, v5
1226; GFX7-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
1227; GFX7-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
1228; GFX7-NEXT:    v_mul_hi_u32 v14, s1, v12
1229; GFX7-NEXT:    v_add_i32_e32 v5, vcc, v5, v13
1230; GFX7-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
1231; GFX7-NEXT:    v_mov_b32_e32 v15, s13
1232; GFX7-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
1233; GFX7-NEXT:    v_mul_hi_u32 v16, s0, v15
1234; GFX7-NEXT:    v_add_i32_e32 v5, vcc, v5, v14
1235; GFX7-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
1236; GFX7-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
1237; GFX7-NEXT:    v_add_i32_e32 v5, vcc, v5, v16
1238; GFX7-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
1239; GFX7-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
1240; GFX7-NEXT:    v_mov_b32_e32 v13, s14
1241; GFX7-NEXT:    s_mul_i32 s7, s7, s8
1242; GFX7-NEXT:    s_mul_i32 s17, s6, s9
1243; GFX7-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
1244; GFX7-NEXT:    s_mul_i32 s16, s0, s8
1245; GFX7-NEXT:    s_mul_i32 s5, s5, s10
1246; GFX7-NEXT:    s_mul_i32 s15, s0, s15
1247; GFX7-NEXT:    v_mul_hi_u32 v13, s0, v13
1248; GFX7-NEXT:    s_add_i32 s0, s7, s17
1249; GFX7-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
1250; GFX7-NEXT:    s_mul_i32 s4, s4, s11
1251; GFX7-NEXT:    s_add_i32 s0, s0, s5
1252; GFX7-NEXT:    v_add_i32_e32 v6, vcc, v10, v6
1253; GFX7-NEXT:    v_mov_b32_e32 v10, s6
1254; GFX7-NEXT:    s_mul_i32 s11, s3, s12
1255; GFX7-NEXT:    s_add_i32 s0, s0, s4
1256; GFX7-NEXT:    s_mul_i32 s12, s2, s13
1257; GFX7-NEXT:    s_add_i32 s0, s0, s11
1258; GFX7-NEXT:    v_mul_hi_u32 v10, v10, s8
1259; GFX7-NEXT:    s_mul_i32 s13, s1, s14
1260; GFX7-NEXT:    s_add_i32 s0, s0, s12
1261; GFX7-NEXT:    v_mul_hi_u32 v8, v8, s9
1262; GFX7-NEXT:    s_add_i32 s0, s0, s13
1263; GFX7-NEXT:    v_mul_hi_u32 v7, v7, s10
1264; GFX7-NEXT:    v_mul_hi_u32 v9, s3, v9
1265; GFX7-NEXT:    s_add_i32 s0, s0, s15
1266; GFX7-NEXT:    v_mul_hi_u32 v11, s2, v12
1267; GFX7-NEXT:    v_add_i32_e32 v10, vcc, s0, v10
1268; GFX7-NEXT:    v_mul_hi_u32 v12, s1, v15
1269; GFX7-NEXT:    v_add_i32_e32 v8, vcc, v10, v8
1270; GFX7-NEXT:    v_add_i32_e32 v7, vcc, v8, v7
1271; GFX7-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
1272; GFX7-NEXT:    v_add_i32_e32 v7, vcc, v7, v11
1273; GFX7-NEXT:    v_add_i32_e32 v7, vcc, v7, v12
1274; GFX7-NEXT:    v_add_i32_e32 v7, vcc, v7, v13
1275; GFX7-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
1276; GFX7-NEXT:    v_readfirstlane_b32 s1, v0
1277; GFX7-NEXT:    v_readfirstlane_b32 s2, v1
1278; GFX7-NEXT:    v_readfirstlane_b32 s3, v2
1279; GFX7-NEXT:    v_readfirstlane_b32 s4, v3
1280; GFX7-NEXT:    v_readfirstlane_b32 s5, v4
1281; GFX7-NEXT:    v_readfirstlane_b32 s6, v5
1282; GFX7-NEXT:    v_readfirstlane_b32 s7, v6
1283; GFX7-NEXT:    s_mov_b32 s0, s16
1284; GFX7-NEXT:    ; return to shader part epilog
1285;
1286; GFX8-LABEL: s_mul_i256:
1287; GFX8:       ; %bb.0:
1288; GFX8-NEXT:    v_mov_b32_e32 v0, s8
1289; GFX8-NEXT:    v_mul_hi_u32 v0, s0, v0
1290; GFX8-NEXT:    s_mul_i32 s17, s1, s8
1291; GFX8-NEXT:    s_mul_i32 s18, s0, s9
1292; GFX8-NEXT:    s_add_u32 s17, s17, s18
1293; GFX8-NEXT:    s_cselect_b32 s18, 1, 0
1294; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s17, v0
1295; GFX8-NEXT:    s_and_b32 s18, s18, 1
1296; GFX8-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
1297; GFX8-NEXT:    v_add_u32_e32 v1, vcc, s18, v1
1298; GFX8-NEXT:    s_mul_i32 s17, s2, s8
1299; GFX8-NEXT:    s_mul_i32 s18, s1, s9
1300; GFX8-NEXT:    v_mov_b32_e32 v2, s1
1301; GFX8-NEXT:    s_add_u32 s17, s17, s18
1302; GFX8-NEXT:    s_cselect_b32 s18, 1, 0
1303; GFX8-NEXT:    v_mul_hi_u32 v2, v2, s8
1304; GFX8-NEXT:    s_mul_i32 s19, s0, s10
1305; GFX8-NEXT:    s_and_b32 s18, s18, 1
1306; GFX8-NEXT:    s_add_u32 s17, s17, s19
1307; GFX8-NEXT:    v_mov_b32_e32 v3, s9
1308; GFX8-NEXT:    s_cselect_b32 s19, 1, 0
1309; GFX8-NEXT:    v_mul_hi_u32 v4, s0, v3
1310; GFX8-NEXT:    v_add_u32_e32 v2, vcc, s17, v2
1311; GFX8-NEXT:    s_and_b32 s19, s19, 1
1312; GFX8-NEXT:    s_add_i32 s18, s18, s19
1313; GFX8-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
1314; GFX8-NEXT:    v_add_u32_e32 v5, vcc, s18, v5
1315; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v4
1316; GFX8-NEXT:    s_mul_i32 s17, s3, s8
1317; GFX8-NEXT:    s_mul_i32 s18, s2, s9
1318; GFX8-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
1319; GFX8-NEXT:    s_add_u32 s17, s17, s18
1320; GFX8-NEXT:    s_cselect_b32 s18, 1, 0
1321; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v5, v4
1322; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v2, v1
1323; GFX8-NEXT:    s_mul_i32 s19, s1, s10
1324; GFX8-NEXT:    s_and_b32 s18, s18, 1
1325; GFX8-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
1326; GFX8-NEXT:    s_add_u32 s17, s17, s19
1327; GFX8-NEXT:    s_cselect_b32 s19, 1, 0
1328; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v4, v2
1329; GFX8-NEXT:    v_mov_b32_e32 v4, s2
1330; GFX8-NEXT:    v_mul_hi_u32 v5, v4, s8
1331; GFX8-NEXT:    s_and_b32 s19, s19, 1
1332; GFX8-NEXT:    s_mul_i32 s20, s0, s11
1333; GFX8-NEXT:    s_add_i32 s18, s18, s19
1334; GFX8-NEXT:    s_add_u32 s17, s17, s20
1335; GFX8-NEXT:    s_cselect_b32 s19, 1, 0
1336; GFX8-NEXT:    v_mul_hi_u32 v3, s1, v3
1337; GFX8-NEXT:    v_add_u32_e32 v5, vcc, s17, v5
1338; GFX8-NEXT:    s_and_b32 s19, s19, 1
1339; GFX8-NEXT:    v_mov_b32_e32 v6, s10
1340; GFX8-NEXT:    s_add_i32 s18, s18, s19
1341; GFX8-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
1342; GFX8-NEXT:    v_add_u32_e32 v8, vcc, s18, v8
1343; GFX8-NEXT:    v_mul_hi_u32 v7, s0, v6
1344; GFX8-NEXT:    s_mul_i32 s17, s4, s8
1345; GFX8-NEXT:    s_mul_i32 s18, s3, s9
1346; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v5, v3
1347; GFX8-NEXT:    s_add_u32 s17, s17, s18
1348; GFX8-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
1349; GFX8-NEXT:    s_cselect_b32 s18, 1, 0
1350; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v8, v5
1351; GFX8-NEXT:    s_mul_i32 s19, s2, s10
1352; GFX8-NEXT:    s_and_b32 s18, s18, 1
1353; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v3, v7
1354; GFX8-NEXT:    s_add_u32 s17, s17, s19
1355; GFX8-NEXT:    s_cselect_b32 s19, 1, 0
1356; GFX8-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
1357; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v5, v7
1358; GFX8-NEXT:    s_and_b32 s19, s19, 1
1359; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v3, v2
1360; GFX8-NEXT:    s_mul_i32 s20, s1, s11
1361; GFX8-NEXT:    s_add_i32 s18, s18, s19
1362; GFX8-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
1363; GFX8-NEXT:    s_add_u32 s17, s17, s20
1364; GFX8-NEXT:    s_cselect_b32 s19, 1, 0
1365; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v5, v3
1366; GFX8-NEXT:    v_mov_b32_e32 v5, s3
1367; GFX8-NEXT:    s_and_b32 s19, s19, 1
1368; GFX8-NEXT:    v_mul_hi_u32 v7, v5, s8
1369; GFX8-NEXT:    s_mul_i32 s21, s0, s12
1370; GFX8-NEXT:    s_add_i32 s18, s18, s19
1371; GFX8-NEXT:    s_add_u32 s17, s17, s21
1372; GFX8-NEXT:    s_cselect_b32 s19, 1, 0
1373; GFX8-NEXT:    v_add_u32_e32 v7, vcc, s17, v7
1374; GFX8-NEXT:    s_and_b32 s19, s19, 1
1375; GFX8-NEXT:    v_mul_hi_u32 v4, v4, s9
1376; GFX8-NEXT:    s_add_i32 s18, s18, s19
1377; GFX8-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
1378; GFX8-NEXT:    v_add_u32_e32 v11, vcc, s18, v11
1379; GFX8-NEXT:    s_mul_i32 s17, s5, s8
1380; GFX8-NEXT:    s_mul_i32 s18, s4, s9
1381; GFX8-NEXT:    s_add_u32 s17, s17, s18
1382; GFX8-NEXT:    v_mul_hi_u32 v8, s1, v6
1383; GFX8-NEXT:    s_cselect_b32 s18, 1, 0
1384; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v7, v4
1385; GFX8-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
1386; GFX8-NEXT:    v_mov_b32_e32 v9, s11
1387; GFX8-NEXT:    s_mul_i32 s19, s3, s10
1388; GFX8-NEXT:    s_and_b32 s18, s18, 1
1389; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v11, v7
1390; GFX8-NEXT:    s_add_u32 s17, s17, s19
1391; GFX8-NEXT:    v_mul_hi_u32 v10, s0, v9
1392; GFX8-NEXT:    s_cselect_b32 s19, 1, 0
1393; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v4, v8
1394; GFX8-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
1395; GFX8-NEXT:    s_and_b32 s19, s19, 1
1396; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v7, v8
1397; GFX8-NEXT:    s_mul_i32 s20, s2, s11
1398; GFX8-NEXT:    s_add_i32 s18, s18, s19
1399; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v4, v10
1400; GFX8-NEXT:    s_add_u32 s17, s17, s20
1401; GFX8-NEXT:    s_cselect_b32 s19, 1, 0
1402; GFX8-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
1403; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v7, v8
1404; GFX8-NEXT:    s_and_b32 s19, s19, 1
1405; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v4, v3
1406; GFX8-NEXT:    s_mul_i32 s21, s1, s12
1407; GFX8-NEXT:    s_add_i32 s18, s18, s19
1408; GFX8-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
1409; GFX8-NEXT:    s_add_u32 s17, s17, s21
1410; GFX8-NEXT:    s_cselect_b32 s19, 1, 0
1411; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v7, v4
1412; GFX8-NEXT:    v_mov_b32_e32 v7, s4
1413; GFX8-NEXT:    s_and_b32 s19, s19, 1
1414; GFX8-NEXT:    v_mul_hi_u32 v8, v7, s8
1415; GFX8-NEXT:    s_mul_i32 s22, s0, s13
1416; GFX8-NEXT:    s_add_i32 s18, s18, s19
1417; GFX8-NEXT:    s_add_u32 s17, s17, s22
1418; GFX8-NEXT:    s_cselect_b32 s19, 1, 0
1419; GFX8-NEXT:    v_add_u32_e32 v8, vcc, s17, v8
1420; GFX8-NEXT:    s_and_b32 s19, s19, 1
1421; GFX8-NEXT:    v_mul_hi_u32 v10, v5, s9
1422; GFX8-NEXT:    s_add_i32 s18, s18, s19
1423; GFX8-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
1424; GFX8-NEXT:    v_add_u32_e32 v14, vcc, s18, v14
1425; GFX8-NEXT:    s_mul_i32 s17, s6, s8
1426; GFX8-NEXT:    s_mul_i32 s18, s5, s9
1427; GFX8-NEXT:    s_add_u32 s17, s17, s18
1428; GFX8-NEXT:    s_cselect_b32 s18, 1, 0
1429; GFX8-NEXT:    v_mul_hi_u32 v6, s2, v6
1430; GFX8-NEXT:    v_add_u32_e32 v8, vcc, v8, v10
1431; GFX8-NEXT:    s_mul_i32 s19, s4, s10
1432; GFX8-NEXT:    s_and_b32 s18, s18, 1
1433; GFX8-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
1434; GFX8-NEXT:    s_add_u32 s17, s17, s19
1435; GFX8-NEXT:    s_cselect_b32 s19, 1, 0
1436; GFX8-NEXT:    v_add_u32_e32 v10, vcc, v14, v10
1437; GFX8-NEXT:    v_mul_hi_u32 v11, s1, v9
1438; GFX8-NEXT:    v_add_u32_e32 v6, vcc, v8, v6
1439; GFX8-NEXT:    s_and_b32 s19, s19, 1
1440; GFX8-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
1441; GFX8-NEXT:    v_mov_b32_e32 v12, s12
1442; GFX8-NEXT:    s_mul_i32 s20, s3, s11
1443; GFX8-NEXT:    s_add_i32 s18, s18, s19
1444; GFX8-NEXT:    v_add_u32_e32 v8, vcc, v10, v8
1445; GFX8-NEXT:    s_add_u32 s17, s17, s20
1446; GFX8-NEXT:    v_mul_hi_u32 v13, s0, v12
1447; GFX8-NEXT:    s_cselect_b32 s19, 1, 0
1448; GFX8-NEXT:    v_add_u32_e32 v6, vcc, v6, v11
1449; GFX8-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
1450; GFX8-NEXT:    s_and_b32 s19, s19, 1
1451; GFX8-NEXT:    v_add_u32_e32 v8, vcc, v8, v10
1452; GFX8-NEXT:    s_mul_i32 s21, s2, s12
1453; GFX8-NEXT:    s_add_i32 s18, s18, s19
1454; GFX8-NEXT:    v_add_u32_e32 v6, vcc, v6, v13
1455; GFX8-NEXT:    s_add_u32 s17, s17, s21
1456; GFX8-NEXT:    s_cselect_b32 s19, 1, 0
1457; GFX8-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
1458; GFX8-NEXT:    v_add_u32_e32 v8, vcc, v8, v10
1459; GFX8-NEXT:    s_and_b32 s19, s19, 1
1460; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v6, v4
1461; GFX8-NEXT:    s_mul_i32 s22, s1, s13
1462; GFX8-NEXT:    s_add_i32 s18, s18, s19
1463; GFX8-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
1464; GFX8-NEXT:    s_add_u32 s17, s17, s22
1465; GFX8-NEXT:    s_cselect_b32 s19, 1, 0
1466; GFX8-NEXT:    v_add_u32_e32 v6, vcc, v8, v6
1467; GFX8-NEXT:    v_mov_b32_e32 v8, s5
1468; GFX8-NEXT:    v_mul_hi_u32 v10, v8, s8
1469; GFX8-NEXT:    s_and_b32 s19, s19, 1
1470; GFX8-NEXT:    s_mul_i32 s23, s0, s14
1471; GFX8-NEXT:    s_add_i32 s18, s18, s19
1472; GFX8-NEXT:    s_add_u32 s17, s17, s23
1473; GFX8-NEXT:    s_cselect_b32 s19, 1, 0
1474; GFX8-NEXT:    v_mul_hi_u32 v11, v7, s9
1475; GFX8-NEXT:    v_add_u32_e32 v10, vcc, s17, v10
1476; GFX8-NEXT:    s_and_b32 s19, s19, 1
1477; GFX8-NEXT:    s_add_i32 s18, s18, s19
1478; GFX8-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
1479; GFX8-NEXT:    v_add_u32_e32 v17, vcc, s18, v17
1480; GFX8-NEXT:    v_mul_hi_u32 v5, v5, s10
1481; GFX8-NEXT:    v_add_u32_e32 v10, vcc, v10, v11
1482; GFX8-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
1483; GFX8-NEXT:    v_mul_hi_u32 v13, s2, v9
1484; GFX8-NEXT:    v_add_u32_e32 v11, vcc, v17, v11
1485; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v10, v5
1486; GFX8-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
1487; GFX8-NEXT:    v_add_u32_e32 v10, vcc, v11, v10
1488; GFX8-NEXT:    v_mul_hi_u32 v14, s1, v12
1489; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v5, v13
1490; GFX8-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
1491; GFX8-NEXT:    v_mov_b32_e32 v15, s13
1492; GFX8-NEXT:    v_add_u32_e32 v10, vcc, v10, v11
1493; GFX8-NEXT:    v_mul_hi_u32 v16, s0, v15
1494; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v5, v14
1495; GFX8-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
1496; GFX8-NEXT:    v_add_u32_e32 v10, vcc, v10, v11
1497; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v5, v16
1498; GFX8-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
1499; GFX8-NEXT:    v_add_u32_e32 v10, vcc, v10, v11
1500; GFX8-NEXT:    v_mov_b32_e32 v13, s14
1501; GFX8-NEXT:    s_mul_i32 s7, s7, s8
1502; GFX8-NEXT:    s_mul_i32 s17, s6, s9
1503; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v5, v6
1504; GFX8-NEXT:    s_mul_i32 s16, s0, s8
1505; GFX8-NEXT:    s_mul_i32 s5, s5, s10
1506; GFX8-NEXT:    s_mul_i32 s15, s0, s15
1507; GFX8-NEXT:    v_mul_hi_u32 v13, s0, v13
1508; GFX8-NEXT:    s_add_i32 s0, s7, s17
1509; GFX8-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
1510; GFX8-NEXT:    s_mul_i32 s4, s4, s11
1511; GFX8-NEXT:    s_add_i32 s0, s0, s5
1512; GFX8-NEXT:    v_add_u32_e32 v6, vcc, v10, v6
1513; GFX8-NEXT:    v_mov_b32_e32 v10, s6
1514; GFX8-NEXT:    s_mul_i32 s11, s3, s12
1515; GFX8-NEXT:    s_add_i32 s0, s0, s4
1516; GFX8-NEXT:    s_mul_i32 s12, s2, s13
1517; GFX8-NEXT:    s_add_i32 s0, s0, s11
1518; GFX8-NEXT:    v_mul_hi_u32 v10, v10, s8
1519; GFX8-NEXT:    s_mul_i32 s13, s1, s14
1520; GFX8-NEXT:    s_add_i32 s0, s0, s12
1521; GFX8-NEXT:    v_mul_hi_u32 v8, v8, s9
1522; GFX8-NEXT:    s_add_i32 s0, s0, s13
1523; GFX8-NEXT:    v_mul_hi_u32 v7, v7, s10
1524; GFX8-NEXT:    v_mul_hi_u32 v9, s3, v9
1525; GFX8-NEXT:    s_add_i32 s0, s0, s15
1526; GFX8-NEXT:    v_mul_hi_u32 v11, s2, v12
1527; GFX8-NEXT:    v_add_u32_e32 v10, vcc, s0, v10
1528; GFX8-NEXT:    v_mul_hi_u32 v12, s1, v15
1529; GFX8-NEXT:    v_add_u32_e32 v8, vcc, v10, v8
1530; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v8, v7
1531; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v7, v9
1532; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v7, v11
1533; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v7, v12
1534; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v7, v13
1535; GFX8-NEXT:    v_add_u32_e32 v6, vcc, v7, v6
1536; GFX8-NEXT:    v_readfirstlane_b32 s1, v0
1537; GFX8-NEXT:    v_readfirstlane_b32 s2, v1
1538; GFX8-NEXT:    v_readfirstlane_b32 s3, v2
1539; GFX8-NEXT:    v_readfirstlane_b32 s4, v3
1540; GFX8-NEXT:    v_readfirstlane_b32 s5, v4
1541; GFX8-NEXT:    v_readfirstlane_b32 s6, v5
1542; GFX8-NEXT:    v_readfirstlane_b32 s7, v6
1543; GFX8-NEXT:    s_mov_b32 s0, s16
1544; GFX8-NEXT:    ; return to shader part epilog
1545;
1546; GFX9-LABEL: s_mul_i256:
1547; GFX9:       ; %bb.0:
1548; GFX9-NEXT:    s_mul_i32 s17, s1, s8
1549; GFX9-NEXT:    s_mul_i32 s18, s0, s9
1550; GFX9-NEXT:    s_add_u32 s17, s17, s18
1551; GFX9-NEXT:    s_cselect_b32 s18, 1, 0
1552; GFX9-NEXT:    s_mul_hi_u32 s19, s0, s8
1553; GFX9-NEXT:    s_and_b32 s18, s18, 1
1554; GFX9-NEXT:    s_add_u32 s17, s17, s19
1555; GFX9-NEXT:    s_cselect_b32 s19, 1, 0
1556; GFX9-NEXT:    s_and_b32 s19, s19, 1
1557; GFX9-NEXT:    s_add_i32 s18, s18, s19
1558; GFX9-NEXT:    s_mul_i32 s19, s2, s8
1559; GFX9-NEXT:    s_mul_i32 s20, s1, s9
1560; GFX9-NEXT:    s_add_u32 s19, s19, s20
1561; GFX9-NEXT:    s_cselect_b32 s20, 1, 0
1562; GFX9-NEXT:    s_mul_i32 s21, s0, s10
1563; GFX9-NEXT:    s_and_b32 s20, s20, 1
1564; GFX9-NEXT:    s_add_u32 s19, s19, s21
1565; GFX9-NEXT:    s_cselect_b32 s21, 1, 0
1566; GFX9-NEXT:    s_and_b32 s21, s21, 1
1567; GFX9-NEXT:    s_mul_hi_u32 s22, s1, s8
1568; GFX9-NEXT:    s_add_i32 s20, s20, s21
1569; GFX9-NEXT:    s_add_u32 s19, s19, s22
1570; GFX9-NEXT:    s_cselect_b32 s21, 1, 0
1571; GFX9-NEXT:    s_and_b32 s21, s21, 1
1572; GFX9-NEXT:    s_mul_hi_u32 s23, s0, s9
1573; GFX9-NEXT:    s_add_i32 s20, s20, s21
1574; GFX9-NEXT:    s_add_u32 s19, s19, s23
1575; GFX9-NEXT:    s_cselect_b32 s21, 1, 0
1576; GFX9-NEXT:    s_and_b32 s21, s21, 1
1577; GFX9-NEXT:    s_add_i32 s20, s20, s21
1578; GFX9-NEXT:    s_add_u32 s18, s19, s18
1579; GFX9-NEXT:    s_cselect_b32 s19, 1, 0
1580; GFX9-NEXT:    s_and_b32 s19, s19, 1
1581; GFX9-NEXT:    s_add_i32 s20, s20, s19
1582; GFX9-NEXT:    s_mul_i32 s19, s3, s8
1583; GFX9-NEXT:    s_mul_i32 s21, s2, s9
1584; GFX9-NEXT:    s_add_u32 s19, s19, s21
1585; GFX9-NEXT:    s_cselect_b32 s21, 1, 0
1586; GFX9-NEXT:    s_mul_i32 s22, s1, s10
1587; GFX9-NEXT:    s_and_b32 s21, s21, 1
1588; GFX9-NEXT:    s_add_u32 s19, s19, s22
1589; GFX9-NEXT:    s_cselect_b32 s22, 1, 0
1590; GFX9-NEXT:    s_and_b32 s22, s22, 1
1591; GFX9-NEXT:    s_mul_i32 s23, s0, s11
1592; GFX9-NEXT:    s_add_i32 s21, s21, s22
1593; GFX9-NEXT:    s_add_u32 s19, s19, s23
1594; GFX9-NEXT:    s_cselect_b32 s22, 1, 0
1595; GFX9-NEXT:    s_and_b32 s22, s22, 1
1596; GFX9-NEXT:    s_mul_hi_u32 s24, s2, s8
1597; GFX9-NEXT:    s_add_i32 s21, s21, s22
1598; GFX9-NEXT:    s_add_u32 s19, s19, s24
1599; GFX9-NEXT:    s_cselect_b32 s22, 1, 0
1600; GFX9-NEXT:    s_and_b32 s22, s22, 1
1601; GFX9-NEXT:    s_mul_hi_u32 s25, s1, s9
1602; GFX9-NEXT:    s_add_i32 s21, s21, s22
1603; GFX9-NEXT:    s_add_u32 s19, s19, s25
1604; GFX9-NEXT:    s_cselect_b32 s22, 1, 0
1605; GFX9-NEXT:    s_and_b32 s22, s22, 1
1606; GFX9-NEXT:    s_mul_hi_u32 s26, s0, s10
1607; GFX9-NEXT:    s_add_i32 s21, s21, s22
1608; GFX9-NEXT:    s_add_u32 s19, s19, s26
1609; GFX9-NEXT:    s_cselect_b32 s22, 1, 0
1610; GFX9-NEXT:    s_and_b32 s22, s22, 1
1611; GFX9-NEXT:    s_add_i32 s21, s21, s22
1612; GFX9-NEXT:    s_add_u32 s19, s19, s20
1613; GFX9-NEXT:    s_cselect_b32 s20, 1, 0
1614; GFX9-NEXT:    s_and_b32 s20, s20, 1
1615; GFX9-NEXT:    s_add_i32 s21, s21, s20
1616; GFX9-NEXT:    s_mul_i32 s20, s4, s8
1617; GFX9-NEXT:    s_mul_i32 s22, s3, s9
1618; GFX9-NEXT:    s_add_u32 s20, s20, s22
1619; GFX9-NEXT:    s_cselect_b32 s22, 1, 0
1620; GFX9-NEXT:    s_mul_i32 s23, s2, s10
1621; GFX9-NEXT:    s_and_b32 s22, s22, 1
1622; GFX9-NEXT:    s_add_u32 s20, s20, s23
1623; GFX9-NEXT:    s_cselect_b32 s23, 1, 0
1624; GFX9-NEXT:    s_and_b32 s23, s23, 1
1625; GFX9-NEXT:    s_mul_i32 s24, s1, s11
1626; GFX9-NEXT:    s_add_i32 s22, s22, s23
1627; GFX9-NEXT:    s_add_u32 s20, s20, s24
1628; GFX9-NEXT:    s_cselect_b32 s23, 1, 0
1629; GFX9-NEXT:    s_and_b32 s23, s23, 1
1630; GFX9-NEXT:    s_mul_i32 s25, s0, s12
1631; GFX9-NEXT:    s_add_i32 s22, s22, s23
1632; GFX9-NEXT:    s_add_u32 s20, s20, s25
1633; GFX9-NEXT:    s_cselect_b32 s23, 1, 0
1634; GFX9-NEXT:    s_and_b32 s23, s23, 1
1635; GFX9-NEXT:    s_mul_hi_u32 s26, s3, s8
1636; GFX9-NEXT:    s_add_i32 s22, s22, s23
1637; GFX9-NEXT:    s_add_u32 s20, s20, s26
1638; GFX9-NEXT:    s_cselect_b32 s23, 1, 0
1639; GFX9-NEXT:    s_and_b32 s23, s23, 1
1640; GFX9-NEXT:    s_mul_hi_u32 s27, s2, s9
1641; GFX9-NEXT:    s_add_i32 s22, s22, s23
1642; GFX9-NEXT:    s_add_u32 s20, s20, s27
1643; GFX9-NEXT:    s_cselect_b32 s23, 1, 0
1644; GFX9-NEXT:    s_and_b32 s23, s23, 1
1645; GFX9-NEXT:    s_mul_hi_u32 s28, s1, s10
1646; GFX9-NEXT:    s_add_i32 s22, s22, s23
1647; GFX9-NEXT:    s_add_u32 s20, s20, s28
1648; GFX9-NEXT:    s_cselect_b32 s23, 1, 0
1649; GFX9-NEXT:    s_and_b32 s23, s23, 1
1650; GFX9-NEXT:    s_mul_hi_u32 s29, s0, s11
1651; GFX9-NEXT:    s_add_i32 s22, s22, s23
1652; GFX9-NEXT:    s_add_u32 s20, s20, s29
1653; GFX9-NEXT:    s_cselect_b32 s23, 1, 0
1654; GFX9-NEXT:    s_and_b32 s23, s23, 1
1655; GFX9-NEXT:    s_add_i32 s22, s22, s23
1656; GFX9-NEXT:    s_add_u32 s20, s20, s21
1657; GFX9-NEXT:    s_cselect_b32 s21, 1, 0
1658; GFX9-NEXT:    s_and_b32 s21, s21, 1
1659; GFX9-NEXT:    s_add_i32 s22, s22, s21
1660; GFX9-NEXT:    s_mul_i32 s21, s5, s8
1661; GFX9-NEXT:    s_mul_i32 s23, s4, s9
1662; GFX9-NEXT:    s_add_u32 s21, s21, s23
1663; GFX9-NEXT:    s_cselect_b32 s23, 1, 0
1664; GFX9-NEXT:    s_mul_i32 s24, s3, s10
1665; GFX9-NEXT:    s_and_b32 s23, s23, 1
1666; GFX9-NEXT:    s_add_u32 s21, s21, s24
1667; GFX9-NEXT:    s_cselect_b32 s24, 1, 0
1668; GFX9-NEXT:    s_and_b32 s24, s24, 1
1669; GFX9-NEXT:    s_mul_i32 s25, s2, s11
1670; GFX9-NEXT:    s_add_i32 s23, s23, s24
1671; GFX9-NEXT:    s_add_u32 s21, s21, s25
1672; GFX9-NEXT:    s_cselect_b32 s24, 1, 0
1673; GFX9-NEXT:    s_and_b32 s24, s24, 1
1674; GFX9-NEXT:    s_mul_i32 s26, s1, s12
1675; GFX9-NEXT:    s_add_i32 s23, s23, s24
1676; GFX9-NEXT:    s_add_u32 s21, s21, s26
1677; GFX9-NEXT:    s_cselect_b32 s24, 1, 0
1678; GFX9-NEXT:    s_and_b32 s24, s24, 1
1679; GFX9-NEXT:    s_mul_i32 s27, s0, s13
1680; GFX9-NEXT:    s_add_i32 s23, s23, s24
1681; GFX9-NEXT:    s_add_u32 s21, s21, s27
1682; GFX9-NEXT:    s_cselect_b32 s24, 1, 0
1683; GFX9-NEXT:    s_and_b32 s24, s24, 1
1684; GFX9-NEXT:    s_mul_hi_u32 s28, s4, s8
1685; GFX9-NEXT:    s_add_i32 s23, s23, s24
1686; GFX9-NEXT:    s_add_u32 s21, s21, s28
1687; GFX9-NEXT:    s_cselect_b32 s24, 1, 0
1688; GFX9-NEXT:    s_and_b32 s24, s24, 1
1689; GFX9-NEXT:    s_mul_hi_u32 s29, s3, s9
1690; GFX9-NEXT:    s_add_i32 s23, s23, s24
1691; GFX9-NEXT:    s_add_u32 s21, s21, s29
1692; GFX9-NEXT:    s_cselect_b32 s24, 1, 0
1693; GFX9-NEXT:    s_and_b32 s24, s24, 1
1694; GFX9-NEXT:    s_mul_hi_u32 s30, s2, s10
1695; GFX9-NEXT:    s_add_i32 s23, s23, s24
1696; GFX9-NEXT:    s_add_u32 s21, s21, s30
1697; GFX9-NEXT:    s_cselect_b32 s24, 1, 0
1698; GFX9-NEXT:    s_and_b32 s24, s24, 1
1699; GFX9-NEXT:    s_mul_hi_u32 s31, s1, s11
1700; GFX9-NEXT:    s_add_i32 s23, s23, s24
1701; GFX9-NEXT:    s_add_u32 s21, s21, s31
1702; GFX9-NEXT:    s_cselect_b32 s24, 1, 0
1703; GFX9-NEXT:    s_and_b32 s24, s24, 1
1704; GFX9-NEXT:    s_mul_hi_u32 s33, s0, s12
1705; GFX9-NEXT:    s_add_i32 s23, s23, s24
1706; GFX9-NEXT:    s_add_u32 s21, s21, s33
1707; GFX9-NEXT:    s_cselect_b32 s24, 1, 0
1708; GFX9-NEXT:    s_and_b32 s24, s24, 1
1709; GFX9-NEXT:    s_add_i32 s23, s23, s24
1710; GFX9-NEXT:    s_add_u32 s21, s21, s22
1711; GFX9-NEXT:    s_cselect_b32 s22, 1, 0
1712; GFX9-NEXT:    s_and_b32 s22, s22, 1
1713; GFX9-NEXT:    s_add_i32 s23, s23, s22
1714; GFX9-NEXT:    s_mul_i32 s22, s6, s8
1715; GFX9-NEXT:    s_mul_i32 s24, s5, s9
1716; GFX9-NEXT:    s_add_u32 s22, s22, s24
1717; GFX9-NEXT:    s_cselect_b32 s24, 1, 0
1718; GFX9-NEXT:    s_mul_i32 s25, s4, s10
1719; GFX9-NEXT:    s_and_b32 s24, s24, 1
1720; GFX9-NEXT:    s_add_u32 s22, s22, s25
1721; GFX9-NEXT:    s_cselect_b32 s25, 1, 0
1722; GFX9-NEXT:    s_and_b32 s25, s25, 1
1723; GFX9-NEXT:    s_mul_i32 s26, s3, s11
1724; GFX9-NEXT:    s_add_i32 s24, s24, s25
1725; GFX9-NEXT:    s_add_u32 s22, s22, s26
1726; GFX9-NEXT:    s_cselect_b32 s25, 1, 0
1727; GFX9-NEXT:    s_and_b32 s25, s25, 1
1728; GFX9-NEXT:    s_mul_i32 s27, s2, s12
1729; GFX9-NEXT:    s_add_i32 s24, s24, s25
1730; GFX9-NEXT:    s_add_u32 s22, s22, s27
1731; GFX9-NEXT:    s_cselect_b32 s25, 1, 0
1732; GFX9-NEXT:    s_and_b32 s25, s25, 1
1733; GFX9-NEXT:    s_mul_i32 s28, s1, s13
1734; GFX9-NEXT:    s_add_i32 s24, s24, s25
1735; GFX9-NEXT:    s_add_u32 s22, s22, s28
1736; GFX9-NEXT:    s_cselect_b32 s25, 1, 0
1737; GFX9-NEXT:    s_and_b32 s25, s25, 1
1738; GFX9-NEXT:    s_mul_i32 s29, s0, s14
1739; GFX9-NEXT:    s_add_i32 s24, s24, s25
1740; GFX9-NEXT:    s_add_u32 s22, s22, s29
1741; GFX9-NEXT:    s_cselect_b32 s25, 1, 0
1742; GFX9-NEXT:    s_and_b32 s25, s25, 1
1743; GFX9-NEXT:    s_mul_hi_u32 s30, s5, s8
1744; GFX9-NEXT:    s_add_i32 s24, s24, s25
1745; GFX9-NEXT:    s_add_u32 s22, s22, s30
1746; GFX9-NEXT:    s_cselect_b32 s25, 1, 0
1747; GFX9-NEXT:    s_and_b32 s25, s25, 1
1748; GFX9-NEXT:    s_mul_hi_u32 s31, s4, s9
1749; GFX9-NEXT:    s_add_i32 s24, s24, s25
1750; GFX9-NEXT:    s_add_u32 s22, s22, s31
1751; GFX9-NEXT:    s_cselect_b32 s25, 1, 0
1752; GFX9-NEXT:    s_and_b32 s25, s25, 1
1753; GFX9-NEXT:    s_mul_hi_u32 s33, s3, s10
1754; GFX9-NEXT:    s_add_i32 s24, s24, s25
1755; GFX9-NEXT:    s_add_u32 s22, s22, s33
1756; GFX9-NEXT:    s_cselect_b32 s25, 1, 0
1757; GFX9-NEXT:    s_and_b32 s25, s25, 1
1758; GFX9-NEXT:    s_mul_hi_u32 s34, s2, s11
1759; GFX9-NEXT:    s_add_i32 s24, s24, s25
1760; GFX9-NEXT:    s_add_u32 s22, s22, s34
1761; GFX9-NEXT:    s_cselect_b32 s25, 1, 0
1762; GFX9-NEXT:    s_and_b32 s25, s25, 1
1763; GFX9-NEXT:    s_mul_hi_u32 s35, s1, s12
1764; GFX9-NEXT:    s_add_i32 s24, s24, s25
1765; GFX9-NEXT:    s_add_u32 s22, s22, s35
1766; GFX9-NEXT:    s_cselect_b32 s25, 1, 0
1767; GFX9-NEXT:    s_and_b32 s25, s25, 1
1768; GFX9-NEXT:    s_mul_hi_u32 s36, s0, s13
1769; GFX9-NEXT:    s_add_i32 s24, s24, s25
1770; GFX9-NEXT:    s_add_u32 s22, s22, s36
1771; GFX9-NEXT:    s_cselect_b32 s25, 1, 0
1772; GFX9-NEXT:    s_and_b32 s25, s25, 1
1773; GFX9-NEXT:    s_add_i32 s24, s24, s25
1774; GFX9-NEXT:    s_add_u32 s22, s22, s23
1775; GFX9-NEXT:    s_cselect_b32 s23, 1, 0
1776; GFX9-NEXT:    s_and_b32 s23, s23, 1
1777; GFX9-NEXT:    s_add_i32 s24, s24, s23
1778; GFX9-NEXT:    s_mul_i32 s23, s6, s9
1779; GFX9-NEXT:    s_mul_i32 s7, s7, s8
1780; GFX9-NEXT:    s_mul_i32 s25, s5, s10
1781; GFX9-NEXT:    s_add_i32 s7, s7, s23
1782; GFX9-NEXT:    s_mul_i32 s26, s4, s11
1783; GFX9-NEXT:    s_add_i32 s7, s7, s25
1784; GFX9-NEXT:    s_mul_i32 s27, s3, s12
1785; GFX9-NEXT:    s_add_i32 s7, s7, s26
1786; GFX9-NEXT:    s_mul_i32 s28, s2, s13
1787; GFX9-NEXT:    s_add_i32 s7, s7, s27
1788; GFX9-NEXT:    s_mul_i32 s29, s1, s14
1789; GFX9-NEXT:    s_add_i32 s7, s7, s28
1790; GFX9-NEXT:    s_mul_i32 s15, s0, s15
1791; GFX9-NEXT:    s_add_i32 s7, s7, s29
1792; GFX9-NEXT:    s_mul_hi_u32 s6, s6, s8
1793; GFX9-NEXT:    s_add_i32 s7, s7, s15
1794; GFX9-NEXT:    s_mul_hi_u32 s5, s5, s9
1795; GFX9-NEXT:    s_add_i32 s6, s7, s6
1796; GFX9-NEXT:    s_add_i32 s5, s6, s5
1797; GFX9-NEXT:    s_mul_hi_u32 s4, s4, s10
1798; GFX9-NEXT:    s_add_i32 s4, s5, s4
1799; GFX9-NEXT:    s_mul_hi_u32 s3, s3, s11
1800; GFX9-NEXT:    s_add_i32 s3, s4, s3
1801; GFX9-NEXT:    s_mul_hi_u32 s2, s2, s12
1802; GFX9-NEXT:    s_add_i32 s2, s3, s2
1803; GFX9-NEXT:    s_mul_hi_u32 s1, s1, s13
1804; GFX9-NEXT:    s_mul_i32 s16, s0, s8
1805; GFX9-NEXT:    s_add_i32 s1, s2, s1
1806; GFX9-NEXT:    s_mul_hi_u32 s0, s0, s14
1807; GFX9-NEXT:    s_add_i32 s0, s1, s0
1808; GFX9-NEXT:    s_add_i32 s7, s0, s24
1809; GFX9-NEXT:    s_mov_b32 s0, s16
1810; GFX9-NEXT:    s_mov_b32 s1, s17
1811; GFX9-NEXT:    s_mov_b32 s2, s18
1812; GFX9-NEXT:    s_mov_b32 s3, s19
1813; GFX9-NEXT:    s_mov_b32 s4, s20
1814; GFX9-NEXT:    s_mov_b32 s5, s21
1815; GFX9-NEXT:    s_mov_b32 s6, s22
1816; GFX9-NEXT:    ; return to shader part epilog
1817;
1818; GFX10-LABEL: s_mul_i256:
1819; GFX10:       ; %bb.0:
1820; GFX10-NEXT:    s_mul_i32 s16, s1, s8
1821; GFX10-NEXT:    s_mul_i32 s17, s0, s9
1822; GFX10-NEXT:    s_mul_hi_u32 s18, s0, s8
1823; GFX10-NEXT:    s_add_u32 s16, s16, s17
1824; GFX10-NEXT:    s_cselect_b32 s17, 1, 0
1825; GFX10-NEXT:    s_mul_i32 s19, s1, s9
1826; GFX10-NEXT:    s_and_b32 s17, s17, 1
1827; GFX10-NEXT:    s_add_u32 s16, s16, s18
1828; GFX10-NEXT:    s_cselect_b32 s18, 1, 0
1829; GFX10-NEXT:    s_mul_i32 s20, s0, s10
1830; GFX10-NEXT:    s_and_b32 s18, s18, 1
1831; GFX10-NEXT:    s_mul_hi_u32 s21, s1, s8
1832; GFX10-NEXT:    s_add_i32 s17, s17, s18
1833; GFX10-NEXT:    s_mul_i32 s18, s2, s8
1834; GFX10-NEXT:    s_mul_i32 s22, s0, s11
1835; GFX10-NEXT:    s_add_u32 s18, s18, s19
1836; GFX10-NEXT:    s_cselect_b32 s19, 1, 0
1837; GFX10-NEXT:    s_mul_i32 s23, s1, s11
1838; GFX10-NEXT:    s_and_b32 s19, s19, 1
1839; GFX10-NEXT:    s_add_u32 s18, s18, s20
1840; GFX10-NEXT:    s_cselect_b32 s20, 1, 0
1841; GFX10-NEXT:    s_mul_i32 s24, s0, s12
1842; GFX10-NEXT:    s_and_b32 s20, s20, 1
1843; GFX10-NEXT:    s_mul_i32 s25, s4, s9
1844; GFX10-NEXT:    s_add_i32 s19, s19, s20
1845; GFX10-NEXT:    s_add_u32 s18, s18, s21
1846; GFX10-NEXT:    s_cselect_b32 s20, 1, 0
1847; GFX10-NEXT:    s_mul_hi_u32 s21, s0, s9
1848; GFX10-NEXT:    s_and_b32 s20, s20, 1
1849; GFX10-NEXT:    s_mul_i32 s26, s2, s11
1850; GFX10-NEXT:    s_add_i32 s19, s19, s20
1851; GFX10-NEXT:    s_add_u32 s18, s18, s21
1852; GFX10-NEXT:    s_cselect_b32 s20, 1, 0
1853; GFX10-NEXT:    s_mul_i32 s21, s1, s10
1854; GFX10-NEXT:    s_and_b32 s20, s20, 1
1855; GFX10-NEXT:    s_mul_i32 s27, s0, s13
1856; GFX10-NEXT:    s_add_i32 s19, s19, s20
1857; GFX10-NEXT:    s_add_u32 s17, s18, s17
1858; GFX10-NEXT:    s_cselect_b32 s18, 1, 0
1859; GFX10-NEXT:    s_mul_i32 s20, s2, s9
1860; GFX10-NEXT:    s_and_b32 s18, s18, 1
1861; GFX10-NEXT:    s_mul_hi_u32 s28, s3, s9
1862; GFX10-NEXT:    s_add_i32 s19, s19, s18
1863; GFX10-NEXT:    s_mul_i32 s18, s3, s8
1864; GFX10-NEXT:    s_mul_i32 s7, s7, s8
1865; GFX10-NEXT:    s_add_u32 s18, s18, s20
1866; GFX10-NEXT:    s_cselect_b32 s20, 1, 0
1867; GFX10-NEXT:    s_mul_i32 s15, s0, s15
1868; GFX10-NEXT:    s_and_b32 s20, s20, 1
1869; GFX10-NEXT:    s_add_u32 s18, s18, s21
1870; GFX10-NEXT:    s_cselect_b32 s21, 1, 0
1871; GFX10-NEXT:    s_and_b32 s21, s21, 1
1872; GFX10-NEXT:    s_add_i32 s20, s20, s21
1873; GFX10-NEXT:    s_add_u32 s18, s18, s22
1874; GFX10-NEXT:    s_cselect_b32 s21, 1, 0
1875; GFX10-NEXT:    s_mul_hi_u32 s22, s2, s8
1876; GFX10-NEXT:    s_and_b32 s21, s21, 1
1877; GFX10-NEXT:    s_add_i32 s20, s20, s21
1878; GFX10-NEXT:    s_add_u32 s18, s18, s22
1879; GFX10-NEXT:    s_cselect_b32 s21, 1, 0
1880; GFX10-NEXT:    s_mul_hi_u32 s22, s1, s9
1881; GFX10-NEXT:    s_and_b32 s21, s21, 1
1882; GFX10-NEXT:    s_add_i32 s20, s20, s21
1883; GFX10-NEXT:    s_add_u32 s18, s18, s22
1884; GFX10-NEXT:    s_cselect_b32 s21, 1, 0
1885; GFX10-NEXT:    s_mul_hi_u32 s22, s0, s10
1886; GFX10-NEXT:    s_and_b32 s21, s21, 1
1887; GFX10-NEXT:    s_add_i32 s20, s20, s21
1888; GFX10-NEXT:    s_add_u32 s18, s18, s22
1889; GFX10-NEXT:    s_cselect_b32 s21, 1, 0
1890; GFX10-NEXT:    s_mul_i32 s22, s2, s10
1891; GFX10-NEXT:    s_and_b32 s21, s21, 1
1892; GFX10-NEXT:    s_add_i32 s20, s20, s21
1893; GFX10-NEXT:    s_add_u32 s18, s18, s19
1894; GFX10-NEXT:    s_cselect_b32 s19, 1, 0
1895; GFX10-NEXT:    s_mul_i32 s21, s3, s9
1896; GFX10-NEXT:    s_and_b32 s19, s19, 1
1897; GFX10-NEXT:    s_add_i32 s20, s20, s19
1898; GFX10-NEXT:    s_mul_i32 s19, s4, s8
1899; GFX10-NEXT:    s_add_u32 s19, s19, s21
1900; GFX10-NEXT:    s_cselect_b32 s21, 1, 0
1901; GFX10-NEXT:    s_and_b32 s21, s21, 1
1902; GFX10-NEXT:    s_add_u32 s19, s19, s22
1903; GFX10-NEXT:    s_cselect_b32 s22, 1, 0
1904; GFX10-NEXT:    s_and_b32 s22, s22, 1
1905; GFX10-NEXT:    s_add_i32 s21, s21, s22
1906; GFX10-NEXT:    s_add_u32 s19, s19, s23
1907; GFX10-NEXT:    s_cselect_b32 s22, 1, 0
1908; GFX10-NEXT:    s_mul_hi_u32 s23, s3, s8
1909; GFX10-NEXT:    s_and_b32 s22, s22, 1
1910; GFX10-NEXT:    s_add_i32 s21, s21, s22
1911; GFX10-NEXT:    s_add_u32 s19, s19, s24
1912; GFX10-NEXT:    s_cselect_b32 s22, 1, 0
1913; GFX10-NEXT:    s_mul_hi_u32 s24, s2, s9
1914; GFX10-NEXT:    s_and_b32 s22, s22, 1
1915; GFX10-NEXT:    s_add_i32 s21, s21, s22
1916; GFX10-NEXT:    s_add_u32 s19, s19, s23
1917; GFX10-NEXT:    s_cselect_b32 s22, 1, 0
1918; GFX10-NEXT:    s_mul_hi_u32 s23, s1, s10
1919; GFX10-NEXT:    s_and_b32 s22, s22, 1
1920; GFX10-NEXT:    s_add_i32 s21, s21, s22
1921; GFX10-NEXT:    s_add_u32 s19, s19, s24
1922; GFX10-NEXT:    s_cselect_b32 s22, 1, 0
1923; GFX10-NEXT:    s_mul_hi_u32 s24, s0, s11
1924; GFX10-NEXT:    s_and_b32 s22, s22, 1
1925; GFX10-NEXT:    s_add_i32 s21, s21, s22
1926; GFX10-NEXT:    s_add_u32 s19, s19, s23
1927; GFX10-NEXT:    s_cselect_b32 s22, 1, 0
1928; GFX10-NEXT:    s_mul_i32 s23, s5, s8
1929; GFX10-NEXT:    s_and_b32 s22, s22, 1
1930; GFX10-NEXT:    s_add_i32 s21, s21, s22
1931; GFX10-NEXT:    s_add_u32 s19, s19, s24
1932; GFX10-NEXT:    s_cselect_b32 s22, 1, 0
1933; GFX10-NEXT:    s_mul_i32 s24, s3, s10
1934; GFX10-NEXT:    s_and_b32 s22, s22, 1
1935; GFX10-NEXT:    s_add_i32 s21, s21, s22
1936; GFX10-NEXT:    s_add_u32 s19, s19, s20
1937; GFX10-NEXT:    s_cselect_b32 s20, 1, 0
1938; GFX10-NEXT:    s_mul_i32 s22, s1, s12
1939; GFX10-NEXT:    s_and_b32 s20, s20, 1
1940; GFX10-NEXT:    s_add_i32 s21, s21, s20
1941; GFX10-NEXT:    s_add_u32 s23, s23, s25
1942; GFX10-NEXT:    s_cselect_b32 s25, 1, 0
1943; GFX10-NEXT:    s_mul_hi_u32 s20, s4, s8
1944; GFX10-NEXT:    s_and_b32 s25, s25, 1
1945; GFX10-NEXT:    s_add_u32 s23, s23, s24
1946; GFX10-NEXT:    s_cselect_b32 s24, 1, 0
1947; GFX10-NEXT:    s_and_b32 s24, s24, 1
1948; GFX10-NEXT:    s_add_i32 s24, s25, s24
1949; GFX10-NEXT:    s_add_u32 s23, s23, s26
1950; GFX10-NEXT:    s_cselect_b32 s25, 1, 0
1951; GFX10-NEXT:    s_mul_hi_u32 s26, s2, s10
1952; GFX10-NEXT:    s_and_b32 s25, s25, 1
1953; GFX10-NEXT:    s_add_i32 s24, s24, s25
1954; GFX10-NEXT:    s_add_u32 s22, s23, s22
1955; GFX10-NEXT:    s_cselect_b32 s23, 1, 0
1956; GFX10-NEXT:    s_mul_hi_u32 s25, s1, s11
1957; GFX10-NEXT:    s_and_b32 s23, s23, 1
1958; GFX10-NEXT:    s_add_i32 s23, s24, s23
1959; GFX10-NEXT:    s_add_u32 s22, s22, s27
1960; GFX10-NEXT:    s_cselect_b32 s24, 1, 0
1961; GFX10-NEXT:    s_mul_hi_u32 s27, s0, s12
1962; GFX10-NEXT:    s_and_b32 s24, s24, 1
1963; GFX10-NEXT:    s_add_i32 s23, s23, s24
1964; GFX10-NEXT:    s_add_u32 s20, s22, s20
1965; GFX10-NEXT:    s_cselect_b32 s22, 1, 0
1966; GFX10-NEXT:    s_mul_i32 s24, s6, s8
1967; GFX10-NEXT:    s_and_b32 s22, s22, 1
1968; GFX10-NEXT:    s_add_i32 s22, s23, s22
1969; GFX10-NEXT:    s_add_u32 s20, s20, s28
1970; GFX10-NEXT:    s_cselect_b32 s23, 1, 0
1971; GFX10-NEXT:    s_mul_i32 s28, s5, s9
1972; GFX10-NEXT:    s_and_b32 s23, s23, 1
1973; GFX10-NEXT:    s_add_i32 s22, s22, s23
1974; GFX10-NEXT:    s_add_u32 s20, s20, s26
1975; GFX10-NEXT:    s_cselect_b32 s23, 1, 0
1976; GFX10-NEXT:    s_mul_i32 s26, s4, s10
1977; GFX10-NEXT:    s_and_b32 s23, s23, 1
1978; GFX10-NEXT:    s_add_i32 s22, s22, s23
1979; GFX10-NEXT:    s_add_u32 s20, s20, s25
1980; GFX10-NEXT:    s_cselect_b32 s23, 1, 0
1981; GFX10-NEXT:    s_mul_i32 s25, s3, s11
1982; GFX10-NEXT:    s_and_b32 s23, s23, 1
1983; GFX10-NEXT:    s_add_i32 s22, s22, s23
1984; GFX10-NEXT:    s_add_u32 s20, s20, s27
1985; GFX10-NEXT:    s_cselect_b32 s23, 1, 0
1986; GFX10-NEXT:    s_mul_i32 s27, s2, s12
1987; GFX10-NEXT:    s_and_b32 s23, s23, 1
1988; GFX10-NEXT:    s_add_i32 s22, s22, s23
1989; GFX10-NEXT:    s_add_u32 s20, s20, s21
1990; GFX10-NEXT:    s_cselect_b32 s21, 1, 0
1991; GFX10-NEXT:    s_mul_i32 s23, s1, s13
1992; GFX10-NEXT:    s_and_b32 s21, s21, 1
1993; GFX10-NEXT:    s_add_i32 s22, s22, s21
1994; GFX10-NEXT:    s_add_u32 s21, s24, s28
1995; GFX10-NEXT:    s_cselect_b32 s24, 1, 0
1996; GFX10-NEXT:    s_mul_i32 s28, s0, s14
1997; GFX10-NEXT:    s_and_b32 s24, s24, 1
1998; GFX10-NEXT:    s_add_u32 s21, s21, s26
1999; GFX10-NEXT:    s_cselect_b32 s26, 1, 0
2000; GFX10-NEXT:    s_and_b32 s26, s26, 1
2001; GFX10-NEXT:    s_add_i32 s24, s24, s26
2002; GFX10-NEXT:    s_add_u32 s21, s21, s25
2003; GFX10-NEXT:    s_cselect_b32 s25, 1, 0
2004; GFX10-NEXT:    s_mul_hi_u32 s26, s5, s8
2005; GFX10-NEXT:    s_and_b32 s25, s25, 1
2006; GFX10-NEXT:    s_add_i32 s24, s24, s25
2007; GFX10-NEXT:    s_add_u32 s21, s21, s27
2008; GFX10-NEXT:    s_cselect_b32 s25, 1, 0
2009; GFX10-NEXT:    s_mul_hi_u32 s27, s4, s9
2010; GFX10-NEXT:    s_and_b32 s25, s25, 1
2011; GFX10-NEXT:    s_add_i32 s24, s24, s25
2012; GFX10-NEXT:    s_add_u32 s21, s21, s23
2013; GFX10-NEXT:    s_cselect_b32 s23, 1, 0
2014; GFX10-NEXT:    s_mul_hi_u32 s25, s3, s10
2015; GFX10-NEXT:    s_and_b32 s23, s23, 1
2016; GFX10-NEXT:    s_add_i32 s23, s24, s23
2017; GFX10-NEXT:    s_add_u32 s21, s21, s28
2018; GFX10-NEXT:    s_cselect_b32 s24, 1, 0
2019; GFX10-NEXT:    s_mul_hi_u32 s28, s2, s11
2020; GFX10-NEXT:    s_and_b32 s24, s24, 1
2021; GFX10-NEXT:    s_add_i32 s23, s23, s24
2022; GFX10-NEXT:    s_add_u32 s21, s21, s26
2023; GFX10-NEXT:    s_cselect_b32 s24, 1, 0
2024; GFX10-NEXT:    s_mul_hi_u32 s26, s1, s12
2025; GFX10-NEXT:    s_and_b32 s24, s24, 1
2026; GFX10-NEXT:    s_add_i32 s23, s23, s24
2027; GFX10-NEXT:    s_add_u32 s21, s21, s27
2028; GFX10-NEXT:    s_cselect_b32 s24, 1, 0
2029; GFX10-NEXT:    s_mul_hi_u32 s27, s0, s13
2030; GFX10-NEXT:    s_and_b32 s24, s24, 1
2031; GFX10-NEXT:    s_add_i32 s23, s23, s24
2032; GFX10-NEXT:    s_add_u32 s21, s21, s25
2033; GFX10-NEXT:    s_cselect_b32 s24, 1, 0
2034; GFX10-NEXT:    s_mul_i32 s25, s6, s9
2035; GFX10-NEXT:    s_and_b32 s24, s24, 1
2036; GFX10-NEXT:    s_mul_hi_u32 s6, s6, s8
2037; GFX10-NEXT:    s_add_i32 s23, s23, s24
2038; GFX10-NEXT:    s_add_u32 s21, s21, s28
2039; GFX10-NEXT:    s_cselect_b32 s24, 1, 0
2040; GFX10-NEXT:    s_and_b32 s24, s24, 1
2041; GFX10-NEXT:    s_add_i32 s23, s23, s24
2042; GFX10-NEXT:    s_add_u32 s21, s21, s26
2043; GFX10-NEXT:    s_cselect_b32 s24, 1, 0
2044; GFX10-NEXT:    s_mul_i32 s26, s5, s10
2045; GFX10-NEXT:    s_and_b32 s24, s24, 1
2046; GFX10-NEXT:    s_mul_hi_u32 s5, s5, s9
2047; GFX10-NEXT:    s_add_i32 s23, s23, s24
2048; GFX10-NEXT:    s_add_u32 s21, s21, s27
2049; GFX10-NEXT:    s_cselect_b32 s24, 1, 0
2050; GFX10-NEXT:    s_mul_i32 s27, s4, s11
2051; GFX10-NEXT:    s_and_b32 s24, s24, 1
2052; GFX10-NEXT:    s_mul_hi_u32 s4, s4, s10
2053; GFX10-NEXT:    s_add_i32 s23, s23, s24
2054; GFX10-NEXT:    s_add_u32 s21, s21, s22
2055; GFX10-NEXT:    s_cselect_b32 s22, 1, 0
2056; GFX10-NEXT:    s_add_i32 s7, s7, s25
2057; GFX10-NEXT:    s_mul_i32 s24, s3, s12
2058; GFX10-NEXT:    s_add_i32 s7, s7, s26
2059; GFX10-NEXT:    s_mul_i32 s25, s2, s13
2060; GFX10-NEXT:    s_add_i32 s7, s7, s27
2061; GFX10-NEXT:    s_mul_i32 s26, s1, s14
2062; GFX10-NEXT:    s_add_i32 s7, s7, s24
2063; GFX10-NEXT:    s_mul_hi_u32 s3, s3, s11
2064; GFX10-NEXT:    s_add_i32 s7, s7, s25
2065; GFX10-NEXT:    s_mul_hi_u32 s2, s2, s12
2066; GFX10-NEXT:    s_add_i32 s7, s7, s26
2067; GFX10-NEXT:    s_mul_hi_u32 s1, s1, s13
2068; GFX10-NEXT:    s_add_i32 s7, s7, s15
2069; GFX10-NEXT:    s_add_i32 s6, s7, s6
2070; GFX10-NEXT:    s_add_i32 s5, s6, s5
2071; GFX10-NEXT:    s_mov_b32 s6, s21
2072; GFX10-NEXT:    s_add_i32 s4, s5, s4
2073; GFX10-NEXT:    s_mov_b32 s5, s20
2074; GFX10-NEXT:    s_add_i32 s3, s4, s3
2075; GFX10-NEXT:    s_mul_hi_u32 s4, s0, s14
2076; GFX10-NEXT:    s_add_i32 s2, s3, s2
2077; GFX10-NEXT:    s_and_b32 s3, s22, 1
2078; GFX10-NEXT:    s_add_i32 s1, s2, s1
2079; GFX10-NEXT:    s_add_i32 s23, s23, s3
2080; GFX10-NEXT:    s_add_i32 s1, s1, s4
2081; GFX10-NEXT:    s_mul_i32 s0, s0, s8
2082; GFX10-NEXT:    s_add_i32 s7, s1, s23
2083; GFX10-NEXT:    s_mov_b32 s1, s16
2084; GFX10-NEXT:    s_mov_b32 s2, s17
2085; GFX10-NEXT:    s_mov_b32 s3, s18
2086; GFX10-NEXT:    s_mov_b32 s4, s19
2087; GFX10-NEXT:    ; return to shader part epilog
2088  %result = mul i256 %num, %den
2089  %cast = bitcast i256 %result to <8 x i32>
2090  ret <8 x i32> %cast
2091}
2092
2093define i256 @v_mul_i256(i256 %num, i256 %den) {
2094; GFX7-LABEL: v_mul_i256:
2095; GFX7:       ; %bb.0:
2096; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2097; GFX7-NEXT:    v_mul_lo_u32 v16, v1, v8
2098; GFX7-NEXT:    v_mul_lo_u32 v17, v0, v9
2099; GFX7-NEXT:    v_mul_hi_u32 v18, v0, v8
2100; GFX7-NEXT:    v_mul_lo_u32 v19, v2, v8
2101; GFX7-NEXT:    v_mul_lo_u32 v20, v1, v9
2102; GFX7-NEXT:    v_add_i32_e32 v16, vcc, v16, v17
2103; GFX7-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
2104; GFX7-NEXT:    v_add_i32_e32 v16, vcc, v16, v18
2105; GFX7-NEXT:    v_cndmask_b32_e64 v18, 0, 1, vcc
2106; GFX7-NEXT:    v_add_i32_e32 v17, vcc, v17, v18
2107; GFX7-NEXT:    v_mul_lo_u32 v18, v0, v10
2108; GFX7-NEXT:    v_add_i32_e32 v19, vcc, v19, v20
2109; GFX7-NEXT:    v_mul_hi_u32 v21, v1, v8
2110; GFX7-NEXT:    v_cndmask_b32_e64 v20, 0, 1, vcc
2111; GFX7-NEXT:    v_add_i32_e32 v18, vcc, v19, v18
2112; GFX7-NEXT:    v_cndmask_b32_e64 v19, 0, 1, vcc
2113; GFX7-NEXT:    v_add_i32_e32 v19, vcc, v20, v19
2114; GFX7-NEXT:    v_add_i32_e32 v18, vcc, v18, v21
2115; GFX7-NEXT:    v_mul_hi_u32 v21, v0, v9
2116; GFX7-NEXT:    v_cndmask_b32_e64 v20, 0, 1, vcc
2117; GFX7-NEXT:    v_add_i32_e32 v19, vcc, v19, v20
2118; GFX7-NEXT:    v_mul_lo_u32 v22, v0, v11
2119; GFX7-NEXT:    v_add_i32_e32 v18, vcc, v18, v21
2120; GFX7-NEXT:    v_cndmask_b32_e64 v20, 0, 1, vcc
2121; GFX7-NEXT:    v_add_i32_e32 v19, vcc, v19, v20
2122; GFX7-NEXT:    v_add_i32_e32 v17, vcc, v18, v17
2123; GFX7-NEXT:    v_cndmask_b32_e64 v18, 0, 1, vcc
2124; GFX7-NEXT:    v_mul_lo_u32 v20, v3, v8
2125; GFX7-NEXT:    v_mul_lo_u32 v21, v2, v9
2126; GFX7-NEXT:    v_add_i32_e32 v18, vcc, v19, v18
2127; GFX7-NEXT:    v_mul_lo_u32 v19, v1, v10
2128; GFX7-NEXT:    v_mul_lo_u32 v23, v1, v11
2129; GFX7-NEXT:    v_add_i32_e32 v20, vcc, v20, v21
2130; GFX7-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2131; GFX7-NEXT:    v_add_i32_e32 v19, vcc, v20, v19
2132; GFX7-NEXT:    v_cndmask_b32_e64 v20, 0, 1, vcc
2133; GFX7-NEXT:    v_add_i32_e32 v20, vcc, v21, v20
2134; GFX7-NEXT:    v_add_i32_e32 v19, vcc, v19, v22
2135; GFX7-NEXT:    v_mul_hi_u32 v22, v2, v8
2136; GFX7-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2137; GFX7-NEXT:    v_add_i32_e32 v20, vcc, v20, v21
2138; GFX7-NEXT:    v_mul_lo_u32 v7, v7, v8
2139; GFX7-NEXT:    v_add_i32_e32 v19, vcc, v19, v22
2140; GFX7-NEXT:    v_mul_hi_u32 v22, v1, v9
2141; GFX7-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2142; GFX7-NEXT:    v_add_i32_e32 v20, vcc, v20, v21
2143; GFX7-NEXT:    v_mul_lo_u32 v15, v0, v15
2144; GFX7-NEXT:    v_add_i32_e32 v19, vcc, v19, v22
2145; GFX7-NEXT:    v_mul_hi_u32 v22, v0, v10
2146; GFX7-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2147; GFX7-NEXT:    v_add_i32_e32 v20, vcc, v20, v21
2148; GFX7-NEXT:    v_add_i32_e32 v19, vcc, v19, v22
2149; GFX7-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2150; GFX7-NEXT:    v_add_i32_e32 v20, vcc, v20, v21
2151; GFX7-NEXT:    v_add_i32_e32 v18, vcc, v19, v18
2152; GFX7-NEXT:    v_cndmask_b32_e64 v19, 0, 1, vcc
2153; GFX7-NEXT:    v_mul_lo_u32 v21, v4, v8
2154; GFX7-NEXT:    v_mul_lo_u32 v22, v3, v9
2155; GFX7-NEXT:    v_add_i32_e32 v19, vcc, v20, v19
2156; GFX7-NEXT:    v_mul_lo_u32 v20, v2, v10
2157; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v21, v22
2158; GFX7-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2159; GFX7-NEXT:    v_add_i32_e32 v20, vcc, v21, v20
2160; GFX7-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2161; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v22, v21
2162; GFX7-NEXT:    v_add_i32_e32 v20, vcc, v20, v23
2163; GFX7-NEXT:    v_mul_lo_u32 v23, v0, v12
2164; GFX7-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2165; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v21, v22
2166; GFX7-NEXT:    v_add_i32_e32 v20, vcc, v20, v23
2167; GFX7-NEXT:    v_mul_hi_u32 v23, v3, v8
2168; GFX7-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2169; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v21, v22
2170; GFX7-NEXT:    v_add_i32_e32 v20, vcc, v20, v23
2171; GFX7-NEXT:    v_mul_hi_u32 v23, v2, v9
2172; GFX7-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2173; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v21, v22
2174; GFX7-NEXT:    v_add_i32_e32 v20, vcc, v20, v23
2175; GFX7-NEXT:    v_mul_hi_u32 v23, v1, v10
2176; GFX7-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2177; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v21, v22
2178; GFX7-NEXT:    v_add_i32_e32 v20, vcc, v20, v23
2179; GFX7-NEXT:    v_mul_hi_u32 v23, v0, v11
2180; GFX7-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2181; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v21, v22
2182; GFX7-NEXT:    v_add_i32_e32 v20, vcc, v20, v23
2183; GFX7-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2184; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v21, v22
2185; GFX7-NEXT:    v_add_i32_e32 v19, vcc, v20, v19
2186; GFX7-NEXT:    v_cndmask_b32_e64 v20, 0, 1, vcc
2187; GFX7-NEXT:    v_mul_lo_u32 v22, v5, v8
2188; GFX7-NEXT:    v_mul_lo_u32 v23, v4, v9
2189; GFX7-NEXT:    v_add_i32_e32 v20, vcc, v21, v20
2190; GFX7-NEXT:    v_mul_lo_u32 v21, v3, v10
2191; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v23
2192; GFX7-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2193; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v22, v21
2194; GFX7-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2195; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v23, v22
2196; GFX7-NEXT:    v_mul_lo_u32 v23, v2, v11
2197; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v21, v23
2198; GFX7-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2199; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v23
2200; GFX7-NEXT:    v_mul_lo_u32 v23, v1, v12
2201; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v21, v23
2202; GFX7-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2203; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v23
2204; GFX7-NEXT:    v_mul_lo_u32 v23, v0, v13
2205; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v21, v23
2206; GFX7-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2207; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v23
2208; GFX7-NEXT:    v_mul_hi_u32 v23, v4, v8
2209; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v21, v23
2210; GFX7-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2211; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v23
2212; GFX7-NEXT:    v_mul_hi_u32 v23, v3, v9
2213; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v21, v23
2214; GFX7-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2215; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v23
2216; GFX7-NEXT:    v_mul_hi_u32 v23, v2, v10
2217; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v21, v23
2218; GFX7-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2219; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v23
2220; GFX7-NEXT:    v_mul_hi_u32 v23, v1, v11
2221; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v21, v23
2222; GFX7-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2223; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v23
2224; GFX7-NEXT:    v_mul_hi_u32 v23, v0, v12
2225; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v21, v23
2226; GFX7-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2227; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v23
2228; GFX7-NEXT:    v_add_i32_e32 v20, vcc, v21, v20
2229; GFX7-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2230; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v22, v21
2231; GFX7-NEXT:    v_mul_lo_u32 v22, v6, v8
2232; GFX7-NEXT:    v_mul_lo_u32 v23, v5, v9
2233; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v23
2234; GFX7-NEXT:    v_mul_lo_u32 v23, v4, v10
2235; GFX7-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2236; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v23
2237; GFX7-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2238; GFX7-NEXT:    v_add_i32_e32 v23, vcc, v24, v23
2239; GFX7-NEXT:    v_mul_lo_u32 v24, v3, v11
2240; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v24
2241; GFX7-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2242; GFX7-NEXT:    v_add_i32_e32 v23, vcc, v23, v24
2243; GFX7-NEXT:    v_mul_lo_u32 v24, v2, v12
2244; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v24
2245; GFX7-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2246; GFX7-NEXT:    v_add_i32_e32 v23, vcc, v23, v24
2247; GFX7-NEXT:    v_mul_lo_u32 v24, v1, v13
2248; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v24
2249; GFX7-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2250; GFX7-NEXT:    v_add_i32_e32 v23, vcc, v23, v24
2251; GFX7-NEXT:    v_mul_lo_u32 v24, v0, v14
2252; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v24
2253; GFX7-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2254; GFX7-NEXT:    v_add_i32_e32 v23, vcc, v23, v24
2255; GFX7-NEXT:    v_mul_hi_u32 v24, v5, v8
2256; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v24
2257; GFX7-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2258; GFX7-NEXT:    v_add_i32_e32 v23, vcc, v23, v24
2259; GFX7-NEXT:    v_mul_hi_u32 v24, v4, v9
2260; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v24
2261; GFX7-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2262; GFX7-NEXT:    v_add_i32_e32 v23, vcc, v23, v24
2263; GFX7-NEXT:    v_mul_hi_u32 v24, v3, v10
2264; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v24
2265; GFX7-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2266; GFX7-NEXT:    v_add_i32_e32 v23, vcc, v23, v24
2267; GFX7-NEXT:    v_mul_hi_u32 v24, v2, v11
2268; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v24
2269; GFX7-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2270; GFX7-NEXT:    v_add_i32_e32 v23, vcc, v23, v24
2271; GFX7-NEXT:    v_mul_hi_u32 v24, v1, v12
2272; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v24
2273; GFX7-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2274; GFX7-NEXT:    v_add_i32_e32 v23, vcc, v23, v24
2275; GFX7-NEXT:    v_mul_hi_u32 v24, v0, v13
2276; GFX7-NEXT:    v_add_i32_e32 v22, vcc, v22, v24
2277; GFX7-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2278; GFX7-NEXT:    v_add_i32_e32 v23, vcc, v23, v24
2279; GFX7-NEXT:    v_add_i32_e32 v21, vcc, v22, v21
2280; GFX7-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2281; GFX7-NEXT:    v_add_i32_e32 v23, vcc, v23, v22
2282; GFX7-NEXT:    v_mul_lo_u32 v22, v0, v8
2283; GFX7-NEXT:    v_mul_hi_u32 v8, v6, v8
2284; GFX7-NEXT:    v_mul_lo_u32 v6, v6, v9
2285; GFX7-NEXT:    v_mul_hi_u32 v9, v5, v9
2286; GFX7-NEXT:    v_mul_lo_u32 v5, v5, v10
2287; GFX7-NEXT:    v_mul_hi_u32 v10, v4, v10
2288; GFX7-NEXT:    v_mul_lo_u32 v4, v4, v11
2289; GFX7-NEXT:    v_mul_hi_u32 v11, v3, v11
2290; GFX7-NEXT:    v_mul_lo_u32 v3, v3, v12
2291; GFX7-NEXT:    v_mul_hi_u32 v12, v2, v12
2292; GFX7-NEXT:    v_mul_lo_u32 v2, v2, v13
2293; GFX7-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
2294; GFX7-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
2295; GFX7-NEXT:    v_mul_hi_u32 v13, v1, v13
2296; GFX7-NEXT:    v_mul_lo_u32 v1, v1, v14
2297; GFX7-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
2298; GFX7-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
2299; GFX7-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
2300; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
2301; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v1, v15
2302; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v1, v8
2303; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v1, v9
2304; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v1, v10
2305; GFX7-NEXT:    v_mul_hi_u32 v0, v0, v14
2306; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v1, v11
2307; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v1, v12
2308; GFX7-NEXT:    v_add_i32_e32 v1, vcc, v1, v13
2309; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
2310; GFX7-NEXT:    v_add_i32_e32 v7, vcc, v0, v23
2311; GFX7-NEXT:    v_mov_b32_e32 v0, v22
2312; GFX7-NEXT:    v_mov_b32_e32 v1, v16
2313; GFX7-NEXT:    v_mov_b32_e32 v2, v17
2314; GFX7-NEXT:    v_mov_b32_e32 v3, v18
2315; GFX7-NEXT:    v_mov_b32_e32 v4, v19
2316; GFX7-NEXT:    v_mov_b32_e32 v5, v20
2317; GFX7-NEXT:    v_mov_b32_e32 v6, v21
2318; GFX7-NEXT:    s_setpc_b64 s[30:31]
2319;
2320; GFX8-LABEL: v_mul_i256:
2321; GFX8:       ; %bb.0:
2322; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2323; GFX8-NEXT:    v_mul_lo_u32 v16, v1, v8
2324; GFX8-NEXT:    v_mul_lo_u32 v17, v0, v9
2325; GFX8-NEXT:    v_mul_hi_u32 v18, v0, v8
2326; GFX8-NEXT:    v_mul_lo_u32 v19, v2, v8
2327; GFX8-NEXT:    v_mul_lo_u32 v20, v1, v9
2328; GFX8-NEXT:    v_add_u32_e32 v16, vcc, v16, v17
2329; GFX8-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
2330; GFX8-NEXT:    v_add_u32_e32 v16, vcc, v16, v18
2331; GFX8-NEXT:    v_cndmask_b32_e64 v18, 0, 1, vcc
2332; GFX8-NEXT:    v_add_u32_e32 v17, vcc, v17, v18
2333; GFX8-NEXT:    v_mul_lo_u32 v18, v0, v10
2334; GFX8-NEXT:    v_add_u32_e32 v19, vcc, v19, v20
2335; GFX8-NEXT:    v_mul_hi_u32 v21, v1, v8
2336; GFX8-NEXT:    v_cndmask_b32_e64 v20, 0, 1, vcc
2337; GFX8-NEXT:    v_add_u32_e32 v18, vcc, v19, v18
2338; GFX8-NEXT:    v_cndmask_b32_e64 v19, 0, 1, vcc
2339; GFX8-NEXT:    v_add_u32_e32 v19, vcc, v20, v19
2340; GFX8-NEXT:    v_add_u32_e32 v18, vcc, v18, v21
2341; GFX8-NEXT:    v_mul_hi_u32 v21, v0, v9
2342; GFX8-NEXT:    v_cndmask_b32_e64 v20, 0, 1, vcc
2343; GFX8-NEXT:    v_add_u32_e32 v19, vcc, v19, v20
2344; GFX8-NEXT:    v_mul_lo_u32 v22, v0, v11
2345; GFX8-NEXT:    v_add_u32_e32 v18, vcc, v18, v21
2346; GFX8-NEXT:    v_cndmask_b32_e64 v20, 0, 1, vcc
2347; GFX8-NEXT:    v_add_u32_e32 v19, vcc, v19, v20
2348; GFX8-NEXT:    v_add_u32_e32 v17, vcc, v18, v17
2349; GFX8-NEXT:    v_cndmask_b32_e64 v18, 0, 1, vcc
2350; GFX8-NEXT:    v_mul_lo_u32 v20, v3, v8
2351; GFX8-NEXT:    v_mul_lo_u32 v21, v2, v9
2352; GFX8-NEXT:    v_add_u32_e32 v18, vcc, v19, v18
2353; GFX8-NEXT:    v_mul_lo_u32 v19, v1, v10
2354; GFX8-NEXT:    v_mul_lo_u32 v23, v1, v11
2355; GFX8-NEXT:    v_add_u32_e32 v20, vcc, v20, v21
2356; GFX8-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2357; GFX8-NEXT:    v_add_u32_e32 v19, vcc, v20, v19
2358; GFX8-NEXT:    v_cndmask_b32_e64 v20, 0, 1, vcc
2359; GFX8-NEXT:    v_add_u32_e32 v20, vcc, v21, v20
2360; GFX8-NEXT:    v_add_u32_e32 v19, vcc, v19, v22
2361; GFX8-NEXT:    v_mul_hi_u32 v22, v2, v8
2362; GFX8-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2363; GFX8-NEXT:    v_add_u32_e32 v20, vcc, v20, v21
2364; GFX8-NEXT:    v_mul_lo_u32 v7, v7, v8
2365; GFX8-NEXT:    v_add_u32_e32 v19, vcc, v19, v22
2366; GFX8-NEXT:    v_mul_hi_u32 v22, v1, v9
2367; GFX8-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2368; GFX8-NEXT:    v_add_u32_e32 v20, vcc, v20, v21
2369; GFX8-NEXT:    v_mul_lo_u32 v15, v0, v15
2370; GFX8-NEXT:    v_add_u32_e32 v19, vcc, v19, v22
2371; GFX8-NEXT:    v_mul_hi_u32 v22, v0, v10
2372; GFX8-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2373; GFX8-NEXT:    v_add_u32_e32 v20, vcc, v20, v21
2374; GFX8-NEXT:    v_add_u32_e32 v19, vcc, v19, v22
2375; GFX8-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2376; GFX8-NEXT:    v_add_u32_e32 v20, vcc, v20, v21
2377; GFX8-NEXT:    v_add_u32_e32 v18, vcc, v19, v18
2378; GFX8-NEXT:    v_cndmask_b32_e64 v19, 0, 1, vcc
2379; GFX8-NEXT:    v_mul_lo_u32 v21, v4, v8
2380; GFX8-NEXT:    v_mul_lo_u32 v22, v3, v9
2381; GFX8-NEXT:    v_add_u32_e32 v19, vcc, v20, v19
2382; GFX8-NEXT:    v_mul_lo_u32 v20, v2, v10
2383; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v21, v22
2384; GFX8-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2385; GFX8-NEXT:    v_add_u32_e32 v20, vcc, v21, v20
2386; GFX8-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2387; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v22, v21
2388; GFX8-NEXT:    v_add_u32_e32 v20, vcc, v20, v23
2389; GFX8-NEXT:    v_mul_lo_u32 v23, v0, v12
2390; GFX8-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2391; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v21, v22
2392; GFX8-NEXT:    v_add_u32_e32 v20, vcc, v20, v23
2393; GFX8-NEXT:    v_mul_hi_u32 v23, v3, v8
2394; GFX8-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2395; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v21, v22
2396; GFX8-NEXT:    v_add_u32_e32 v20, vcc, v20, v23
2397; GFX8-NEXT:    v_mul_hi_u32 v23, v2, v9
2398; GFX8-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2399; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v21, v22
2400; GFX8-NEXT:    v_add_u32_e32 v20, vcc, v20, v23
2401; GFX8-NEXT:    v_mul_hi_u32 v23, v1, v10
2402; GFX8-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2403; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v21, v22
2404; GFX8-NEXT:    v_add_u32_e32 v20, vcc, v20, v23
2405; GFX8-NEXT:    v_mul_hi_u32 v23, v0, v11
2406; GFX8-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2407; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v21, v22
2408; GFX8-NEXT:    v_add_u32_e32 v20, vcc, v20, v23
2409; GFX8-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2410; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v21, v22
2411; GFX8-NEXT:    v_add_u32_e32 v19, vcc, v20, v19
2412; GFX8-NEXT:    v_cndmask_b32_e64 v20, 0, 1, vcc
2413; GFX8-NEXT:    v_mul_lo_u32 v22, v5, v8
2414; GFX8-NEXT:    v_mul_lo_u32 v23, v4, v9
2415; GFX8-NEXT:    v_add_u32_e32 v20, vcc, v21, v20
2416; GFX8-NEXT:    v_mul_lo_u32 v21, v3, v10
2417; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v23
2418; GFX8-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2419; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v22, v21
2420; GFX8-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2421; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v23, v22
2422; GFX8-NEXT:    v_mul_lo_u32 v23, v2, v11
2423; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v21, v23
2424; GFX8-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2425; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v23
2426; GFX8-NEXT:    v_mul_lo_u32 v23, v1, v12
2427; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v21, v23
2428; GFX8-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2429; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v23
2430; GFX8-NEXT:    v_mul_lo_u32 v23, v0, v13
2431; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v21, v23
2432; GFX8-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2433; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v23
2434; GFX8-NEXT:    v_mul_hi_u32 v23, v4, v8
2435; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v21, v23
2436; GFX8-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2437; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v23
2438; GFX8-NEXT:    v_mul_hi_u32 v23, v3, v9
2439; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v21, v23
2440; GFX8-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2441; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v23
2442; GFX8-NEXT:    v_mul_hi_u32 v23, v2, v10
2443; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v21, v23
2444; GFX8-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2445; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v23
2446; GFX8-NEXT:    v_mul_hi_u32 v23, v1, v11
2447; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v21, v23
2448; GFX8-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2449; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v23
2450; GFX8-NEXT:    v_mul_hi_u32 v23, v0, v12
2451; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v21, v23
2452; GFX8-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2453; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v23
2454; GFX8-NEXT:    v_add_u32_e32 v20, vcc, v21, v20
2455; GFX8-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2456; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v22, v21
2457; GFX8-NEXT:    v_mul_lo_u32 v22, v6, v8
2458; GFX8-NEXT:    v_mul_lo_u32 v23, v5, v9
2459; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v23
2460; GFX8-NEXT:    v_mul_lo_u32 v23, v4, v10
2461; GFX8-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2462; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v23
2463; GFX8-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2464; GFX8-NEXT:    v_add_u32_e32 v23, vcc, v24, v23
2465; GFX8-NEXT:    v_mul_lo_u32 v24, v3, v11
2466; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v24
2467; GFX8-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2468; GFX8-NEXT:    v_add_u32_e32 v23, vcc, v23, v24
2469; GFX8-NEXT:    v_mul_lo_u32 v24, v2, v12
2470; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v24
2471; GFX8-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2472; GFX8-NEXT:    v_add_u32_e32 v23, vcc, v23, v24
2473; GFX8-NEXT:    v_mul_lo_u32 v24, v1, v13
2474; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v24
2475; GFX8-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2476; GFX8-NEXT:    v_add_u32_e32 v23, vcc, v23, v24
2477; GFX8-NEXT:    v_mul_lo_u32 v24, v0, v14
2478; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v24
2479; GFX8-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2480; GFX8-NEXT:    v_add_u32_e32 v23, vcc, v23, v24
2481; GFX8-NEXT:    v_mul_hi_u32 v24, v5, v8
2482; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v24
2483; GFX8-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2484; GFX8-NEXT:    v_add_u32_e32 v23, vcc, v23, v24
2485; GFX8-NEXT:    v_mul_hi_u32 v24, v4, v9
2486; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v24
2487; GFX8-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2488; GFX8-NEXT:    v_add_u32_e32 v23, vcc, v23, v24
2489; GFX8-NEXT:    v_mul_hi_u32 v24, v3, v10
2490; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v24
2491; GFX8-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2492; GFX8-NEXT:    v_add_u32_e32 v23, vcc, v23, v24
2493; GFX8-NEXT:    v_mul_hi_u32 v24, v2, v11
2494; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v24
2495; GFX8-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2496; GFX8-NEXT:    v_add_u32_e32 v23, vcc, v23, v24
2497; GFX8-NEXT:    v_mul_hi_u32 v24, v1, v12
2498; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v24
2499; GFX8-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2500; GFX8-NEXT:    v_add_u32_e32 v23, vcc, v23, v24
2501; GFX8-NEXT:    v_mul_hi_u32 v24, v0, v13
2502; GFX8-NEXT:    v_add_u32_e32 v22, vcc, v22, v24
2503; GFX8-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2504; GFX8-NEXT:    v_add_u32_e32 v23, vcc, v23, v24
2505; GFX8-NEXT:    v_add_u32_e32 v21, vcc, v22, v21
2506; GFX8-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2507; GFX8-NEXT:    v_add_u32_e32 v23, vcc, v23, v22
2508; GFX8-NEXT:    v_mul_lo_u32 v22, v0, v8
2509; GFX8-NEXT:    v_mul_hi_u32 v8, v6, v8
2510; GFX8-NEXT:    v_mul_lo_u32 v6, v6, v9
2511; GFX8-NEXT:    v_mul_hi_u32 v9, v5, v9
2512; GFX8-NEXT:    v_mul_lo_u32 v5, v5, v10
2513; GFX8-NEXT:    v_mul_hi_u32 v10, v4, v10
2514; GFX8-NEXT:    v_mul_lo_u32 v4, v4, v11
2515; GFX8-NEXT:    v_mul_hi_u32 v11, v3, v11
2516; GFX8-NEXT:    v_mul_lo_u32 v3, v3, v12
2517; GFX8-NEXT:    v_mul_hi_u32 v12, v2, v12
2518; GFX8-NEXT:    v_mul_lo_u32 v2, v2, v13
2519; GFX8-NEXT:    v_add_u32_e32 v6, vcc, v7, v6
2520; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v6, v5
2521; GFX8-NEXT:    v_mul_hi_u32 v13, v1, v13
2522; GFX8-NEXT:    v_mul_lo_u32 v1, v1, v14
2523; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v5, v4
2524; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v4, v3
2525; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v3, v2
2526; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v2, v1
2527; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v15
2528; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v8
2529; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v9
2530; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v10
2531; GFX8-NEXT:    v_mul_hi_u32 v0, v0, v14
2532; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v11
2533; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v12
2534; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v13
2535; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v1, v0
2536; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v0, v23
2537; GFX8-NEXT:    v_mov_b32_e32 v0, v22
2538; GFX8-NEXT:    v_mov_b32_e32 v1, v16
2539; GFX8-NEXT:    v_mov_b32_e32 v2, v17
2540; GFX8-NEXT:    v_mov_b32_e32 v3, v18
2541; GFX8-NEXT:    v_mov_b32_e32 v4, v19
2542; GFX8-NEXT:    v_mov_b32_e32 v5, v20
2543; GFX8-NEXT:    v_mov_b32_e32 v6, v21
2544; GFX8-NEXT:    s_setpc_b64 s[30:31]
2545;
2546; GFX9-LABEL: v_mul_i256:
2547; GFX9:       ; %bb.0:
2548; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2549; GFX9-NEXT:    v_mul_lo_u32 v16, v2, v8
2550; GFX9-NEXT:    v_mul_lo_u32 v17, v1, v9
2551; GFX9-NEXT:    v_mul_lo_u32 v18, v0, v10
2552; GFX9-NEXT:    v_mul_hi_u32 v19, v1, v8
2553; GFX9-NEXT:    v_mul_lo_u32 v20, v1, v8
2554; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, v16, v17
2555; GFX9-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
2556; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, v16, v18
2557; GFX9-NEXT:    v_cndmask_b32_e64 v18, 0, 1, vcc
2558; GFX9-NEXT:    v_add_co_u32_e32 v19, vcc, v16, v19
2559; GFX9-NEXT:    v_mul_lo_u32 v21, v0, v9
2560; GFX9-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
2561; GFX9-NEXT:    v_add3_u32 v18, v17, v18, v16
2562; GFX9-NEXT:    v_mul_hi_u32 v16, v0, v8
2563; GFX9-NEXT:    v_add_co_u32_e32 v17, vcc, v20, v21
2564; GFX9-NEXT:    v_mul_hi_u32 v21, v0, v9
2565; GFX9-NEXT:    v_cndmask_b32_e64 v20, 0, 1, vcc
2566; GFX9-NEXT:    v_add_co_u32_e32 v16, vcc, v17, v16
2567; GFX9-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
2568; GFX9-NEXT:    v_add_co_u32_e32 v19, vcc, v19, v21
2569; GFX9-NEXT:    v_add_u32_e32 v17, v20, v17
2570; GFX9-NEXT:    v_mul_lo_u32 v21, v3, v8
2571; GFX9-NEXT:    v_mul_lo_u32 v22, v2, v9
2572; GFX9-NEXT:    v_cndmask_b32_e64 v20, 0, 1, vcc
2573; GFX9-NEXT:    v_add_co_u32_e32 v17, vcc, v19, v17
2574; GFX9-NEXT:    v_cndmask_b32_e64 v19, 0, 1, vcc
2575; GFX9-NEXT:    v_add3_u32 v18, v18, v20, v19
2576; GFX9-NEXT:    v_mul_lo_u32 v19, v1, v10
2577; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, v21, v22
2578; GFX9-NEXT:    v_mul_lo_u32 v22, v0, v11
2579; GFX9-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2580; GFX9-NEXT:    v_add_co_u32_e32 v19, vcc, v20, v19
2581; GFX9-NEXT:    v_cndmask_b32_e64 v20, 0, 1, vcc
2582; GFX9-NEXT:    v_add_co_u32_e32 v19, vcc, v19, v22
2583; GFX9-NEXT:    v_mul_hi_u32 v23, v2, v8
2584; GFX9-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2585; GFX9-NEXT:    v_add3_u32 v20, v21, v20, v22
2586; GFX9-NEXT:    v_mul_hi_u32 v21, v1, v9
2587; GFX9-NEXT:    v_add_co_u32_e32 v19, vcc, v19, v23
2588; GFX9-NEXT:    v_mul_hi_u32 v23, v0, v10
2589; GFX9-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2590; GFX9-NEXT:    v_add_co_u32_e32 v19, vcc, v19, v21
2591; GFX9-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2592; GFX9-NEXT:    v_add3_u32 v20, v20, v22, v21
2593; GFX9-NEXT:    v_add_co_u32_e32 v19, vcc, v19, v23
2594; GFX9-NEXT:    v_mul_lo_u32 v22, v4, v8
2595; GFX9-NEXT:    v_mul_lo_u32 v23, v3, v9
2596; GFX9-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2597; GFX9-NEXT:    v_add_co_u32_e32 v18, vcc, v19, v18
2598; GFX9-NEXT:    v_cndmask_b32_e64 v19, 0, 1, vcc
2599; GFX9-NEXT:    v_add3_u32 v19, v20, v21, v19
2600; GFX9-NEXT:    v_mul_lo_u32 v20, v2, v10
2601; GFX9-NEXT:    v_add_co_u32_e32 v21, vcc, v22, v23
2602; GFX9-NEXT:    v_mul_lo_u32 v23, v1, v11
2603; GFX9-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2604; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, v21, v20
2605; GFX9-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2606; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, v20, v23
2607; GFX9-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2608; GFX9-NEXT:    v_add3_u32 v21, v22, v21, v23
2609; GFX9-NEXT:    v_mul_lo_u32 v22, v0, v12
2610; GFX9-NEXT:    v_mul_hi_u32 v23, v3, v8
2611; GFX9-NEXT:    v_mul_lo_u32 v7, v7, v8
2612; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, v20, v22
2613; GFX9-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2614; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, v20, v23
2615; GFX9-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2616; GFX9-NEXT:    v_add3_u32 v21, v21, v22, v23
2617; GFX9-NEXT:    v_mul_hi_u32 v22, v2, v9
2618; GFX9-NEXT:    v_mul_hi_u32 v23, v1, v10
2619; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, v20, v22
2620; GFX9-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2621; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, v20, v23
2622; GFX9-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2623; GFX9-NEXT:    v_add3_u32 v21, v21, v22, v23
2624; GFX9-NEXT:    v_mul_hi_u32 v22, v0, v11
2625; GFX9-NEXT:    v_mul_lo_u32 v23, v3, v10
2626; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, v20, v22
2627; GFX9-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2628; GFX9-NEXT:    v_add_co_u32_e32 v19, vcc, v20, v19
2629; GFX9-NEXT:    v_cndmask_b32_e64 v20, 0, 1, vcc
2630; GFX9-NEXT:    v_add3_u32 v20, v21, v22, v20
2631; GFX9-NEXT:    v_mul_lo_u32 v21, v5, v8
2632; GFX9-NEXT:    v_mul_lo_u32 v22, v4, v9
2633; GFX9-NEXT:    v_add_co_u32_e32 v21, vcc, v21, v22
2634; GFX9-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2635; GFX9-NEXT:    v_add_co_u32_e32 v21, vcc, v21, v23
2636; GFX9-NEXT:    v_mul_lo_u32 v23, v2, v11
2637; GFX9-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2638; GFX9-NEXT:    v_add_co_u32_e32 v21, vcc, v21, v23
2639; GFX9-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2640; GFX9-NEXT:    v_add3_u32 v22, v22, v24, v23
2641; GFX9-NEXT:    v_mul_lo_u32 v23, v1, v12
2642; GFX9-NEXT:    v_add_co_u32_e32 v21, vcc, v21, v23
2643; GFX9-NEXT:    v_mul_lo_u32 v23, v0, v13
2644; GFX9-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2645; GFX9-NEXT:    v_add_co_u32_e32 v21, vcc, v21, v23
2646; GFX9-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2647; GFX9-NEXT:    v_add3_u32 v22, v22, v24, v23
2648; GFX9-NEXT:    v_mul_hi_u32 v23, v4, v8
2649; GFX9-NEXT:    v_add_co_u32_e32 v21, vcc, v21, v23
2650; GFX9-NEXT:    v_mul_hi_u32 v23, v3, v9
2651; GFX9-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2652; GFX9-NEXT:    v_add_co_u32_e32 v21, vcc, v21, v23
2653; GFX9-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2654; GFX9-NEXT:    v_add3_u32 v22, v22, v24, v23
2655; GFX9-NEXT:    v_mul_hi_u32 v23, v2, v10
2656; GFX9-NEXT:    v_add_co_u32_e32 v21, vcc, v21, v23
2657; GFX9-NEXT:    v_mul_hi_u32 v23, v1, v11
2658; GFX9-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2659; GFX9-NEXT:    v_add_co_u32_e32 v21, vcc, v21, v23
2660; GFX9-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2661; GFX9-NEXT:    v_add3_u32 v22, v22, v24, v23
2662; GFX9-NEXT:    v_mul_hi_u32 v23, v0, v12
2663; GFX9-NEXT:    v_add_co_u32_e32 v21, vcc, v21, v23
2664; GFX9-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2665; GFX9-NEXT:    v_add_co_u32_e32 v20, vcc, v21, v20
2666; GFX9-NEXT:    v_cndmask_b32_e64 v21, 0, 1, vcc
2667; GFX9-NEXT:    v_add3_u32 v21, v22, v23, v21
2668; GFX9-NEXT:    v_mul_lo_u32 v22, v6, v8
2669; GFX9-NEXT:    v_mul_lo_u32 v23, v5, v9
2670; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, v22, v23
2671; GFX9-NEXT:    v_mul_lo_u32 v23, v4, v10
2672; GFX9-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2673; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, v22, v23
2674; GFX9-NEXT:    v_mul_lo_u32 v23, v3, v11
2675; GFX9-NEXT:    v_cndmask_b32_e64 v25, 0, 1, vcc
2676; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, v22, v23
2677; GFX9-NEXT:    v_cndmask_b32_e64 v23, 0, 1, vcc
2678; GFX9-NEXT:    v_add3_u32 v23, v24, v25, v23
2679; GFX9-NEXT:    v_mul_lo_u32 v24, v2, v12
2680; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, v22, v24
2681; GFX9-NEXT:    v_mul_lo_u32 v24, v1, v13
2682; GFX9-NEXT:    v_cndmask_b32_e64 v25, 0, 1, vcc
2683; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, v22, v24
2684; GFX9-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2685; GFX9-NEXT:    v_add3_u32 v23, v23, v25, v24
2686; GFX9-NEXT:    v_mul_lo_u32 v24, v0, v14
2687; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, v22, v24
2688; GFX9-NEXT:    v_mul_hi_u32 v24, v5, v8
2689; GFX9-NEXT:    v_cndmask_b32_e64 v25, 0, 1, vcc
2690; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, v22, v24
2691; GFX9-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2692; GFX9-NEXT:    v_add3_u32 v23, v23, v25, v24
2693; GFX9-NEXT:    v_mul_hi_u32 v24, v4, v9
2694; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, v22, v24
2695; GFX9-NEXT:    v_mul_hi_u32 v24, v3, v10
2696; GFX9-NEXT:    v_cndmask_b32_e64 v25, 0, 1, vcc
2697; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, v22, v24
2698; GFX9-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2699; GFX9-NEXT:    v_add3_u32 v23, v23, v25, v24
2700; GFX9-NEXT:    v_mul_hi_u32 v24, v2, v11
2701; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, v22, v24
2702; GFX9-NEXT:    v_mul_hi_u32 v24, v1, v12
2703; GFX9-NEXT:    v_cndmask_b32_e64 v25, 0, 1, vcc
2704; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, v22, v24
2705; GFX9-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2706; GFX9-NEXT:    v_add3_u32 v23, v23, v25, v24
2707; GFX9-NEXT:    v_mul_hi_u32 v24, v0, v13
2708; GFX9-NEXT:    v_add_co_u32_e32 v22, vcc, v22, v24
2709; GFX9-NEXT:    v_cndmask_b32_e64 v24, 0, 1, vcc
2710; GFX9-NEXT:    v_add_co_u32_e32 v21, vcc, v22, v21
2711; GFX9-NEXT:    v_cndmask_b32_e64 v22, 0, 1, vcc
2712; GFX9-NEXT:    v_add3_u32 v22, v23, v24, v22
2713; GFX9-NEXT:    v_mul_lo_u32 v23, v6, v9
2714; GFX9-NEXT:    v_mul_lo_u32 v24, v4, v11
2715; GFX9-NEXT:    v_mul_hi_u32 v4, v4, v10
2716; GFX9-NEXT:    v_mul_hi_u32 v6, v6, v8
2717; GFX9-NEXT:    v_add_u32_e32 v7, v7, v23
2718; GFX9-NEXT:    v_mul_lo_u32 v23, v5, v10
2719; GFX9-NEXT:    v_mul_hi_u32 v5, v5, v9
2720; GFX9-NEXT:    v_mul_hi_u32 v9, v3, v11
2721; GFX9-NEXT:    v_mul_hi_u32 v10, v2, v12
2722; GFX9-NEXT:    v_mul_lo_u32 v3, v3, v12
2723; GFX9-NEXT:    v_mul_lo_u32 v2, v2, v13
2724; GFX9-NEXT:    v_mul_hi_u32 v11, v1, v13
2725; GFX9-NEXT:    v_mul_lo_u32 v12, v1, v14
2726; GFX9-NEXT:    v_mul_lo_u32 v13, v0, v15
2727; GFX9-NEXT:    v_add3_u32 v7, v7, v23, v24
2728; GFX9-NEXT:    v_add3_u32 v2, v7, v3, v2
2729; GFX9-NEXT:    v_mul_lo_u32 v1, v0, v8
2730; GFX9-NEXT:    v_add3_u32 v2, v2, v12, v13
2731; GFX9-NEXT:    v_mul_hi_u32 v0, v0, v14
2732; GFX9-NEXT:    v_add3_u32 v2, v2, v6, v5
2733; GFX9-NEXT:    v_add3_u32 v2, v2, v4, v9
2734; GFX9-NEXT:    v_add3_u32 v2, v2, v10, v11
2735; GFX9-NEXT:    v_add3_u32 v7, v2, v0, v22
2736; GFX9-NEXT:    v_mov_b32_e32 v0, v1
2737; GFX9-NEXT:    v_mov_b32_e32 v1, v16
2738; GFX9-NEXT:    v_mov_b32_e32 v2, v17
2739; GFX9-NEXT:    v_mov_b32_e32 v3, v18
2740; GFX9-NEXT:    v_mov_b32_e32 v4, v19
2741; GFX9-NEXT:    v_mov_b32_e32 v5, v20
2742; GFX9-NEXT:    v_mov_b32_e32 v6, v21
2743; GFX9-NEXT:    s_setpc_b64 s[30:31]
2744;
2745; GFX10-LABEL: v_mul_i256:
2746; GFX10:       ; %bb.0:
2747; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2748; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
2749; GFX10-NEXT:    v_mul_lo_u32 v16, v1, v8
2750; GFX10-NEXT:    v_mul_lo_u32 v17, v0, v9
2751; GFX10-NEXT:    v_mul_hi_u32 v18, v0, v8
2752; GFX10-NEXT:    v_mul_lo_u32 v19, v2, v8
2753; GFX10-NEXT:    v_mul_lo_u32 v20, v1, v9
2754; GFX10-NEXT:    v_mul_hi_u32 v21, v1, v8
2755; GFX10-NEXT:    v_mul_lo_u32 v22, v3, v8
2756; GFX10-NEXT:    v_mul_lo_u32 v25, v1, v10
2757; GFX10-NEXT:    v_mul_hi_u32 v23, v0, v9
2758; GFX10-NEXT:    v_add_co_u32 v16, s4, v16, v17
2759; GFX10-NEXT:    v_mul_hi_u32 v27, v0, v10
2760; GFX10-NEXT:    v_cndmask_b32_e64 v17, 0, 1, s4
2761; GFX10-NEXT:    v_mul_hi_u32 v29, v3, v9
2762; GFX10-NEXT:    v_mul_hi_u32 v31, v4, v9
2763; GFX10-NEXT:    v_add_co_u32 v16, s4, v16, v18
2764; GFX10-NEXT:    v_mul_lo_u32 v7, v7, v8
2765; GFX10-NEXT:    v_cndmask_b32_e64 v18, 0, 1, s4
2766; GFX10-NEXT:    v_add_co_u32 v19, s4, v19, v20
2767; GFX10-NEXT:    v_mul_lo_u32 v20, v2, v9
2768; GFX10-NEXT:    v_cndmask_b32_e64 v24, 0, 1, s4
2769; GFX10-NEXT:    v_mul_lo_u32 v15, v0, v15
2770; GFX10-NEXT:    v_add_nc_u32_e32 v17, v17, v18
2771; GFX10-NEXT:    v_mul_lo_u32 v18, v0, v10
2772; GFX10-NEXT:    v_add_co_u32 v18, s4, v19, v18
2773; GFX10-NEXT:    v_cndmask_b32_e64 v19, 0, 1, s4
2774; GFX10-NEXT:    v_add_co_u32 v20, s4, v22, v20
2775; GFX10-NEXT:    v_mul_lo_u32 v22, v0, v11
2776; GFX10-NEXT:    v_add_co_u32 v18, s5, v18, v21
2777; GFX10-NEXT:    v_cndmask_b32_e64 v26, 0, 1, s4
2778; GFX10-NEXT:    v_cndmask_b32_e64 v21, 0, 1, s5
2779; GFX10-NEXT:    v_add_co_u32 v20, s4, v20, v25
2780; GFX10-NEXT:    v_add_co_u32 v18, s5, v18, v23
2781; GFX10-NEXT:    v_mul_hi_u32 v23, v1, v9
2782; GFX10-NEXT:    v_add3_u32 v19, v24, v19, v21
2783; GFX10-NEXT:    v_mul_hi_u32 v21, v2, v8
2784; GFX10-NEXT:    v_cndmask_b32_e64 v24, 0, 1, s4
2785; GFX10-NEXT:    v_add_co_u32 v20, s4, v20, v22
2786; GFX10-NEXT:    v_cndmask_b32_e64 v22, 0, 1, s5
2787; GFX10-NEXT:    v_cndmask_b32_e64 v25, 0, 1, s4
2788; GFX10-NEXT:    v_add_co_u32 v17, s5, v18, v17
2789; GFX10-NEXT:    v_cndmask_b32_e64 v18, 0, 1, s5
2790; GFX10-NEXT:    v_add_co_u32 v20, s4, v20, v21
2791; GFX10-NEXT:    v_add3_u32 v21, v26, v24, v25
2792; GFX10-NEXT:    v_cndmask_b32_e64 v24, 0, 1, s4
2793; GFX10-NEXT:    v_mul_lo_u32 v25, v4, v8
2794; GFX10-NEXT:    v_mul_lo_u32 v26, v3, v9
2795; GFX10-NEXT:    v_add_co_u32 v20, s4, v20, v23
2796; GFX10-NEXT:    v_add3_u32 v18, v19, v22, v18
2797; GFX10-NEXT:    v_cndmask_b32_e64 v23, 0, 1, s4
2798; GFX10-NEXT:    v_add_co_u32 v20, s5, v20, v27
2799; GFX10-NEXT:    v_add3_u32 v19, v21, v24, v23
2800; GFX10-NEXT:    v_mul_lo_u32 v21, v2, v10
2801; GFX10-NEXT:    v_add_co_u32 v22, s4, v25, v26
2802; GFX10-NEXT:    v_mul_lo_u32 v24, v1, v11
2803; GFX10-NEXT:    v_cndmask_b32_e64 v25, 0, 1, s4
2804; GFX10-NEXT:    v_mul_hi_u32 v26, v3, v8
2805; GFX10-NEXT:    v_cndmask_b32_e64 v23, 0, 1, s5
2806; GFX10-NEXT:    v_add_co_u32 v21, s4, v22, v21
2807; GFX10-NEXT:    v_mul_lo_u32 v22, v0, v12
2808; GFX10-NEXT:    v_cndmask_b32_e64 v27, 0, 1, s4
2809; GFX10-NEXT:    v_add_co_u32 v21, s4, v21, v24
2810; GFX10-NEXT:    v_cndmask_b32_e64 v24, 0, 1, s4
2811; GFX10-NEXT:    v_add_co_u32 v18, s4, v20, v18
2812; GFX10-NEXT:    v_cndmask_b32_e64 v20, 0, 1, s4
2813; GFX10-NEXT:    v_add_co_u32 v21, s4, v21, v22
2814; GFX10-NEXT:    v_mul_hi_u32 v22, v2, v9
2815; GFX10-NEXT:    v_add3_u32 v24, v25, v27, v24
2816; GFX10-NEXT:    v_cndmask_b32_e64 v25, 0, 1, s4
2817; GFX10-NEXT:    v_add3_u32 v19, v19, v23, v20
2818; GFX10-NEXT:    v_add_co_u32 v21, s4, v21, v26
2819; GFX10-NEXT:    v_mul_hi_u32 v20, v1, v10
2820; GFX10-NEXT:    v_cndmask_b32_e64 v26, 0, 1, s4
2821; GFX10-NEXT:    v_mul_hi_u32 v27, v0, v11
2822; GFX10-NEXT:    v_add_co_u32 v21, s4, v21, v22
2823; GFX10-NEXT:    v_mul_lo_u32 v22, v5, v8
2824; GFX10-NEXT:    v_add3_u32 v23, v24, v25, v26
2825; GFX10-NEXT:    v_mul_lo_u32 v24, v4, v9
2826; GFX10-NEXT:    v_cndmask_b32_e64 v25, 0, 1, s4
2827; GFX10-NEXT:    v_add_co_u32 v20, s4, v21, v20
2828; GFX10-NEXT:    v_mul_lo_u32 v26, v3, v10
2829; GFX10-NEXT:    v_cndmask_b32_e64 v21, 0, 1, s4
2830; GFX10-NEXT:    v_add_co_u32 v20, s5, v20, v27
2831; GFX10-NEXT:    v_add_co_u32 v22, s4, v22, v24
2832; GFX10-NEXT:    v_add3_u32 v21, v23, v25, v21
2833; GFX10-NEXT:    v_mul_lo_u32 v23, v2, v11
2834; GFX10-NEXT:    v_cndmask_b32_e64 v24, 0, 1, s4
2835; GFX10-NEXT:    v_cndmask_b32_e64 v25, 0, 1, s5
2836; GFX10-NEXT:    v_add_co_u32 v22, s4, v22, v26
2837; GFX10-NEXT:    v_mul_lo_u32 v26, v1, v12
2838; GFX10-NEXT:    v_cndmask_b32_e64 v27, 0, 1, s4
2839; GFX10-NEXT:    v_add_co_u32 v19, s5, v20, v19
2840; GFX10-NEXT:    v_add_co_u32 v22, s4, v22, v23
2841; GFX10-NEXT:    v_mul_lo_u32 v23, v0, v13
2842; GFX10-NEXT:    v_cndmask_b32_e64 v28, 0, 1, s4
2843; GFX10-NEXT:    v_cndmask_b32_e64 v20, 0, 1, s5
2844; GFX10-NEXT:    v_add_co_u32 v22, s4, v22, v26
2845; GFX10-NEXT:    v_mul_hi_u32 v26, v4, v8
2846; GFX10-NEXT:    v_cndmask_b32_e64 v30, 0, 1, s4
2847; GFX10-NEXT:    v_add3_u32 v20, v21, v25, v20
2848; GFX10-NEXT:    v_add_co_u32 v22, s4, v22, v23
2849; GFX10-NEXT:    v_add3_u32 v23, v24, v27, v28
2850; GFX10-NEXT:    v_cndmask_b32_e64 v24, 0, 1, s4
2851; GFX10-NEXT:    v_mul_lo_u32 v27, v6, v8
2852; GFX10-NEXT:    v_mul_lo_u32 v28, v5, v9
2853; GFX10-NEXT:    v_add_co_u32 v21, s4, v22, v26
2854; GFX10-NEXT:    v_mul_hi_u32 v22, v2, v10
2855; GFX10-NEXT:    v_add3_u32 v23, v23, v30, v24
2856; GFX10-NEXT:    v_cndmask_b32_e64 v24, 0, 1, s4
2857; GFX10-NEXT:    v_mul_hi_u32 v26, v1, v11
2858; GFX10-NEXT:    v_add_co_u32 v21, s4, v21, v29
2859; GFX10-NEXT:    v_mul_hi_u32 v29, v0, v12
2860; GFX10-NEXT:    v_cndmask_b32_e64 v25, 0, 1, s4
2861; GFX10-NEXT:    v_add_co_u32 v21, s4, v21, v22
2862; GFX10-NEXT:    v_add3_u32 v23, v23, v24, v25
2863; GFX10-NEXT:    v_mul_lo_u32 v24, v4, v10
2864; GFX10-NEXT:    v_cndmask_b32_e64 v22, 0, 1, s4
2865; GFX10-NEXT:    v_add_co_u32 v25, s4, v27, v28
2866; GFX10-NEXT:    v_add_co_u32 v21, s5, v21, v26
2867; GFX10-NEXT:    v_mul_lo_u32 v27, v3, v11
2868; GFX10-NEXT:    v_cndmask_b32_e64 v26, 0, 1, s5
2869; GFX10-NEXT:    v_cndmask_b32_e64 v28, 0, 1, s4
2870; GFX10-NEXT:    v_add_co_u32 v24, s4, v25, v24
2871; GFX10-NEXT:    v_add_co_u32 v21, s5, v21, v29
2872; GFX10-NEXT:    v_add3_u32 v22, v23, v22, v26
2873; GFX10-NEXT:    v_mul_lo_u32 v23, v2, v12
2874; GFX10-NEXT:    v_cndmask_b32_e64 v25, 0, 1, s4
2875; GFX10-NEXT:    v_add_co_u32 v24, s4, v24, v27
2876; GFX10-NEXT:    v_mul_lo_u32 v27, v1, v13
2877; GFX10-NEXT:    v_cndmask_b32_e64 v29, 0, 1, s4
2878; GFX10-NEXT:    v_cndmask_b32_e64 v26, 0, 1, s5
2879; GFX10-NEXT:    v_add_co_u32 v20, s5, v21, v20
2880; GFX10-NEXT:    v_add_co_u32 v23, s4, v24, v23
2881; GFX10-NEXT:    v_mul_lo_u32 v24, v0, v14
2882; GFX10-NEXT:    v_cndmask_b32_e64 v30, 0, 1, s4
2883; GFX10-NEXT:    v_cndmask_b32_e64 v21, 0, 1, s5
2884; GFX10-NEXT:    v_add_co_u32 v23, s4, v23, v27
2885; GFX10-NEXT:    v_mul_hi_u32 v27, v5, v8
2886; GFX10-NEXT:    v_cndmask_b32_e64 v32, 0, 1, s4
2887; GFX10-NEXT:    v_add3_u32 v21, v22, v26, v21
2888; GFX10-NEXT:    v_mul_hi_u32 v26, v2, v11
2889; GFX10-NEXT:    v_add_co_u32 v23, s4, v23, v24
2890; GFX10-NEXT:    v_add3_u32 v24, v28, v25, v29
2891; GFX10-NEXT:    v_cndmask_b32_e64 v28, 0, 1, s4
2892; GFX10-NEXT:    v_mul_hi_u32 v25, v3, v10
2893; GFX10-NEXT:    v_mul_lo_u32 v29, v3, v12
2894; GFX10-NEXT:    v_add_co_u32 v23, s4, v23, v27
2895; GFX10-NEXT:    v_add3_u32 v24, v24, v30, v32
2896; GFX10-NEXT:    v_cndmask_b32_e64 v27, 0, 1, s4
2897; GFX10-NEXT:    v_mul_hi_u32 v3, v3, v11
2898; GFX10-NEXT:    v_add_co_u32 v23, s4, v23, v31
2899; GFX10-NEXT:    v_add3_u32 v22, v24, v28, v27
2900; GFX10-NEXT:    v_cndmask_b32_e64 v24, 0, 1, s4
2901; GFX10-NEXT:    v_mul_lo_u32 v28, v6, v9
2902; GFX10-NEXT:    v_add_co_u32 v23, s4, v23, v25
2903; GFX10-NEXT:    v_mul_hi_u32 v27, v1, v12
2904; GFX10-NEXT:    v_cndmask_b32_e64 v25, 0, 1, s4
2905; GFX10-NEXT:    v_mul_hi_u32 v6, v6, v8
2906; GFX10-NEXT:    v_add_co_u32 v23, s4, v23, v26
2907; GFX10-NEXT:    v_add3_u32 v22, v22, v24, v25
2908; GFX10-NEXT:    v_mul_lo_u32 v24, v5, v10
2909; GFX10-NEXT:    v_mul_lo_u32 v25, v4, v11
2910; GFX10-NEXT:    v_add_nc_u32_e32 v7, v7, v28
2911; GFX10-NEXT:    v_mul_lo_u32 v28, v2, v13
2912; GFX10-NEXT:    v_mul_hi_u32 v5, v5, v9
2913; GFX10-NEXT:    v_cndmask_b32_e64 v26, 0, 1, s4
2914; GFX10-NEXT:    v_add_co_u32 v23, s4, v23, v27
2915; GFX10-NEXT:    v_mul_hi_u32 v4, v4, v10
2916; GFX10-NEXT:    v_cndmask_b32_e64 v27, 0, 1, s4
2917; GFX10-NEXT:    v_add3_u32 v7, v7, v24, v25
2918; GFX10-NEXT:    v_mul_lo_u32 v24, v1, v14
2919; GFX10-NEXT:    v_mul_hi_u32 v25, v0, v13
2920; GFX10-NEXT:    v_mul_hi_u32 v2, v2, v12
2921; GFX10-NEXT:    v_mul_hi_u32 v1, v1, v13
2922; GFX10-NEXT:    v_add3_u32 v7, v7, v29, v28
2923; GFX10-NEXT:    v_add3_u32 v22, v22, v26, v27
2924; GFX10-NEXT:    v_add3_u32 v7, v7, v24, v15
2925; GFX10-NEXT:    v_add_co_u32 v9, s4, v23, v25
2926; GFX10-NEXT:    v_cndmask_b32_e64 v10, 0, 1, s4
2927; GFX10-NEXT:    v_add3_u32 v5, v7, v6, v5
2928; GFX10-NEXT:    v_add_co_u32 v6, s4, v9, v21
2929; GFX10-NEXT:    v_cndmask_b32_e64 v7, 0, 1, s4
2930; GFX10-NEXT:    v_add3_u32 v3, v5, v4, v3
2931; GFX10-NEXT:    v_mul_hi_u32 v4, v0, v14
2932; GFX10-NEXT:    v_mul_lo_u32 v0, v0, v8
2933; GFX10-NEXT:    v_add3_u32 v5, v22, v10, v7
2934; GFX10-NEXT:    v_add3_u32 v1, v3, v2, v1
2935; GFX10-NEXT:    v_mov_b32_e32 v2, v17
2936; GFX10-NEXT:    v_mov_b32_e32 v3, v18
2937; GFX10-NEXT:    v_add3_u32 v7, v1, v4, v5
2938; GFX10-NEXT:    v_mov_b32_e32 v1, v16
2939; GFX10-NEXT:    v_mov_b32_e32 v4, v19
2940; GFX10-NEXT:    v_mov_b32_e32 v5, v20
2941; GFX10-NEXT:    s_setpc_b64 s[30:31]
2942  %result = mul i256 %num, %den
2943  ret i256 %result
2944}
2945