1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck --check-prefix=SI %s
3; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck --check-prefix=GFX9 %s
4; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefix=GFX10 %s
5; RUN: llc -march=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck --check-prefix=GFX11 %s
6
7define { i64, i1 } @umulo_i64_v_v(i64 %x, i64 %y) {
8; SI-LABEL: umulo_i64_v_v:
9; SI:       ; %bb.0: ; %bb
10; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
11; SI-NEXT:    v_mul_hi_u32 v4, v1, v2
12; SI-NEXT:    v_mul_lo_u32 v5, v1, v2
13; SI-NEXT:    v_mul_hi_u32 v6, v0, v3
14; SI-NEXT:    v_mul_lo_u32 v7, v0, v3
15; SI-NEXT:    v_mul_hi_u32 v8, v0, v2
16; SI-NEXT:    v_mul_hi_u32 v9, v1, v3
17; SI-NEXT:    v_mul_lo_u32 v3, v1, v3
18; SI-NEXT:    v_mul_lo_u32 v0, v0, v2
19; SI-NEXT:    v_add_i32_e32 v1, vcc, v8, v7
20; SI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v6, vcc
21; SI-NEXT:    v_add_i32_e32 v6, vcc, v1, v5
22; SI-NEXT:    v_add_i32_e64 v1, s[4:5], v1, v5
23; SI-NEXT:    v_addc_u32_e32 v2, vcc, v2, v4, vcc
24; SI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v9, vcc
25; SI-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
26; SI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc
27; SI-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[2:3]
28; SI-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
29; SI-NEXT:    s_setpc_b64 s[30:31]
30;
31; GFX9-LABEL: umulo_i64_v_v:
32; GFX9:       ; %bb.0: ; %bb
33; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
34; GFX9-NEXT:    v_mov_b32_e32 v5, v0
35; GFX9-NEXT:    v_mov_b32_e32 v4, v1
36; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v5, v2, 0
37; GFX9-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v5, v3, 0
38; GFX9-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v4, v2, 0
39; GFX9-NEXT:    v_mov_b32_e32 v10, v1
40; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, v10, v6
41; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v7, vcc
42; GFX9-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v4, v3, 0
43; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, v10, v8
44; GFX9-NEXT:    v_addc_co_u32_e32 v8, vcc, v11, v9, vcc
45; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
46; GFX9-NEXT:    v_mul_lo_u32 v4, v4, v2
47; GFX9-NEXT:    v_mul_lo_u32 v5, v5, v3
48; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v8, v6
49; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v7, vcc
50; GFX9-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[2:3]
51; GFX9-NEXT:    v_add3_u32 v1, v1, v5, v4
52; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
53; GFX9-NEXT:    s_setpc_b64 s[30:31]
54;
55; GFX10-LABEL: umulo_i64_v_v:
56; GFX10:       ; %bb.0: ; %bb
57; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
58; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
59; GFX10-NEXT:    v_mov_b32_e32 v4, v0
60; GFX10-NEXT:    v_mov_b32_e32 v5, v1
61; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s4, v4, v2, 0
62; GFX10-NEXT:    v_mad_u64_u32 v[6:7], s4, v4, v3, 0
63; GFX10-NEXT:    v_mad_u64_u32 v[9:10], s4, v5, v2, 0
64; GFX10-NEXT:    v_mad_u64_u32 v[11:12], s4, v5, v3, 0
65; GFX10-NEXT:    v_mov_b32_e32 v8, v1
66; GFX10-NEXT:    v_mul_lo_u32 v5, v5, v2
67; GFX10-NEXT:    v_mul_lo_u32 v4, v4, v3
68; GFX10-NEXT:    v_add_co_u32 v6, vcc_lo, v8, v6
69; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v7, vcc_lo
70; GFX10-NEXT:    v_add3_u32 v1, v1, v4, v5
71; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v6, v9
72; GFX10-NEXT:    v_add_co_ci_u32_e32 v2, vcc_lo, v7, v10, vcc_lo
73; GFX10-NEXT:    v_add_co_ci_u32_e32 v6, vcc_lo, 0, v12, vcc_lo
74; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v11
75; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v6, vcc_lo
76; GFX10-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
77; GFX10-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
78; GFX10-NEXT:    s_setpc_b64 s[30:31]
79;
80; GFX11-LABEL: umulo_i64_v_v:
81; GFX11:       ; %bb.0: ; %bb
82; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
83; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
84; GFX11-NEXT:    v_mov_b32_e32 v4, v0
85; GFX11-NEXT:    v_mov_b32_e32 v5, v1
86; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
87; GFX11-NEXT:    v_mad_u64_u32 v[0:1], null, v4, v2, 0
88; GFX11-NEXT:    v_mad_u64_u32 v[6:7], null, v4, v3, 0
89; GFX11-NEXT:    v_mad_u64_u32 v[9:10], null, v5, v2, 0
90; GFX11-NEXT:    v_mad_u64_u32 v[11:12], null, v5, v3, 0
91; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
92; GFX11-NEXT:    v_mov_b32_e32 v8, v1
93; GFX11-NEXT:    v_mul_lo_u32 v5, v5, v2
94; GFX11-NEXT:    v_mul_lo_u32 v4, v4, v3
95; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v8, v6
96; GFX11-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v7, vcc_lo
97; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
98; GFX11-NEXT:    v_add3_u32 v1, v1, v4, v5
99; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v6, v9
100; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
101; GFX11-NEXT:    v_add_co_ci_u32_e32 v2, vcc_lo, v7, v10, vcc_lo
102; GFX11-NEXT:    v_add_co_ci_u32_e32 v6, vcc_lo, 0, v12, vcc_lo
103; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v11
104; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
105; GFX11-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v6, vcc_lo
106; GFX11-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
107; GFX11-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
108; GFX11-NEXT:    s_setpc_b64 s[30:31]
109bb:
110  %umulo = tail call { i64, i1 } @llvm.umul.with.overflow.i64(i64 %x, i64 %y)
111  ret { i64, i1 } %umulo
112}
113
114define { i64, i1 } @smulo_i64_v_v(i64 %x, i64 %y) {
115; SI-LABEL: smulo_i64_v_v:
116; SI:       ; %bb.0: ; %bb
117; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
118; SI-NEXT:    v_mul_hi_u32 v6, v1, v2
119; SI-NEXT:    v_mul_lo_u32 v5, v1, v2
120; SI-NEXT:    v_mul_hi_u32 v7, v0, v3
121; SI-NEXT:    v_mul_lo_u32 v8, v0, v3
122; SI-NEXT:    v_mul_hi_u32 v9, v0, v2
123; SI-NEXT:    v_mul_hi_i32 v10, v1, v3
124; SI-NEXT:    v_mul_lo_u32 v11, v1, v3
125; SI-NEXT:    v_mul_lo_u32 v4, v0, v2
126; SI-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
127; SI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
128; SI-NEXT:    v_add_i32_e32 v9, vcc, v8, v5
129; SI-NEXT:    v_add_i32_e64 v5, s[4:5], v8, v5
130; SI-NEXT:    v_addc_u32_e32 v8, vcc, v7, v6, vcc
131; SI-NEXT:    v_ashrrev_i32_e32 v6, 31, v5
132; SI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v10, vcc
133; SI-NEXT:    v_mov_b32_e32 v7, v6
134; SI-NEXT:    v_add_i32_e32 v8, vcc, v8, v11
135; SI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
136; SI-NEXT:    v_sub_i32_e32 v2, vcc, v8, v2
137; SI-NEXT:    v_subbrev_u32_e32 v10, vcc, 0, v9, vcc
138; SI-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v1
139; SI-NEXT:    v_cndmask_b32_e32 v1, v9, v10, vcc
140; SI-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
141; SI-NEXT:    v_sub_i32_e32 v0, vcc, v2, v0
142; SI-NEXT:    v_subbrev_u32_e32 v8, vcc, 0, v1, vcc
143; SI-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v3
144; SI-NEXT:    v_cndmask_b32_e32 v1, v1, v8, vcc
145; SI-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
146; SI-NEXT:    v_cmp_ne_u64_e32 vcc, v[0:1], v[6:7]
147; SI-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
148; SI-NEXT:    v_mov_b32_e32 v0, v4
149; SI-NEXT:    v_mov_b32_e32 v1, v5
150; SI-NEXT:    s_setpc_b64 s[30:31]
151;
152; GFX9-LABEL: smulo_i64_v_v:
153; GFX9:       ; %bb.0: ; %bb
154; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
155; GFX9-NEXT:    v_mov_b32_e32 v5, v0
156; GFX9-NEXT:    v_mov_b32_e32 v4, v1
157; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v5, v2, 0
158; GFX9-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v5, v3, 0
159; GFX9-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v4, v2, 0
160; GFX9-NEXT:    v_mov_b32_e32 v10, v1
161; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, v10, v6
162; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v7, vcc
163; GFX9-NEXT:    v_mad_i64_i32 v[6:7], s[4:5], v4, v3, 0
164; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, v10, v8
165; GFX9-NEXT:    v_addc_co_u32_e32 v8, vcc, v11, v9, vcc
166; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
167; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, v8, v6
168; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
169; GFX9-NEXT:    v_sub_co_u32_e32 v8, vcc, v6, v2
170; GFX9-NEXT:    v_subbrev_co_u32_e32 v9, vcc, 0, v7, vcc
171; GFX9-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v4
172; GFX9-NEXT:    v_cndmask_b32_e32 v6, v6, v8, vcc
173; GFX9-NEXT:    v_cndmask_b32_e32 v7, v7, v9, vcc
174; GFX9-NEXT:    v_sub_co_u32_e32 v8, vcc, v6, v5
175; GFX9-NEXT:    v_mul_lo_u32 v4, v4, v2
176; GFX9-NEXT:    v_mul_lo_u32 v5, v5, v3
177; GFX9-NEXT:    v_subbrev_co_u32_e32 v9, vcc, 0, v7, vcc
178; GFX9-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v3
179; GFX9-NEXT:    v_add3_u32 v1, v1, v5, v4
180; GFX9-NEXT:    v_ashrrev_i32_e32 v4, 31, v1
181; GFX9-NEXT:    v_cndmask_b32_e32 v3, v7, v9, vcc
182; GFX9-NEXT:    v_cndmask_b32_e32 v2, v6, v8, vcc
183; GFX9-NEXT:    v_mov_b32_e32 v5, v4
184; GFX9-NEXT:    v_cmp_ne_u64_e32 vcc, v[2:3], v[4:5]
185; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
186; GFX9-NEXT:    s_setpc_b64 s[30:31]
187;
188; GFX10-LABEL: smulo_i64_v_v:
189; GFX10:       ; %bb.0: ; %bb
190; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
191; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
192; GFX10-NEXT:    v_mov_b32_e32 v4, v0
193; GFX10-NEXT:    v_mov_b32_e32 v5, v1
194; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s4, v4, v2, 0
195; GFX10-NEXT:    v_mad_u64_u32 v[6:7], s4, v4, v3, 0
196; GFX10-NEXT:    v_mad_u64_u32 v[9:10], s4, v5, v2, 0
197; GFX10-NEXT:    v_mad_i64_i32 v[11:12], s4, v5, v3, 0
198; GFX10-NEXT:    v_mov_b32_e32 v8, v1
199; GFX10-NEXT:    v_add_co_u32 v6, vcc_lo, v8, v6
200; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v7, vcc_lo
201; GFX10-NEXT:    v_mul_lo_u32 v8, v5, v2
202; GFX10-NEXT:    v_add_co_u32 v6, vcc_lo, v6, v9
203; GFX10-NEXT:    v_add_co_ci_u32_e32 v6, vcc_lo, v7, v10, vcc_lo
204; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v12, vcc_lo
205; GFX10-NEXT:    v_mul_lo_u32 v9, v4, v3
206; GFX10-NEXT:    v_add_co_u32 v6, vcc_lo, v6, v11
207; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v7, vcc_lo
208; GFX10-NEXT:    v_sub_co_u32 v2, vcc_lo, v6, v2
209; GFX10-NEXT:    v_subrev_co_ci_u32_e32 v10, vcc_lo, 0, v7, vcc_lo
210; GFX10-NEXT:    v_cmp_gt_i32_e32 vcc_lo, 0, v5
211; GFX10-NEXT:    v_add3_u32 v1, v1, v9, v8
212; GFX10-NEXT:    v_cndmask_b32_e32 v6, v6, v2, vcc_lo
213; GFX10-NEXT:    v_cndmask_b32_e32 v5, v7, v10, vcc_lo
214; GFX10-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
215; GFX10-NEXT:    v_sub_co_u32 v4, vcc_lo, v6, v4
216; GFX10-NEXT:    v_subrev_co_ci_u32_e32 v7, vcc_lo, 0, v5, vcc_lo
217; GFX10-NEXT:    v_cmp_gt_i32_e32 vcc_lo, 0, v3
218; GFX10-NEXT:    v_mov_b32_e32 v3, v2
219; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc_lo
220; GFX10-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc_lo
221; GFX10-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[2:3]
222; GFX10-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
223; GFX10-NEXT:    s_setpc_b64 s[30:31]
224;
225; GFX11-LABEL: smulo_i64_v_v:
226; GFX11:       ; %bb.0: ; %bb
227; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
228; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
229; GFX11-NEXT:    v_mov_b32_e32 v4, v0
230; GFX11-NEXT:    v_mov_b32_e32 v5, v1
231; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
232; GFX11-NEXT:    v_mad_u64_u32 v[0:1], null, v4, v2, 0
233; GFX11-NEXT:    v_mad_u64_u32 v[6:7], null, v4, v3, 0
234; GFX11-NEXT:    v_mad_u64_u32 v[9:10], null, v5, v2, 0
235; GFX11-NEXT:    v_mad_i64_i32 v[11:12], null, v5, v3, 0
236; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
237; GFX11-NEXT:    v_mov_b32_e32 v8, v1
238; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v8, v6
239; GFX11-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v7, vcc_lo
240; GFX11-NEXT:    v_mul_lo_u32 v8, v5, v2
241; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
242; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, v9
243; GFX11-NEXT:    v_add_co_ci_u32_e32 v6, vcc_lo, v7, v10, vcc_lo
244; GFX11-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v12, vcc_lo
245; GFX11-NEXT:    v_mul_lo_u32 v9, v4, v3
246; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
247; GFX11-NEXT:    v_add_co_u32 v6, vcc_lo, v6, v11
248; GFX11-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v7, vcc_lo
249; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
250; GFX11-NEXT:    v_sub_co_u32 v2, vcc_lo, v6, v2
251; GFX11-NEXT:    v_subrev_co_ci_u32_e32 v10, vcc_lo, 0, v7, vcc_lo
252; GFX11-NEXT:    v_cmp_gt_i32_e32 vcc_lo, 0, v5
253; GFX11-NEXT:    v_add3_u32 v1, v1, v9, v8
254; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
255; GFX11-NEXT:    v_cndmask_b32_e32 v6, v6, v2, vcc_lo
256; GFX11-NEXT:    v_cndmask_b32_e32 v5, v7, v10, vcc_lo
257; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
258; GFX11-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
259; GFX11-NEXT:    v_sub_co_u32 v4, vcc_lo, v6, v4
260; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
261; GFX11-NEXT:    v_subrev_co_ci_u32_e32 v7, vcc_lo, 0, v5, vcc_lo
262; GFX11-NEXT:    v_cmp_gt_i32_e32 vcc_lo, 0, v3
263; GFX11-NEXT:    v_mov_b32_e32 v3, v2
264; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
265; GFX11-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc_lo
266; GFX11-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc_lo
267; GFX11-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[2:3]
268; GFX11-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
269; GFX11-NEXT:    s_setpc_b64 s[30:31]
270bb:
271  %smulo = tail call { i64, i1 } @llvm.smul.with.overflow.i64(i64 %x, i64 %y)
272  ret { i64, i1 } %smulo
273}
274
275define amdgpu_kernel void @umulo_i64_s(i64 %x, i64 %y) {
276; SI-LABEL: umulo_i64_s:
277; SI:       ; %bb.0: ; %bb
278; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
279; SI-NEXT:    s_mov_b32 s7, 0xf000
280; SI-NEXT:    s_waitcnt lgkmcnt(0)
281; SI-NEXT:    v_mov_b32_e32 v0, s2
282; SI-NEXT:    v_mul_hi_u32 v1, s1, v0
283; SI-NEXT:    s_mul_i32 s4, s1, s2
284; SI-NEXT:    v_mov_b32_e32 v2, s3
285; SI-NEXT:    v_mul_hi_u32 v3, s0, v2
286; SI-NEXT:    s_mul_i32 s5, s0, s3
287; SI-NEXT:    v_mul_hi_u32 v0, s0, v0
288; SI-NEXT:    v_mul_hi_u32 v2, s1, v2
289; SI-NEXT:    s_mul_i32 s1, s1, s3
290; SI-NEXT:    s_mul_i32 s0, s0, s2
291; SI-NEXT:    v_add_i32_e32 v4, vcc, s5, v0
292; SI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
293; SI-NEXT:    v_mov_b32_e32 v5, s0
294; SI-NEXT:    v_add_i32_e32 v4, vcc, s4, v4
295; SI-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
296; SI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
297; SI-NEXT:    v_add_i32_e32 v3, vcc, s5, v0
298; SI-NEXT:    v_add_i32_e32 v0, vcc, s1, v1
299; SI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v2, vcc
300; SI-NEXT:    v_add_i32_e32 v2, vcc, s4, v3
301; SI-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
302; SI-NEXT:    v_cndmask_b32_e64 v1, v2, 0, vcc
303; SI-NEXT:    v_cndmask_b32_e64 v0, v5, 0, vcc
304; SI-NEXT:    s_mov_b32 s6, -1
305; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
306; SI-NEXT:    s_endpgm
307;
308; GFX9-LABEL: umulo_i64_s:
309; GFX9:       ; %bb.0: ; %bb
310; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
311; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
312; GFX9-NEXT:    s_mul_i32 s7, s0, s3
313; GFX9-NEXT:    s_mul_hi_u32 s8, s0, s2
314; GFX9-NEXT:    s_mul_hi_u32 s5, s0, s3
315; GFX9-NEXT:    s_add_u32 s9, s8, s7
316; GFX9-NEXT:    s_mul_i32 s6, s1, s2
317; GFX9-NEXT:    s_addc_u32 s5, 0, s5
318; GFX9-NEXT:    s_mul_hi_u32 s4, s1, s2
319; GFX9-NEXT:    s_add_u32 s9, s9, s6
320; GFX9-NEXT:    s_mul_hi_u32 s10, s1, s3
321; GFX9-NEXT:    s_addc_u32 s4, s5, s4
322; GFX9-NEXT:    s_addc_u32 s5, s10, 0
323; GFX9-NEXT:    s_mul_i32 s1, s1, s3
324; GFX9-NEXT:    s_add_u32 s4, s4, s1
325; GFX9-NEXT:    s_addc_u32 s5, 0, s5
326; GFX9-NEXT:    s_add_i32 s1, s8, s7
327; GFX9-NEXT:    s_add_i32 s1, s1, s6
328; GFX9-NEXT:    s_cmp_lg_u64 s[4:5], 0
329; GFX9-NEXT:    s_mul_i32 s2, s0, s2
330; GFX9-NEXT:    v_mov_b32_e32 v0, s1
331; GFX9-NEXT:    s_cselect_b64 s[0:1], -1, 0
332; GFX9-NEXT:    v_cndmask_b32_e64 v1, v0, 0, s[0:1]
333; GFX9-NEXT:    v_mov_b32_e32 v0, s2
334; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, 0, s[0:1]
335; GFX9-NEXT:    global_store_dwordx2 v[0:1], v[0:1], off
336; GFX9-NEXT:    s_endpgm
337;
338; GFX10-LABEL: umulo_i64_s:
339; GFX10:       ; %bb.0: ; %bb
340; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
341; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
342; GFX10-NEXT:    s_mul_i32 s7, s0, s3
343; GFX10-NEXT:    s_mul_hi_u32 s8, s0, s2
344; GFX10-NEXT:    s_mul_hi_u32 s5, s0, s3
345; GFX10-NEXT:    s_mul_hi_u32 s4, s1, s2
346; GFX10-NEXT:    s_mul_i32 s6, s1, s2
347; GFX10-NEXT:    s_mul_hi_u32 s9, s1, s3
348; GFX10-NEXT:    s_mul_i32 s1, s1, s3
349; GFX10-NEXT:    s_add_u32 s3, s8, s7
350; GFX10-NEXT:    s_addc_u32 s5, 0, s5
351; GFX10-NEXT:    s_add_u32 s3, s3, s6
352; GFX10-NEXT:    s_addc_u32 s3, s5, s4
353; GFX10-NEXT:    s_addc_u32 s5, s9, 0
354; GFX10-NEXT:    s_add_u32 s4, s3, s1
355; GFX10-NEXT:    s_addc_u32 s5, 0, s5
356; GFX10-NEXT:    s_add_i32 s1, s8, s7
357; GFX10-NEXT:    s_mul_i32 s0, s0, s2
358; GFX10-NEXT:    s_add_i32 s1, s1, s6
359; GFX10-NEXT:    s_cmp_lg_u64 s[4:5], 0
360; GFX10-NEXT:    s_cselect_b32 s2, -1, 0
361; GFX10-NEXT:    v_cndmask_b32_e64 v1, s1, 0, s2
362; GFX10-NEXT:    v_cndmask_b32_e64 v0, s0, 0, s2
363; GFX10-NEXT:    global_store_dwordx2 v[0:1], v[0:1], off
364; GFX10-NEXT:    s_endpgm
365;
366; GFX11-LABEL: umulo_i64_s:
367; GFX11:       ; %bb.0: ; %bb
368; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
369; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
370; GFX11-NEXT:    s_mul_i32 s7, s0, s3
371; GFX11-NEXT:    s_mul_hi_u32 s8, s0, s2
372; GFX11-NEXT:    s_mul_hi_u32 s5, s0, s3
373; GFX11-NEXT:    s_mul_hi_u32 s4, s1, s2
374; GFX11-NEXT:    s_mul_i32 s6, s1, s2
375; GFX11-NEXT:    s_mul_hi_u32 s9, s1, s3
376; GFX11-NEXT:    s_mul_i32 s1, s1, s3
377; GFX11-NEXT:    s_add_u32 s3, s8, s7
378; GFX11-NEXT:    s_addc_u32 s5, 0, s5
379; GFX11-NEXT:    s_add_u32 s3, s3, s6
380; GFX11-NEXT:    s_addc_u32 s3, s5, s4
381; GFX11-NEXT:    s_addc_u32 s5, s9, 0
382; GFX11-NEXT:    s_add_u32 s4, s3, s1
383; GFX11-NEXT:    s_addc_u32 s5, 0, s5
384; GFX11-NEXT:    s_add_i32 s1, s8, s7
385; GFX11-NEXT:    s_mul_i32 s0, s0, s2
386; GFX11-NEXT:    s_add_i32 s1, s1, s6
387; GFX11-NEXT:    s_cmp_lg_u64 s[4:5], 0
388; GFX11-NEXT:    s_cselect_b32 s2, -1, 0
389; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
390; GFX11-NEXT:    v_cndmask_b32_e64 v1, s1, 0, s2
391; GFX11-NEXT:    v_cndmask_b32_e64 v0, s0, 0, s2
392; GFX11-NEXT:    global_store_b64 v[0:1], v[0:1], off
393; GFX11-NEXT:    s_endpgm
394bb:
395  %umulo = tail call { i64, i1 } @llvm.umul.with.overflow.i64(i64 %x, i64 %y)
396  %mul = extractvalue { i64, i1 } %umulo, 0
397  %overflow = extractvalue { i64, i1 } %umulo, 1
398  %res = select i1 %overflow, i64 0, i64 %mul
399  store i64 %res, i64 addrspace(1)* undef
400  ret void
401}
402
403define amdgpu_kernel void @smulo_i64_s(i64 %x, i64 %y) {
404; SI-LABEL: smulo_i64_s:
405; SI:       ; %bb.0: ; %bb
406; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
407; SI-NEXT:    s_mov_b32 s7, 0xf000
408; SI-NEXT:    s_waitcnt lgkmcnt(0)
409; SI-NEXT:    v_mov_b32_e32 v0, s2
410; SI-NEXT:    v_mul_hi_u32 v1, s1, v0
411; SI-NEXT:    s_mul_i32 s4, s1, s2
412; SI-NEXT:    v_mov_b32_e32 v2, s3
413; SI-NEXT:    v_mul_hi_u32 v3, s0, v2
414; SI-NEXT:    s_mul_i32 s5, s0, s3
415; SI-NEXT:    v_mul_hi_u32 v0, s0, v0
416; SI-NEXT:    v_mul_hi_i32 v2, s1, v2
417; SI-NEXT:    s_mul_i32 s6, s1, s3
418; SI-NEXT:    s_cmp_lt_i32 s1, 0
419; SI-NEXT:    s_mul_i32 s1, s0, s2
420; SI-NEXT:    v_add_i32_e32 v4, vcc, s5, v0
421; SI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
422; SI-NEXT:    v_mov_b32_e32 v5, s1
423; SI-NEXT:    v_add_i32_e32 v4, vcc, s4, v4
424; SI-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
425; SI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
426; SI-NEXT:    v_add_i32_e32 v0, vcc, s5, v0
427; SI-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
428; SI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
429; SI-NEXT:    v_add_i32_e32 v4, vcc, s4, v0
430; SI-NEXT:    v_subrev_i32_e32 v3, vcc, s2, v1
431; SI-NEXT:    v_subbrev_u32_e32 v6, vcc, 0, v2, vcc
432; SI-NEXT:    s_cselect_b64 vcc, -1, 0
433; SI-NEXT:    s_cmp_lt_i32 s3, 0
434; SI-NEXT:    v_ashrrev_i32_e32 v0, 31, v4
435; SI-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
436; SI-NEXT:    v_cndmask_b32_e32 v6, v1, v3, vcc
437; SI-NEXT:    v_mov_b32_e32 v1, v0
438; SI-NEXT:    v_subrev_i32_e32 v7, vcc, s0, v6
439; SI-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v2, vcc
440; SI-NEXT:    s_cselect_b64 vcc, -1, 0
441; SI-NEXT:    v_cndmask_b32_e32 v3, v2, v3, vcc
442; SI-NEXT:    v_cndmask_b32_e32 v2, v6, v7, vcc
443; SI-NEXT:    v_cmp_ne_u64_e32 vcc, v[2:3], v[0:1]
444; SI-NEXT:    v_cndmask_b32_e64 v1, v4, 0, vcc
445; SI-NEXT:    v_cndmask_b32_e64 v0, v5, 0, vcc
446; SI-NEXT:    s_mov_b32 s6, -1
447; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
448; SI-NEXT:    s_endpgm
449;
450; GFX9-LABEL: smulo_i64_s:
451; GFX9:       ; %bb.0: ; %bb
452; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
453; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
454; GFX9-NEXT:    s_mul_i32 s7, s0, s3
455; GFX9-NEXT:    s_mul_hi_u32 s8, s0, s2
456; GFX9-NEXT:    s_mul_hi_u32 s6, s0, s3
457; GFX9-NEXT:    s_add_u32 s9, s8, s7
458; GFX9-NEXT:    s_mul_i32 s5, s1, s2
459; GFX9-NEXT:    s_addc_u32 s6, 0, s6
460; GFX9-NEXT:    s_mul_hi_u32 s4, s1, s2
461; GFX9-NEXT:    s_add_u32 s9, s9, s5
462; GFX9-NEXT:    s_mul_hi_i32 s10, s1, s3
463; GFX9-NEXT:    s_addc_u32 s4, s6, s4
464; GFX9-NEXT:    s_addc_u32 s6, s10, 0
465; GFX9-NEXT:    s_mul_i32 s9, s1, s3
466; GFX9-NEXT:    s_add_u32 s4, s4, s9
467; GFX9-NEXT:    s_addc_u32 s6, 0, s6
468; GFX9-NEXT:    s_sub_u32 s9, s4, s2
469; GFX9-NEXT:    s_subb_u32 s10, s6, 0
470; GFX9-NEXT:    s_cmp_lt_i32 s1, 0
471; GFX9-NEXT:    v_mov_b32_e32 v0, s6
472; GFX9-NEXT:    v_mov_b32_e32 v1, s10
473; GFX9-NEXT:    s_cselect_b64 vcc, -1, 0
474; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
475; GFX9-NEXT:    v_mov_b32_e32 v1, s4
476; GFX9-NEXT:    v_mov_b32_e32 v2, s9
477; GFX9-NEXT:    v_cndmask_b32_e32 v2, v1, v2, vcc
478; GFX9-NEXT:    v_subrev_co_u32_e32 v3, vcc, s0, v2
479; GFX9-NEXT:    v_subbrev_co_u32_e32 v1, vcc, 0, v0, vcc
480; GFX9-NEXT:    s_cmp_lt_i32 s3, 0
481; GFX9-NEXT:    s_cselect_b64 vcc, -1, 0
482; GFX9-NEXT:    s_add_i32 s1, s8, s7
483; GFX9-NEXT:    s_add_i32 s1, s1, s5
484; GFX9-NEXT:    s_ashr_i32 s4, s1, 31
485; GFX9-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
486; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
487; GFX9-NEXT:    s_mov_b32 s5, s4
488; GFX9-NEXT:    s_mul_i32 s0, s0, s2
489; GFX9-NEXT:    v_cmp_ne_u64_e32 vcc, s[4:5], v[0:1]
490; GFX9-NEXT:    v_mov_b32_e32 v2, s1
491; GFX9-NEXT:    v_mov_b32_e32 v0, s0
492; GFX9-NEXT:    v_cndmask_b32_e64 v1, v2, 0, vcc
493; GFX9-NEXT:    v_cndmask_b32_e64 v0, v0, 0, vcc
494; GFX9-NEXT:    global_store_dwordx2 v[0:1], v[0:1], off
495; GFX9-NEXT:    s_endpgm
496;
497; GFX10-LABEL: smulo_i64_s:
498; GFX10:       ; %bb.0: ; %bb
499; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
500; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
501; GFX10-NEXT:    s_mul_i32 s7, s0, s3
502; GFX10-NEXT:    s_mul_hi_u32 s8, s0, s2
503; GFX10-NEXT:    s_mul_hi_u32 s6, s0, s3
504; GFX10-NEXT:    s_mul_i32 s5, s1, s2
505; GFX10-NEXT:    s_add_u32 s11, s8, s7
506; GFX10-NEXT:    s_mul_hi_u32 s4, s1, s2
507; GFX10-NEXT:    s_addc_u32 s6, 0, s6
508; GFX10-NEXT:    s_mul_hi_i32 s9, s1, s3
509; GFX10-NEXT:    s_add_u32 s11, s11, s5
510; GFX10-NEXT:    s_mul_i32 s10, s1, s3
511; GFX10-NEXT:    s_addc_u32 s4, s6, s4
512; GFX10-NEXT:    s_addc_u32 s6, s9, 0
513; GFX10-NEXT:    s_add_u32 s4, s4, s10
514; GFX10-NEXT:    s_addc_u32 s6, 0, s6
515; GFX10-NEXT:    s_sub_u32 s9, s4, s2
516; GFX10-NEXT:    s_subb_u32 s10, s6, 0
517; GFX10-NEXT:    v_mov_b32_e32 v1, s9
518; GFX10-NEXT:    s_cmp_lt_i32 s1, 0
519; GFX10-NEXT:    v_mov_b32_e32 v0, s10
520; GFX10-NEXT:    s_cselect_b32 vcc_lo, -1, 0
521; GFX10-NEXT:    s_cmp_lt_i32 s3, 0
522; GFX10-NEXT:    v_cndmask_b32_e32 v2, s4, v1, vcc_lo
523; GFX10-NEXT:    v_cndmask_b32_e32 v0, s6, v0, vcc_lo
524; GFX10-NEXT:    v_sub_co_u32 v3, vcc_lo, v2, s0
525; GFX10-NEXT:    v_subrev_co_ci_u32_e32 v1, vcc_lo, 0, v0, vcc_lo
526; GFX10-NEXT:    s_cselect_b32 vcc_lo, -1, 0
527; GFX10-NEXT:    s_add_i32 s1, s8, s7
528; GFX10-NEXT:    s_mul_i32 s0, s0, s2
529; GFX10-NEXT:    s_add_i32 s1, s1, s5
530; GFX10-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc_lo
531; GFX10-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc_lo
532; GFX10-NEXT:    s_ashr_i32 s4, s1, 31
533; GFX10-NEXT:    s_mov_b32 s5, s4
534; GFX10-NEXT:    v_cmp_ne_u64_e32 vcc_lo, s[4:5], v[0:1]
535; GFX10-NEXT:    v_cndmask_b32_e64 v1, s1, 0, vcc_lo
536; GFX10-NEXT:    v_cndmask_b32_e64 v0, s0, 0, vcc_lo
537; GFX10-NEXT:    global_store_dwordx2 v[0:1], v[0:1], off
538; GFX10-NEXT:    s_endpgm
539;
540; GFX11-LABEL: smulo_i64_s:
541; GFX11:       ; %bb.0: ; %bb
542; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
543; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
544; GFX11-NEXT:    s_mul_i32 s7, s0, s3
545; GFX11-NEXT:    s_mul_hi_u32 s8, s0, s2
546; GFX11-NEXT:    s_mul_hi_u32 s6, s0, s3
547; GFX11-NEXT:    s_mul_i32 s5, s1, s2
548; GFX11-NEXT:    s_add_u32 s11, s8, s7
549; GFX11-NEXT:    s_mul_hi_u32 s4, s1, s2
550; GFX11-NEXT:    s_addc_u32 s6, 0, s6
551; GFX11-NEXT:    s_mul_hi_i32 s9, s1, s3
552; GFX11-NEXT:    s_add_u32 s11, s11, s5
553; GFX11-NEXT:    s_mul_i32 s10, s1, s3
554; GFX11-NEXT:    s_addc_u32 s4, s6, s4
555; GFX11-NEXT:    s_addc_u32 s6, s9, 0
556; GFX11-NEXT:    s_add_u32 s4, s4, s10
557; GFX11-NEXT:    s_addc_u32 s6, 0, s6
558; GFX11-NEXT:    s_sub_u32 s9, s4, s2
559; GFX11-NEXT:    s_subb_u32 s10, s6, 0
560; GFX11-NEXT:    v_mov_b32_e32 v1, s9
561; GFX11-NEXT:    s_cmp_lt_i32 s1, 0
562; GFX11-NEXT:    v_mov_b32_e32 v0, s10
563; GFX11-NEXT:    s_cselect_b32 vcc_lo, -1, 0
564; GFX11-NEXT:    s_cmp_lt_i32 s3, 0
565; GFX11-NEXT:    v_cndmask_b32_e32 v2, s4, v1, vcc_lo
566; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
567; GFX11-NEXT:    v_cndmask_b32_e32 v0, s6, v0, vcc_lo
568; GFX11-NEXT:    v_sub_co_u32 v3, vcc_lo, v2, s0
569; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
570; GFX11-NEXT:    v_subrev_co_ci_u32_e32 v1, vcc_lo, 0, v0, vcc_lo
571; GFX11-NEXT:    s_cselect_b32 vcc_lo, -1, 0
572; GFX11-NEXT:    s_add_i32 s1, s8, s7
573; GFX11-NEXT:    s_mul_i32 s0, s0, s2
574; GFX11-NEXT:    s_add_i32 s1, s1, s5
575; GFX11-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc_lo
576; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc_lo
577; GFX11-NEXT:    s_ashr_i32 s4, s1, 31
578; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
579; GFX11-NEXT:    s_mov_b32 s5, s4
580; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
581; GFX11-NEXT:    v_cmp_ne_u64_e32 vcc_lo, s[4:5], v[0:1]
582; GFX11-NEXT:    v_cndmask_b32_e64 v1, s1, 0, vcc_lo
583; GFX11-NEXT:    v_cndmask_b32_e64 v0, s0, 0, vcc_lo
584; GFX11-NEXT:    global_store_b64 v[0:1], v[0:1], off
585; GFX11-NEXT:    s_endpgm
586bb:
587  %umulo = tail call { i64, i1 } @llvm.smul.with.overflow.i64(i64 %x, i64 %y)
588  %mul = extractvalue { i64, i1 } %umulo, 0
589  %overflow = extractvalue { i64, i1 } %umulo, 1
590  %res = select i1 %overflow, i64 0, i64 %mul
591  store i64 %res, i64 addrspace(1)* undef
592  ret void
593}
594
595define { i64, i1 } @smulo_i64_v_4(i64 %i) {
596; SI-LABEL: smulo_i64_v_4:
597; SI:       ; %bb.0: ; %bb
598; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
599; SI-NEXT:    v_lshl_b64 v[5:6], v[0:1], 2
600; SI-NEXT:    v_alignbit_b32 v4, v1, v0, 30
601; SI-NEXT:    v_ashr_i64 v[2:3], v[5:6], 2
602; SI-NEXT:    v_cmp_ne_u64_e32 vcc, v[2:3], v[0:1]
603; SI-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
604; SI-NEXT:    v_mov_b32_e32 v0, v5
605; SI-NEXT:    v_mov_b32_e32 v1, v4
606; SI-NEXT:    s_setpc_b64 s[30:31]
607;
608; GFX9-LABEL: smulo_i64_v_4:
609; GFX9:       ; %bb.0: ; %bb
610; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
611; GFX9-NEXT:    v_lshlrev_b64 v[4:5], 2, v[0:1]
612; GFX9-NEXT:    v_alignbit_b32 v3, v1, v0, 30
613; GFX9-NEXT:    v_ashrrev_i64 v[5:6], 2, v[4:5]
614; GFX9-NEXT:    v_cmp_ne_u64_e32 vcc, v[5:6], v[0:1]
615; GFX9-NEXT:    v_mov_b32_e32 v0, v4
616; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
617; GFX9-NEXT:    v_mov_b32_e32 v1, v3
618; GFX9-NEXT:    s_setpc_b64 s[30:31]
619;
620; GFX10-LABEL: smulo_i64_v_4:
621; GFX10:       ; %bb.0: ; %bb
622; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
623; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
624; GFX10-NEXT:    v_lshlrev_b64 v[4:5], 2, v[0:1]
625; GFX10-NEXT:    v_alignbit_b32 v3, v1, v0, 30
626; GFX10-NEXT:    v_ashrrev_i64 v[5:6], 2, v[4:5]
627; GFX10-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[5:6], v[0:1]
628; GFX10-NEXT:    v_mov_b32_e32 v0, v4
629; GFX10-NEXT:    v_mov_b32_e32 v1, v3
630; GFX10-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
631; GFX10-NEXT:    s_setpc_b64 s[30:31]
632;
633; GFX11-LABEL: smulo_i64_v_4:
634; GFX11:       ; %bb.0: ; %bb
635; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
636; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
637; GFX11-NEXT:    v_lshlrev_b64 v[4:5], 2, v[0:1]
638; GFX11-NEXT:    v_alignbit_b32 v3, v1, v0, 30
639; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
640; GFX11-NEXT:    v_ashrrev_i64 v[5:6], 2, v[4:5]
641; GFX11-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[5:6], v[0:1]
642; GFX11-NEXT:    v_mov_b32_e32 v0, v4
643; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
644; GFX11-NEXT:    v_mov_b32_e32 v1, v3
645; GFX11-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
646; GFX11-NEXT:    s_setpc_b64 s[30:31]
647bb:
648  %umulo = tail call { i64, i1 } @llvm.smul.with.overflow.i64(i64 %i, i64 4)
649  ret { i64, i1 } %umulo
650}
651
652define { i64, i1 } @umulo_i64_v_4(i64 %i) {
653; SI-LABEL: umulo_i64_v_4:
654; SI:       ; %bb.0: ; %bb
655; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
656; SI-NEXT:    v_and_b32_e32 v7, 0x3fffffff, v1
657; SI-NEXT:    v_mov_b32_e32 v6, v0
658; SI-NEXT:    v_lshl_b64 v[4:5], v[0:1], 2
659; SI-NEXT:    v_alignbit_b32 v3, v1, v0, 30
660; SI-NEXT:    v_cmp_ne_u64_e32 vcc, v[6:7], v[0:1]
661; SI-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
662; SI-NEXT:    v_mov_b32_e32 v0, v4
663; SI-NEXT:    v_mov_b32_e32 v1, v3
664; SI-NEXT:    s_setpc_b64 s[30:31]
665;
666; GFX9-LABEL: umulo_i64_v_4:
667; GFX9:       ; %bb.0: ; %bb
668; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
669; GFX9-NEXT:    v_and_b32_e32 v7, 0x3fffffff, v1
670; GFX9-NEXT:    v_mov_b32_e32 v6, v0
671; GFX9-NEXT:    v_lshlrev_b64 v[4:5], 2, v[0:1]
672; GFX9-NEXT:    v_cmp_ne_u64_e32 vcc, v[6:7], v[0:1]
673; GFX9-NEXT:    v_alignbit_b32 v3, v1, v0, 30
674; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
675; GFX9-NEXT:    v_mov_b32_e32 v0, v4
676; GFX9-NEXT:    v_mov_b32_e32 v1, v3
677; GFX9-NEXT:    s_setpc_b64 s[30:31]
678;
679; GFX10-LABEL: umulo_i64_v_4:
680; GFX10:       ; %bb.0: ; %bb
681; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
682; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
683; GFX10-NEXT:    v_and_b32_e32 v7, 0x3fffffff, v1
684; GFX10-NEXT:    v_mov_b32_e32 v6, v0
685; GFX10-NEXT:    v_lshlrev_b64 v[4:5], 2, v[0:1]
686; GFX10-NEXT:    v_alignbit_b32 v3, v1, v0, 30
687; GFX10-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[6:7], v[0:1]
688; GFX10-NEXT:    v_mov_b32_e32 v0, v4
689; GFX10-NEXT:    v_mov_b32_e32 v1, v3
690; GFX10-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
691; GFX10-NEXT:    s_setpc_b64 s[30:31]
692;
693; GFX11-LABEL: umulo_i64_v_4:
694; GFX11:       ; %bb.0: ; %bb
695; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
696; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
697; GFX11-NEXT:    v_and_b32_e32 v7, 0x3fffffff, v1
698; GFX11-NEXT:    v_mov_b32_e32 v6, v0
699; GFX11-NEXT:    v_lshlrev_b64 v[4:5], 2, v[0:1]
700; GFX11-NEXT:    v_alignbit_b32 v3, v1, v0, 30
701; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
702; GFX11-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[6:7], v[0:1]
703; GFX11-NEXT:    v_mov_b32_e32 v0, v4
704; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3)
705; GFX11-NEXT:    v_mov_b32_e32 v1, v3
706; GFX11-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
707; GFX11-NEXT:    s_setpc_b64 s[30:31]
708bb:
709  %umulo = tail call { i64, i1 } @llvm.umul.with.overflow.i64(i64 %i, i64 4)
710  ret { i64, i1 } %umulo
711}
712
713declare { i64, i1 } @llvm.umul.with.overflow.i64(i64, i64)
714declare { i64, i1 } @llvm.smul.with.overflow.i64(i64, i64)
715