1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -march=amdgcn -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -check-prefixes=CI %s
3; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefixes=SI %s
4; RUN: llc -march=amdgcn -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9 %s
5; RUN: llc -march=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11 %s
6
7; On GFX11, ensure vdst and src2 do not partially overlap. Full overlap is ok.
8
9define i64 @mad_i64_i32_sextops(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
10; CI-LABEL: mad_i64_i32_sextops:
11; CI:       ; %bb.0:
12; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13; CI-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3]
14; CI-NEXT:    s_setpc_b64 s[30:31]
15;
16; SI-LABEL: mad_i64_i32_sextops:
17; SI:       ; %bb.0:
18; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
19; SI-NEXT:    v_mul_lo_u32 v4, v0, v1
20; SI-NEXT:    v_mul_hi_i32 v1, v0, v1
21; SI-NEXT:    v_add_i32_e32 v0, vcc, v4, v2
22; SI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
23; SI-NEXT:    s_setpc_b64 s[30:31]
24;
25; GFX9-LABEL: mad_i64_i32_sextops:
26; GFX9:       ; %bb.0:
27; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
28; GFX9-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3]
29; GFX9-NEXT:    s_setpc_b64 s[30:31]
30;
31; GFX11-LABEL: mad_i64_i32_sextops:
32; GFX11:       ; %bb.0:
33; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
34; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
35; GFX11-NEXT:    v_mov_b32_e32 v4, v1
36; GFX11-NEXT:    v_mov_b32_e32 v5, v0
37; GFX11-NEXT:    v_mad_i64_i32 v[0:1], null, v5, v4, v[2:3]
38; GFX11-NEXT:    s_setpc_b64 s[30:31]
39  %sext0 = sext i32 %arg0 to i64
40  %sext1 = sext i32 %arg1 to i64
41  %mul = mul i64 %sext0, %sext1
42  %mad = add i64 %mul, %arg2
43  ret i64 %mad
44}
45
46define i64 @mad_i64_i32_sextops_commute(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
47; CI-LABEL: mad_i64_i32_sextops_commute:
48; CI:       ; %bb.0:
49; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
50; CI-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3]
51; CI-NEXT:    s_setpc_b64 s[30:31]
52;
53; SI-LABEL: mad_i64_i32_sextops_commute:
54; SI:       ; %bb.0:
55; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
56; SI-NEXT:    v_mul_lo_u32 v4, v0, v1
57; SI-NEXT:    v_mul_hi_i32 v1, v0, v1
58; SI-NEXT:    v_add_i32_e32 v0, vcc, v2, v4
59; SI-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
60; SI-NEXT:    s_setpc_b64 s[30:31]
61;
62; GFX9-LABEL: mad_i64_i32_sextops_commute:
63; GFX9:       ; %bb.0:
64; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
65; GFX9-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3]
66; GFX9-NEXT:    s_setpc_b64 s[30:31]
67;
68; GFX11-LABEL: mad_i64_i32_sextops_commute:
69; GFX11:       ; %bb.0:
70; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
71; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
72; GFX11-NEXT:    v_mov_b32_e32 v4, v1
73; GFX11-NEXT:    v_mov_b32_e32 v5, v0
74; GFX11-NEXT:    v_mad_i64_i32 v[0:1], null, v5, v4, v[2:3]
75; GFX11-NEXT:    s_setpc_b64 s[30:31]
76  %sext0 = sext i32 %arg0 to i64
77  %sext1 = sext i32 %arg1 to i64
78  %mul = mul i64 %sext0, %sext1
79  %mad = add i64 %arg2, %mul
80  ret i64 %mad
81}
82
83define i64 @mad_u64_u32_zextops(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
84; CI-LABEL: mad_u64_u32_zextops:
85; CI:       ; %bb.0:
86; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
87; CI-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[2:3]
88; CI-NEXT:    s_setpc_b64 s[30:31]
89;
90; SI-LABEL: mad_u64_u32_zextops:
91; SI:       ; %bb.0:
92; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
93; SI-NEXT:    v_mul_lo_u32 v4, v0, v1
94; SI-NEXT:    v_mul_hi_u32 v1, v0, v1
95; SI-NEXT:    v_add_i32_e32 v0, vcc, v4, v2
96; SI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
97; SI-NEXT:    s_setpc_b64 s[30:31]
98;
99; GFX9-LABEL: mad_u64_u32_zextops:
100; GFX9:       ; %bb.0:
101; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
102; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[2:3]
103; GFX9-NEXT:    s_setpc_b64 s[30:31]
104;
105; GFX11-LABEL: mad_u64_u32_zextops:
106; GFX11:       ; %bb.0:
107; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
108; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
109; GFX11-NEXT:    v_mov_b32_e32 v4, v1
110; GFX11-NEXT:    v_mov_b32_e32 v5, v0
111; GFX11-NEXT:    v_mad_u64_u32 v[0:1], null, v5, v4, v[2:3]
112; GFX11-NEXT:    s_setpc_b64 s[30:31]
113  %sext0 = zext i32 %arg0 to i64
114  %sext1 = zext i32 %arg1 to i64
115  %mul = mul i64 %sext0, %sext1
116  %mad = add i64 %mul, %arg2
117  ret i64 %mad
118}
119
120define i64 @mad_u64_u32_zextops_commute(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
121; CI-LABEL: mad_u64_u32_zextops_commute:
122; CI:       ; %bb.0:
123; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
124; CI-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[2:3]
125; CI-NEXT:    s_setpc_b64 s[30:31]
126;
127; SI-LABEL: mad_u64_u32_zextops_commute:
128; SI:       ; %bb.0:
129; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
130; SI-NEXT:    v_mul_lo_u32 v4, v0, v1
131; SI-NEXT:    v_mul_hi_u32 v1, v0, v1
132; SI-NEXT:    v_add_i32_e32 v0, vcc, v2, v4
133; SI-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
134; SI-NEXT:    s_setpc_b64 s[30:31]
135;
136; GFX9-LABEL: mad_u64_u32_zextops_commute:
137; GFX9:       ; %bb.0:
138; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
139; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[2:3]
140; GFX9-NEXT:    s_setpc_b64 s[30:31]
141;
142; GFX11-LABEL: mad_u64_u32_zextops_commute:
143; GFX11:       ; %bb.0:
144; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
145; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
146; GFX11-NEXT:    v_mov_b32_e32 v4, v1
147; GFX11-NEXT:    v_mov_b32_e32 v5, v0
148; GFX11-NEXT:    v_mad_u64_u32 v[0:1], null, v5, v4, v[2:3]
149; GFX11-NEXT:    s_setpc_b64 s[30:31]
150  %sext0 = zext i32 %arg0 to i64
151  %sext1 = zext i32 %arg1 to i64
152  %mul = mul i64 %sext0, %sext1
153  %mad = add i64 %arg2, %mul
154  ret i64 %mad
155}
156
157define i128 @mad_i64_i32_sextops_i32_i128(i32 %arg0, i32 %arg1, i128 %arg2) #0 {
158; CI-LABEL: mad_i64_i32_sextops_i32_i128:
159; CI:       ; %bb.0:
160; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
161; CI-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v0, v1, 0
162; CI-NEXT:    v_ashrrev_i32_e32 v13, 31, v0
163; CI-NEXT:    v_mov_b32_e32 v8, 0
164; CI-NEXT:    v_mad_u64_u32 v[9:10], s[4:5], v13, v1, v[7:8]
165; CI-NEXT:    v_ashrrev_i32_e32 v14, 31, v1
166; CI-NEXT:    v_mad_i64_i32 v[11:12], s[4:5], v1, v13, 0
167; CI-NEXT:    v_mov_b32_e32 v7, v10
168; CI-NEXT:    v_mov_b32_e32 v10, v8
169; CI-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v0, v14, v[9:10]
170; CI-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v14, v0, v[11:12]
171; CI-NEXT:    v_add_i32_e32 v9, vcc, v7, v9
172; CI-NEXT:    v_addc_u32_e64 v10, s[4:5], 0, 0, vcc
173; CI-NEXT:    v_mad_u64_u32 v[9:10], s[4:5], v13, v14, v[9:10]
174; CI-NEXT:    v_add_i32_e32 v7, vcc, v9, v0
175; CI-NEXT:    v_addc_u32_e32 v9, vcc, v10, v1, vcc
176; CI-NEXT:    v_mov_b32_e32 v1, v8
177; CI-NEXT:    v_add_i32_e32 v0, vcc, v6, v2
178; CI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
179; CI-NEXT:    v_addc_u32_e32 v2, vcc, v7, v4, vcc
180; CI-NEXT:    v_addc_u32_e32 v3, vcc, v9, v5, vcc
181; CI-NEXT:    s_setpc_b64 s[30:31]
182;
183; SI-LABEL: mad_i64_i32_sextops_i32_i128:
184; SI:       ; %bb.0:
185; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
186; SI-NEXT:    v_ashrrev_i32_e32 v6, 31, v0
187; SI-NEXT:    v_mul_lo_u32 v11, v6, v1
188; SI-NEXT:    v_mul_hi_u32 v12, v0, v1
189; SI-NEXT:    v_ashrrev_i32_e32 v7, 31, v1
190; SI-NEXT:    v_mul_hi_u32 v14, v6, v1
191; SI-NEXT:    v_mul_lo_u32 v13, v0, v7
192; SI-NEXT:    v_mul_hi_u32 v10, v0, v7
193; SI-NEXT:    v_add_i32_e32 v12, vcc, v11, v12
194; SI-NEXT:    v_addc_u32_e32 v14, vcc, 0, v14, vcc
195; SI-NEXT:    v_mul_hi_u32 v8, v6, v7
196; SI-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
197; SI-NEXT:    v_addc_u32_e32 v10, vcc, 0, v10, vcc
198; SI-NEXT:    v_mul_i32_i24_e32 v9, v6, v7
199; SI-NEXT:    v_add_i32_e32 v10, vcc, v14, v10
200; SI-NEXT:    v_mul_hi_i32 v6, v1, v6
201; SI-NEXT:    v_mul_hi_i32 v7, v7, v0
202; SI-NEXT:    v_addc_u32_e64 v14, s[4:5], 0, 0, vcc
203; SI-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
204; SI-NEXT:    v_addc_u32_e32 v8, vcc, v8, v14, vcc
205; SI-NEXT:    v_add_i32_e32 v10, vcc, v13, v11
206; SI-NEXT:    v_mul_lo_u32 v0, v0, v1
207; SI-NEXT:    v_addc_u32_e32 v6, vcc, v7, v6, vcc
208; SI-NEXT:    v_add_i32_e32 v7, vcc, v9, v10
209; SI-NEXT:    v_addc_u32_e32 v6, vcc, v8, v6, vcc
210; SI-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
211; SI-NEXT:    v_addc_u32_e32 v1, vcc, v12, v3, vcc
212; SI-NEXT:    v_addc_u32_e32 v2, vcc, v7, v4, vcc
213; SI-NEXT:    v_addc_u32_e32 v3, vcc, v6, v5, vcc
214; SI-NEXT:    s_setpc_b64 s[30:31]
215;
216; GFX9-LABEL: mad_i64_i32_sextops_i32_i128:
217; GFX9:       ; %bb.0:
218; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
219; GFX9-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v0, v1, 0
220; GFX9-NEXT:    v_ashrrev_i32_e32 v13, 31, v0
221; GFX9-NEXT:    v_mov_b32_e32 v9, 0
222; GFX9-NEXT:    v_mov_b32_e32 v8, v7
223; GFX9-NEXT:    v_mad_u64_u32 v[10:11], s[4:5], v13, v1, v[8:9]
224; GFX9-NEXT:    v_ashrrev_i32_e32 v14, 31, v1
225; GFX9-NEXT:    v_mov_b32_e32 v8, v11
226; GFX9-NEXT:    v_mov_b32_e32 v11, v9
227; GFX9-NEXT:    v_mad_u64_u32 v[10:11], s[4:5], v0, v14, v[10:11]
228; GFX9-NEXT:    v_mov_b32_e32 v12, v11
229; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, v8, v12
230; GFX9-NEXT:    v_addc_co_u32_e64 v9, s[4:5], 0, 0, vcc
231; GFX9-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v13, v14, v[8:9]
232; GFX9-NEXT:    v_mad_i64_i32 v[12:13], s[4:5], v1, v13, 0
233; GFX9-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v14, v0, v[12:13]
234; GFX9-NEXT:    v_add_co_u32_e32 v7, vcc, v8, v0
235; GFX9-NEXT:    v_addc_co_u32_e32 v8, vcc, v9, v1, vcc
236; GFX9-NEXT:    v_mov_b32_e32 v1, v10
237; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v6, v2
238; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
239; GFX9-NEXT:    v_addc_co_u32_e32 v2, vcc, v7, v4, vcc
240; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v8, v5, vcc
241; GFX9-NEXT:    s_setpc_b64 s[30:31]
242;
243; GFX11-LABEL: mad_i64_i32_sextops_i32_i128:
244; GFX11:       ; %bb.0:
245; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
246; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
247; GFX11-NEXT:    v_mad_u64_u32 v[6:7], null, v0, v1, 0
248; GFX11-NEXT:    v_mov_b32_e32 v8, 0
249; GFX11-NEXT:    v_ashrrev_i32_e32 v14, 31, v0
250; GFX11-NEXT:    v_ashrrev_i32_e32 v15, 31, v1
251; GFX11-NEXT:    v_mad_u64_u32 v[9:10], null, v14, v1, v[7:8]
252; GFX11-NEXT:    v_mov_b32_e32 v7, v10
253; GFX11-NEXT:    v_mov_b32_e32 v10, v8
254; GFX11-NEXT:    v_mad_u64_u32 v[11:12], null, v0, v15, v[9:10]
255; GFX11-NEXT:    v_mad_i64_i32 v[9:10], null, v1, v14, 0
256; GFX11-NEXT:    v_mov_b32_e32 v8, v12
257; GFX11-NEXT:    v_mad_i64_i32 v[12:13], null, v15, v0, v[9:10]
258; GFX11-NEXT:    v_add_co_u32 v7, s0, v7, v8
259; GFX11-NEXT:    v_add_co_ci_u32_e64 v8, null, 0, 0, s0
260; GFX11-NEXT:    v_mad_u64_u32 v[0:1], null, v14, v15, v[7:8]
261; GFX11-NEXT:    v_mov_b32_e32 v7, v11
262; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v0, v12
263; GFX11-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, v1, v13, vcc_lo
264; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v6, v2
265; GFX11-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v7, v3, vcc_lo
266; GFX11-NEXT:    v_add_co_ci_u32_e32 v2, vcc_lo, v8, v4, vcc_lo
267; GFX11-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, v9, v5, vcc_lo
268; GFX11-NEXT:    s_setpc_b64 s[30:31]
269  %sext0 = sext i32 %arg0 to i128
270  %sext1 = sext i32 %arg1 to i128
271  %mul = mul i128 %sext0, %sext1
272  %mad = add i128 %mul, %arg2
273  ret i128 %mad
274}
275
276define i63 @mad_i64_i32_sextops_i32_i63(i32 %arg0, i32 %arg1, i63 %arg2) #0 {
277; CI-LABEL: mad_i64_i32_sextops_i32_i63:
278; CI:       ; %bb.0:
279; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
280; CI-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3]
281; CI-NEXT:    s_setpc_b64 s[30:31]
282;
283; SI-LABEL: mad_i64_i32_sextops_i32_i63:
284; SI:       ; %bb.0:
285; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
286; SI-NEXT:    v_mul_lo_u32 v4, v0, v1
287; SI-NEXT:    v_mul_hi_i32 v1, v0, v1
288; SI-NEXT:    v_add_i32_e32 v0, vcc, v4, v2
289; SI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
290; SI-NEXT:    s_setpc_b64 s[30:31]
291;
292; GFX9-LABEL: mad_i64_i32_sextops_i32_i63:
293; GFX9:       ; %bb.0:
294; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
295; GFX9-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3]
296; GFX9-NEXT:    s_setpc_b64 s[30:31]
297;
298; GFX11-LABEL: mad_i64_i32_sextops_i32_i63:
299; GFX11:       ; %bb.0:
300; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
301; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
302; GFX11-NEXT:    v_mov_b32_e32 v4, v1
303; GFX11-NEXT:    v_mov_b32_e32 v5, v0
304; GFX11-NEXT:    v_mad_i64_i32 v[0:1], null, v5, v4, v[2:3]
305; GFX11-NEXT:    s_setpc_b64 s[30:31]
306  %sext0 = sext i32 %arg0 to i63
307  %sext1 = sext i32 %arg1 to i63
308  %mul = mul i63 %sext0, %sext1
309  %mad = add i63 %mul, %arg2
310  ret i63 %mad
311}
312
313define i63 @mad_i64_i32_sextops_i31_i63(i31 %arg0, i31 %arg1, i63 %arg2) #0 {
314; CI-LABEL: mad_i64_i32_sextops_i31_i63:
315; CI:       ; %bb.0:
316; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
317; CI-NEXT:    v_bfe_i32 v1, v1, 0, 31
318; CI-NEXT:    v_bfe_i32 v0, v0, 0, 31
319; CI-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3]
320; CI-NEXT:    s_setpc_b64 s[30:31]
321;
322; SI-LABEL: mad_i64_i32_sextops_i31_i63:
323; SI:       ; %bb.0:
324; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
325; SI-NEXT:    v_lshlrev_b32_e32 v4, 1, v0
326; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v1
327; SI-NEXT:    v_ashr_i64 v[4:5], v[3:4], 33
328; SI-NEXT:    v_ashr_i64 v[0:1], v[0:1], 33
329; SI-NEXT:    v_mul_lo_u32 v1, v4, v0
330; SI-NEXT:    v_mul_hi_i32 v4, v4, v0
331; SI-NEXT:    v_add_i32_e32 v0, vcc, v1, v2
332; SI-NEXT:    v_addc_u32_e32 v1, vcc, v4, v3, vcc
333; SI-NEXT:    s_setpc_b64 s[30:31]
334;
335; GFX9-LABEL: mad_i64_i32_sextops_i31_i63:
336; GFX9:       ; %bb.0:
337; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
338; GFX9-NEXT:    v_bfe_i32 v1, v1, 0, 31
339; GFX9-NEXT:    v_bfe_i32 v0, v0, 0, 31
340; GFX9-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3]
341; GFX9-NEXT:    s_setpc_b64 s[30:31]
342;
343; GFX11-LABEL: mad_i64_i32_sextops_i31_i63:
344; GFX11:       ; %bb.0:
345; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
346; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
347; GFX11-NEXT:    v_bfe_i32 v4, v1, 0, 31
348; GFX11-NEXT:    v_bfe_i32 v5, v0, 0, 31
349; GFX11-NEXT:    v_mad_i64_i32 v[0:1], null, v5, v4, v[2:3]
350; GFX11-NEXT:    s_setpc_b64 s[30:31]
351  %sext0 = sext i31 %arg0 to i63
352  %sext1 = sext i31 %arg1 to i63
353  %mul = mul i63 %sext0, %sext1
354  %mad = add i63 %mul, %arg2
355  ret i63 %mad
356}
357
358define i64 @mad_i64_i32_extops_i32_i64(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
359; CI-LABEL: mad_i64_i32_extops_i32_i64:
360; CI:       ; %bb.0:
361; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
362; CI-NEXT:    v_ashrrev_i32_e32 v4, 31, v0
363; CI-NEXT:    v_mul_lo_u32 v4, v4, v1
364; CI-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[2:3]
365; CI-NEXT:    v_add_i32_e32 v1, vcc, v4, v1
366; CI-NEXT:    s_setpc_b64 s[30:31]
367;
368; SI-LABEL: mad_i64_i32_extops_i32_i64:
369; SI:       ; %bb.0:
370; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
371; SI-NEXT:    v_ashrrev_i32_e32 v4, 31, v0
372; SI-NEXT:    v_mul_hi_u32 v5, v0, v1
373; SI-NEXT:    v_mul_lo_u32 v4, v4, v1
374; SI-NEXT:    v_mul_lo_u32 v0, v0, v1
375; SI-NEXT:    v_add_i32_e32 v1, vcc, v5, v4
376; SI-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
377; SI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
378; SI-NEXT:    s_setpc_b64 s[30:31]
379;
380; GFX9-LABEL: mad_i64_i32_extops_i32_i64:
381; GFX9:       ; %bb.0:
382; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
383; GFX9-NEXT:    v_mov_b32_e32 v4, v1
384; GFX9-NEXT:    v_ashrrev_i32_e32 v5, 31, v0
385; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v4, v[2:3]
386; GFX9-NEXT:    v_mov_b32_e32 v2, v1
387; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v5, v4, v[2:3]
388; GFX9-NEXT:    v_mov_b32_e32 v1, v2
389; GFX9-NEXT:    s_setpc_b64 s[30:31]
390;
391; GFX11-LABEL: mad_i64_i32_extops_i32_i64:
392; GFX11:       ; %bb.0:
393; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
394; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
395; GFX11-NEXT:    v_mov_b32_e32 v4, v1
396; GFX11-NEXT:    v_mov_b32_e32 v5, v0
397; GFX11-NEXT:    v_mad_u64_u32 v[0:1], null, v5, v4, v[2:3]
398; GFX11-NEXT:    v_ashrrev_i32_e32 v5, 31, v5
399; GFX11-NEXT:    v_mov_b32_e32 v3, v1
400; GFX11-NEXT:    v_mad_u64_u32 v[1:2], null, v5, v4, v[3:4]
401; GFX11-NEXT:    s_setpc_b64 s[30:31]
402  %ext0 = sext i32 %arg0 to i64
403  %ext1 = zext i32 %arg1 to i64
404  %mul = mul i64 %ext0, %ext1
405  %mad = add i64 %mul, %arg2
406  ret i64 %mad
407}
408
409define i64 @mad_u64_u32_bitops(i64 %arg0, i64 %arg1, i64 %arg2) #0 {
410; CI-LABEL: mad_u64_u32_bitops:
411; CI:       ; %bb.0:
412; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
413; CI-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v2, v[4:5]
414; CI-NEXT:    s_setpc_b64 s[30:31]
415;
416; SI-LABEL: mad_u64_u32_bitops:
417; SI:       ; %bb.0:
418; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
419; SI-NEXT:    v_mul_lo_u32 v1, v0, v2
420; SI-NEXT:    v_mul_hi_u32 v2, v0, v2
421; SI-NEXT:    v_add_i32_e32 v0, vcc, v1, v4
422; SI-NEXT:    v_addc_u32_e32 v1, vcc, v2, v5, vcc
423; SI-NEXT:    s_setpc_b64 s[30:31]
424;
425; GFX9-LABEL: mad_u64_u32_bitops:
426; GFX9:       ; %bb.0:
427; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
428; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v2, v[4:5]
429; GFX9-NEXT:    s_setpc_b64 s[30:31]
430;
431; GFX11-LABEL: mad_u64_u32_bitops:
432; GFX11:       ; %bb.0:
433; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
434; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
435; GFX11-NEXT:    v_mov_b32_e32 v3, v0
436; GFX11-NEXT:    v_mad_u64_u32 v[0:1], null, v3, v2, v[4:5]
437; GFX11-NEXT:    s_setpc_b64 s[30:31]
438  %trunc.lhs = and i64 %arg0, 4294967295
439  %trunc.rhs = and i64 %arg1, 4294967295
440  %mul = mul i64 %trunc.lhs, %trunc.rhs
441  %add = add i64 %mul, %arg2
442  ret i64 %add
443}
444
445define i64 @mad_u64_u32_bitops_lhs_mask_small(i64 %arg0, i64 %arg1, i64 %arg2) #0 {
446; CI-LABEL: mad_u64_u32_bitops_lhs_mask_small:
447; CI:       ; %bb.0:
448; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
449; CI-NEXT:    v_and_b32_e32 v3, 1, v1
450; CI-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v2, v[4:5]
451; CI-NEXT:    v_mul_lo_u32 v2, v3, v2
452; CI-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
453; CI-NEXT:    s_setpc_b64 s[30:31]
454;
455; SI-LABEL: mad_u64_u32_bitops_lhs_mask_small:
456; SI:       ; %bb.0:
457; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
458; SI-NEXT:    v_and_b32_e32 v1, 1, v1
459; SI-NEXT:    v_mul_hi_u32 v3, v0, v2
460; SI-NEXT:    v_mul_lo_u32 v1, v1, v2
461; SI-NEXT:    v_mul_lo_u32 v0, v0, v2
462; SI-NEXT:    v_add_i32_e32 v1, vcc, v3, v1
463; SI-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
464; SI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v5, vcc
465; SI-NEXT:    s_setpc_b64 s[30:31]
466;
467; GFX9-LABEL: mad_u64_u32_bitops_lhs_mask_small:
468; GFX9:       ; %bb.0:
469; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
470; GFX9-NEXT:    v_and_b32_e32 v3, 1, v1
471; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v2, v[4:5]
472; GFX9-NEXT:    v_mov_b32_e32 v4, v1
473; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v3, v2, v[4:5]
474; GFX9-NEXT:    v_mov_b32_e32 v1, v2
475; GFX9-NEXT:    s_setpc_b64 s[30:31]
476;
477; GFX11-LABEL: mad_u64_u32_bitops_lhs_mask_small:
478; GFX11:       ; %bb.0:
479; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
480; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
481; GFX11-NEXT:    v_mov_b32_e32 v3, v2
482; GFX11-NEXT:    v_mov_b32_e32 v2, v0
483; GFX11-NEXT:    v_mov_b32_e32 v6, v1
484; GFX11-NEXT:    v_mad_u64_u32 v[0:1], null, v2, v3, v[4:5]
485; GFX11-NEXT:    v_and_b32_e32 v5, 1, v6
486; GFX11-NEXT:    v_mov_b32_e32 v4, v1
487; GFX11-NEXT:    v_mad_u64_u32 v[1:2], null, v5, v3, v[4:5]
488; GFX11-NEXT:    s_setpc_b64 s[30:31]
489  %trunc.lhs = and i64 %arg0, 8589934591
490  %trunc.rhs = and i64 %arg1, 4294967295
491  %mul = mul i64 %trunc.lhs, %trunc.rhs
492  %add = add i64 %mul, %arg2
493  ret i64 %add
494}
495
496define i64 @mad_u64_u32_bitops_rhs_mask_small(i64 %arg0, i64 %arg1, i64 %arg2) #0 {
497; CI-LABEL: mad_u64_u32_bitops_rhs_mask_small:
498; CI:       ; %bb.0:
499; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
500; CI-NEXT:    v_mov_b32_e32 v6, v0
501; CI-NEXT:    v_and_b32_e32 v3, 1, v3
502; CI-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v6, v2, v[4:5]
503; CI-NEXT:    v_mul_lo_u32 v2, v6, v3
504; CI-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
505; CI-NEXT:    s_setpc_b64 s[30:31]
506;
507; SI-LABEL: mad_u64_u32_bitops_rhs_mask_small:
508; SI:       ; %bb.0:
509; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
510; SI-NEXT:    v_and_b32_e32 v1, 1, v3
511; SI-NEXT:    v_mul_hi_u32 v3, v0, v2
512; SI-NEXT:    v_mul_lo_u32 v1, v0, v1
513; SI-NEXT:    v_mul_lo_u32 v0, v0, v2
514; SI-NEXT:    v_add_i32_e32 v1, vcc, v3, v1
515; SI-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
516; SI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v5, vcc
517; SI-NEXT:    s_setpc_b64 s[30:31]
518;
519; GFX9-LABEL: mad_u64_u32_bitops_rhs_mask_small:
520; GFX9:       ; %bb.0:
521; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
522; GFX9-NEXT:    v_mov_b32_e32 v6, v0
523; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v6, v2, v[4:5]
524; GFX9-NEXT:    v_and_b32_e32 v3, 1, v3
525; GFX9-NEXT:    v_mov_b32_e32 v2, v1
526; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v6, v3, v[2:3]
527; GFX9-NEXT:    v_mov_b32_e32 v1, v2
528; GFX9-NEXT:    s_setpc_b64 s[30:31]
529;
530; GFX11-LABEL: mad_u64_u32_bitops_rhs_mask_small:
531; GFX11:       ; %bb.0:
532; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
533; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
534; GFX11-NEXT:    v_mov_b32_e32 v6, v0
535; GFX11-NEXT:    v_mad_u64_u32 v[0:1], null, v6, v2, v[4:5]
536; GFX11-NEXT:    v_and_b32_e32 v4, 1, v3
537; GFX11-NEXT:    v_mov_b32_e32 v3, v1
538; GFX11-NEXT:    v_mad_u64_u32 v[1:2], null, v6, v4, v[3:4]
539; GFX11-NEXT:    s_setpc_b64 s[30:31]
540  %trunc.lhs = and i64 %arg0, 4294967295
541  %trunc.rhs = and i64 %arg1, 8589934591
542  %mul = mul i64 %trunc.lhs, %trunc.rhs
543  %add = add i64 %mul, %arg2
544  ret i64 %add
545}
546
547define i64 @mad_i64_i32_bitops(i64 %arg0, i64 %arg1, i64 %arg2) #0 {
548; CI-LABEL: mad_i64_i32_bitops:
549; CI:       ; %bb.0:
550; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
551; CI-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v2, v[4:5]
552; CI-NEXT:    s_setpc_b64 s[30:31]
553;
554; SI-LABEL: mad_i64_i32_bitops:
555; SI:       ; %bb.0:
556; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
557; SI-NEXT:    v_mul_lo_u32 v1, v0, v2
558; SI-NEXT:    v_mul_hi_i32 v2, v0, v2
559; SI-NEXT:    v_add_i32_e32 v0, vcc, v1, v4
560; SI-NEXT:    v_addc_u32_e32 v1, vcc, v2, v5, vcc
561; SI-NEXT:    s_setpc_b64 s[30:31]
562;
563; GFX9-LABEL: mad_i64_i32_bitops:
564; GFX9:       ; %bb.0:
565; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
566; GFX9-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v2, v[4:5]
567; GFX9-NEXT:    s_setpc_b64 s[30:31]
568;
569; GFX11-LABEL: mad_i64_i32_bitops:
570; GFX11:       ; %bb.0:
571; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
572; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
573; GFX11-NEXT:    v_mov_b32_e32 v3, v0
574; GFX11-NEXT:    v_mad_i64_i32 v[0:1], null, v3, v2, v[4:5]
575; GFX11-NEXT:    s_setpc_b64 s[30:31]
576  %shl.lhs = shl i64 %arg0, 32
577  %trunc.lhs = ashr i64 %shl.lhs, 32
578  %shl.rhs = shl i64 %arg1, 32
579  %trunc.rhs = ashr i64 %shl.rhs, 32
580  %mul = mul i64 %trunc.lhs, %trunc.rhs
581  %add = add i64 %mul, %arg2
582  ret i64 %add
583}
584
585; Example from bug report
586define i64 @mad_i64_i32_unpack_i64ops(i64 %arg0) #0 {
587; CI-LABEL: mad_i64_i32_unpack_i64ops:
588; CI:       ; %bb.0:
589; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
590; CI-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v1, v0, v[0:1]
591; CI-NEXT:    s_setpc_b64 s[30:31]
592;
593; SI-LABEL: mad_i64_i32_unpack_i64ops:
594; SI:       ; %bb.0:
595; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
596; SI-NEXT:    v_mul_lo_u32 v2, v1, v0
597; SI-NEXT:    v_mul_hi_u32 v3, v1, v0
598; SI-NEXT:    v_add_i32_e32 v0, vcc, v2, v0
599; SI-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
600; SI-NEXT:    s_setpc_b64 s[30:31]
601;
602; GFX9-LABEL: mad_i64_i32_unpack_i64ops:
603; GFX9:       ; %bb.0:
604; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
605; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v1, v0, v[0:1]
606; GFX9-NEXT:    s_setpc_b64 s[30:31]
607;
608; GFX11-LABEL: mad_i64_i32_unpack_i64ops:
609; GFX11:       ; %bb.0:
610; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
611; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
612; GFX11-NEXT:    v_mad_u64_u32 v[2:3], null, v1, v0, v[0:1]
613; GFX11-NEXT:    v_mov_b32_e32 v0, v2
614; GFX11-NEXT:    v_mov_b32_e32 v1, v3
615; GFX11-NEXT:    s_setpc_b64 s[30:31]
616  %tmp4 = lshr i64 %arg0, 32
617  %tmp5 = and i64 %arg0, 4294967295
618  %mul = mul nuw i64 %tmp4, %tmp5
619  %mad = add i64 %mul, %arg0
620  ret i64 %mad
621}
622
623define amdgpu_kernel void @mad_i64_i32_uniform(i64 addrspace(1)* %out, i32 %arg0, i32 %arg1, i64 %arg2) #0 {
624; CI-LABEL: mad_i64_i32_uniform:
625; CI:       ; %bb.0:
626; CI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xb
627; CI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
628; CI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
629; CI-NEXT:    s_waitcnt lgkmcnt(0)
630; CI-NEXT:    v_mov_b32_e32 v2, s3
631; CI-NEXT:    v_mov_b32_e32 v0, s4
632; CI-NEXT:    v_mov_b32_e32 v1, s5
633; CI-NEXT:    v_mad_u64_u32 v[0:1], s[2:3], s2, v2, v[0:1]
634; CI-NEXT:    s_mov_b32 s3, 0xf000
635; CI-NEXT:    s_mov_b32 s2, -1
636; CI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
637; CI-NEXT:    s_endpgm
638;
639; SI-LABEL: mad_i64_i32_uniform:
640; SI:       ; %bb.0:
641; SI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xb
642; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
643; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
644; SI-NEXT:    s_mov_b32 s7, 0xf000
645; SI-NEXT:    s_mov_b32 s6, -1
646; SI-NEXT:    s_waitcnt lgkmcnt(0)
647; SI-NEXT:    v_mov_b32_e32 v0, s3
648; SI-NEXT:    v_mul_hi_u32 v1, s2, v0
649; SI-NEXT:    s_mul_i32 s2, s2, s3
650; SI-NEXT:    v_mov_b32_e32 v0, s2
651; SI-NEXT:    v_mov_b32_e32 v2, s1
652; SI-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
653; SI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v2, vcc
654; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
655; SI-NEXT:    s_endpgm
656;
657; GFX9-LABEL: mad_i64_i32_uniform:
658; GFX9:       ; %bb.0:
659; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
660; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
661; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0x24
662; GFX9-NEXT:    v_mov_b32_e32 v2, 0
663; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
664; GFX9-NEXT:    s_mul_i32 s0, s2, s3
665; GFX9-NEXT:    s_mul_hi_u32 s1, s2, s3
666; GFX9-NEXT:    s_add_u32 s0, s0, s4
667; GFX9-NEXT:    s_addc_u32 s1, s1, s5
668; GFX9-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
669; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
670; GFX9-NEXT:    s_endpgm
671;
672; GFX11-LABEL: mad_i64_i32_uniform:
673; GFX11:       ; %bb.0:
674; GFX11-NEXT:    s_clause 0x2
675; GFX11-NEXT:    s_load_b64 s[2:3], s[0:1], 0x2c
676; GFX11-NEXT:    s_load_b64 s[4:5], s[0:1], 0x34
677; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
678; GFX11-NEXT:    v_mov_b32_e32 v2, 0
679; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
680; GFX11-NEXT:    s_mul_i32 s6, s2, s3
681; GFX11-NEXT:    s_mul_hi_u32 s3, s2, s3
682; GFX11-NEXT:    s_add_u32 s2, s6, s4
683; GFX11-NEXT:    s_addc_u32 s3, s3, s5
684; GFX11-NEXT:    v_mov_b32_e32 v0, s2
685; GFX11-NEXT:    v_mov_b32_e32 v1, s3
686; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
687; GFX11-NEXT:    s_endpgm
688  %ext0 = zext i32 %arg0 to i64
689  %ext1 = zext i32 %arg1 to i64
690  %mul = mul i64 %ext0, %ext1
691  %mad = add i64 %mul, %arg2
692  store i64 %mad, i64 addrspace(1)* %out
693  ret void
694}
695
696define i64 @mad_i64_i32_twice(i32 %arg0, i32 %arg1, i64 %arg2, i64 %arg3) #0 {
697; CI-LABEL: mad_i64_i32_twice:
698; CI:       ; %bb.0:
699; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
700; CI-NEXT:    v_mad_i64_i32 v[2:3], s[4:5], v0, v1, v[2:3]
701; CI-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[4:5]
702; CI-NEXT:    v_xor_b32_e32 v1, v3, v1
703; CI-NEXT:    v_xor_b32_e32 v0, v2, v0
704; CI-NEXT:    s_setpc_b64 s[30:31]
705;
706; SI-LABEL: mad_i64_i32_twice:
707; SI:       ; %bb.0:
708; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
709; SI-NEXT:    v_mul_lo_u32 v6, v0, v1
710; SI-NEXT:    v_mul_hi_i32 v0, v0, v1
711; SI-NEXT:    v_add_i32_e32 v2, vcc, v6, v2
712; SI-NEXT:    v_addc_u32_e32 v1, vcc, v0, v3, vcc
713; SI-NEXT:    v_add_i32_e32 v3, vcc, v6, v4
714; SI-NEXT:    v_addc_u32_e32 v0, vcc, v0, v5, vcc
715; SI-NEXT:    v_xor_b32_e32 v1, v1, v0
716; SI-NEXT:    v_xor_b32_e32 v0, v2, v3
717; SI-NEXT:    s_setpc_b64 s[30:31]
718;
719; GFX9-LABEL: mad_i64_i32_twice:
720; GFX9:       ; %bb.0:
721; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
722; GFX9-NEXT:    v_mad_i64_i32 v[2:3], s[4:5], v0, v1, v[2:3]
723; GFX9-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[4:5]
724; GFX9-NEXT:    v_xor_b32_e32 v1, v3, v1
725; GFX9-NEXT:    v_xor_b32_e32 v0, v2, v0
726; GFX9-NEXT:    s_setpc_b64 s[30:31]
727;
728; GFX11-LABEL: mad_i64_i32_twice:
729; GFX11:       ; %bb.0:
730; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
731; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
732; GFX11-NEXT:    v_mad_i64_i32 v[6:7], null, v0, v1, v[2:3]
733; GFX11-NEXT:    v_mad_i64_i32 v[2:3], null, v0, v1, v[4:5]
734; GFX11-NEXT:    v_xor_b32_e32 v0, v6, v2
735; GFX11-NEXT:    v_xor_b32_e32 v1, v7, v3
736; GFX11-NEXT:    s_setpc_b64 s[30:31]
737  %sext0 = sext i32 %arg0 to i64
738  %sext1 = sext i32 %arg1 to i64
739  %mul = mul i64 %sext0, %sext1
740  %mad1 = add i64 %mul, %arg2
741  %mad2 = add i64 %mul, %arg3
742  %out = xor i64 %mad1, %mad2
743  ret i64 %out
744}
745
746define i64 @mad_i64_i32_thrice(i32 %arg0, i32 %arg1, i64 %arg2, i64 %arg3, i64 %arg4) #0 {
747; CI-LABEL: mad_i64_i32_thrice:
748; CI:       ; %bb.0:
749; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
750; CI-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, 0
751; CI-NEXT:    v_add_i32_e32 v2, vcc, v0, v2
752; CI-NEXT:    v_addc_u32_e32 v3, vcc, v1, v3, vcc
753; CI-NEXT:    v_add_i32_e32 v4, vcc, v0, v4
754; CI-NEXT:    v_addc_u32_e32 v5, vcc, v1, v5, vcc
755; CI-NEXT:    v_add_i32_e32 v0, vcc, v0, v6
756; CI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v7, vcc
757; CI-NEXT:    v_xor_b32_e32 v3, v3, v5
758; CI-NEXT:    v_xor_b32_e32 v2, v2, v4
759; CI-NEXT:    v_xor_b32_e32 v1, v3, v1
760; CI-NEXT:    v_xor_b32_e32 v0, v2, v0
761; CI-NEXT:    s_setpc_b64 s[30:31]
762;
763; SI-LABEL: mad_i64_i32_thrice:
764; SI:       ; %bb.0:
765; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
766; SI-NEXT:    v_mul_lo_u32 v8, v0, v1
767; SI-NEXT:    v_mul_hi_i32 v0, v0, v1
768; SI-NEXT:    v_add_i32_e32 v1, vcc, v8, v2
769; SI-NEXT:    v_addc_u32_e32 v2, vcc, v0, v3, vcc
770; SI-NEXT:    v_add_i32_e32 v3, vcc, v8, v4
771; SI-NEXT:    v_addc_u32_e32 v4, vcc, v0, v5, vcc
772; SI-NEXT:    v_add_i32_e32 v5, vcc, v8, v6
773; SI-NEXT:    v_addc_u32_e32 v0, vcc, v0, v7, vcc
774; SI-NEXT:    v_xor_b32_e32 v2, v2, v4
775; SI-NEXT:    v_xor_b32_e32 v3, v1, v3
776; SI-NEXT:    v_xor_b32_e32 v1, v2, v0
777; SI-NEXT:    v_xor_b32_e32 v0, v3, v5
778; SI-NEXT:    s_setpc_b64 s[30:31]
779;
780; GFX9-LABEL: mad_i64_i32_thrice:
781; GFX9:       ; %bb.0:
782; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
783; GFX9-NEXT:    v_mad_i64_i32 v[2:3], s[4:5], v0, v1, v[2:3]
784; GFX9-NEXT:    v_mad_i64_i32 v[4:5], s[4:5], v0, v1, v[4:5]
785; GFX9-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[6:7]
786; GFX9-NEXT:    v_xor_b32_e32 v3, v3, v5
787; GFX9-NEXT:    v_xor_b32_e32 v2, v2, v4
788; GFX9-NEXT:    v_xor_b32_e32 v1, v3, v1
789; GFX9-NEXT:    v_xor_b32_e32 v0, v2, v0
790; GFX9-NEXT:    s_setpc_b64 s[30:31]
791;
792; GFX11-LABEL: mad_i64_i32_thrice:
793; GFX11:       ; %bb.0:
794; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
795; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
796; GFX11-NEXT:    v_mad_i64_i32 v[8:9], null, v0, v1, 0
797; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v8, v2
798; GFX11-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v9, v3, vcc_lo
799; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v8, v4
800; GFX11-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, v9, v5, vcc_lo
801; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v8, v6
802; GFX11-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, v9, v7, vcc_lo
803; GFX11-NEXT:    v_xor_b32_e32 v0, v0, v2
804; GFX11-NEXT:    v_xor_b32_e32 v1, v1, v3
805; GFX11-NEXT:    v_xor_b32_e32 v0, v0, v4
806; GFX11-NEXT:    v_xor_b32_e32 v1, v1, v5
807; GFX11-NEXT:    s_setpc_b64 s[30:31]
808  %sext0 = sext i32 %arg0 to i64
809  %sext1 = sext i32 %arg1 to i64
810  %mul = mul i64 %sext0, %sext1
811  %mad1 = add i64 %mul, %arg2
812  %mad2 = add i64 %mul, %arg3
813  %mad3 = add i64 %mul, %arg4
814  %out.p = xor i64 %mad1, %mad2
815  %out = xor i64 %out.p, %mad3
816  ret i64 %out
817}
818
819define i64 @mad_i64_i32_secondary_use(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
820; CI-LABEL: mad_i64_i32_secondary_use:
821; CI:       ; %bb.0:
822; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
823; CI-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, 0
824; CI-NEXT:    v_add_i32_e32 v2, vcc, v0, v2
825; CI-NEXT:    v_addc_u32_e32 v3, vcc, v1, v3, vcc
826; CI-NEXT:    v_xor_b32_e32 v1, v3, v1
827; CI-NEXT:    v_xor_b32_e32 v0, v2, v0
828; CI-NEXT:    s_setpc_b64 s[30:31]
829;
830; SI-LABEL: mad_i64_i32_secondary_use:
831; SI:       ; %bb.0:
832; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
833; SI-NEXT:    v_mul_lo_u32 v4, v0, v1
834; SI-NEXT:    v_mul_hi_i32 v0, v0, v1
835; SI-NEXT:    v_add_i32_e32 v2, vcc, v4, v2
836; SI-NEXT:    v_addc_u32_e32 v1, vcc, v0, v3, vcc
837; SI-NEXT:    v_xor_b32_e32 v1, v1, v0
838; SI-NEXT:    v_xor_b32_e32 v0, v2, v4
839; SI-NEXT:    s_setpc_b64 s[30:31]
840;
841; GFX9-LABEL: mad_i64_i32_secondary_use:
842; GFX9:       ; %bb.0:
843; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
844; GFX9-NEXT:    v_mad_i64_i32 v[4:5], s[4:5], v0, v1, 0
845; GFX9-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3]
846; GFX9-NEXT:    v_xor_b32_e32 v1, v1, v5
847; GFX9-NEXT:    v_xor_b32_e32 v0, v0, v4
848; GFX9-NEXT:    s_setpc_b64 s[30:31]
849;
850; GFX11-LABEL: mad_i64_i32_secondary_use:
851; GFX11:       ; %bb.0:
852; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
853; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
854; GFX11-NEXT:    v_mad_i64_i32 v[4:5], null, v0, v1, 0
855; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v4, v2
856; GFX11-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v5, v3, vcc_lo
857; GFX11-NEXT:    v_xor_b32_e32 v0, v0, v4
858; GFX11-NEXT:    v_xor_b32_e32 v1, v1, v5
859; GFX11-NEXT:    s_setpc_b64 s[30:31]
860  %sext0 = sext i32 %arg0 to i64
861  %sext1 = sext i32 %arg1 to i64
862  %mul = mul i64 %sext0, %sext1
863  %mad = add i64 %mul, %arg2
864  %out = xor i64 %mad, %mul
865  ret i64 %out
866}
867
868define i48 @mad_i48_i48(i48 %arg0, i48 %arg1, i48 %arg2) #0 {
869; CI-LABEL: mad_i48_i48:
870; CI:       ; %bb.0:
871; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
872; CI-NEXT:    v_mov_b32_e32 v6, v1
873; CI-NEXT:    v_mov_b32_e32 v7, v0
874; CI-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v7, v2, v[4:5]
875; CI-NEXT:    v_mul_lo_u32 v2, v6, v2
876; CI-NEXT:    v_mul_lo_u32 v3, v7, v3
877; CI-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
878; CI-NEXT:    v_add_i32_e32 v1, vcc, v3, v1
879; CI-NEXT:    s_setpc_b64 s[30:31]
880;
881; SI-LABEL: mad_i48_i48:
882; SI:       ; %bb.0:
883; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
884; SI-NEXT:    v_mul_lo_u32 v3, v0, v3
885; SI-NEXT:    v_mul_hi_u32 v6, v0, v2
886; SI-NEXT:    v_mul_lo_u32 v1, v1, v2
887; SI-NEXT:    v_mul_lo_u32 v0, v0, v2
888; SI-NEXT:    v_add_i32_e32 v3, vcc, v6, v3
889; SI-NEXT:    v_add_i32_e32 v1, vcc, v3, v1
890; SI-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
891; SI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v5, vcc
892; SI-NEXT:    s_setpc_b64 s[30:31]
893;
894; GFX9-LABEL: mad_i48_i48:
895; GFX9:       ; %bb.0:
896; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
897; GFX9-NEXT:    v_mov_b32_e32 v6, v1
898; GFX9-NEXT:    v_mov_b32_e32 v7, v0
899; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v7, v2, v[4:5]
900; GFX9-NEXT:    v_mul_lo_u32 v3, v7, v3
901; GFX9-NEXT:    v_mul_lo_u32 v2, v6, v2
902; GFX9-NEXT:    v_add3_u32 v1, v2, v1, v3
903; GFX9-NEXT:    s_setpc_b64 s[30:31]
904;
905; GFX11-LABEL: mad_i48_i48:
906; GFX11:       ; %bb.0:
907; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
908; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
909; GFX11-NEXT:    v_mov_b32_e32 v6, v1
910; GFX11-NEXT:    v_mov_b32_e32 v7, v0
911; GFX11-NEXT:    v_mad_u64_u32 v[0:1], null, v7, v2, v[4:5]
912; GFX11-NEXT:    v_mul_lo_u32 v3, v7, v3
913; GFX11-NEXT:    v_mul_lo_u32 v2, v6, v2
914; GFX11-NEXT:    v_add3_u32 v1, v2, v1, v3
915; GFX11-NEXT:    s_setpc_b64 s[30:31]
916  %m = mul i48 %arg0, %arg1
917  %a = add i48 %m, %arg2
918  ret i48 %a
919}
920
921attributes #0 = { nounwind }
922attributes #1 = { nounwind readnone speculatable }
923