1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -march=amdgcn -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -check-prefixes=CI %s
3; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefixes=SI %s
4; RUN: llc -march=amdgcn -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9 %s
5
6define i64 @mad_i64_i32_sextops(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
7; CI-LABEL: mad_i64_i32_sextops:
8; CI:       ; %bb.0:
9; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10; CI-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3]
11; CI-NEXT:    s_setpc_b64 s[30:31]
12;
13; SI-LABEL: mad_i64_i32_sextops:
14; SI:       ; %bb.0:
15; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16; SI-NEXT:    v_mul_lo_u32 v4, v0, v1
17; SI-NEXT:    v_mul_hi_i32 v1, v0, v1
18; SI-NEXT:    v_add_i32_e32 v0, vcc, v4, v2
19; SI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
20; SI-NEXT:    s_setpc_b64 s[30:31]
21;
22; GFX9-LABEL: mad_i64_i32_sextops:
23; GFX9:       ; %bb.0:
24; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
25; GFX9-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3]
26; GFX9-NEXT:    s_setpc_b64 s[30:31]
27  %sext0 = sext i32 %arg0 to i64
28  %sext1 = sext i32 %arg1 to i64
29  %mul = mul i64 %sext0, %sext1
30  %mad = add i64 %mul, %arg2
31  ret i64 %mad
32}
33
34define i64 @mad_i64_i32_sextops_commute(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
35; CI-LABEL: mad_i64_i32_sextops_commute:
36; CI:       ; %bb.0:
37; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
38; CI-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3]
39; CI-NEXT:    s_setpc_b64 s[30:31]
40;
41; SI-LABEL: mad_i64_i32_sextops_commute:
42; SI:       ; %bb.0:
43; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
44; SI-NEXT:    v_mul_lo_u32 v4, v0, v1
45; SI-NEXT:    v_mul_hi_i32 v1, v0, v1
46; SI-NEXT:    v_add_i32_e32 v0, vcc, v2, v4
47; SI-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
48; SI-NEXT:    s_setpc_b64 s[30:31]
49;
50; GFX9-LABEL: mad_i64_i32_sextops_commute:
51; GFX9:       ; %bb.0:
52; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
53; GFX9-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3]
54; GFX9-NEXT:    s_setpc_b64 s[30:31]
55  %sext0 = sext i32 %arg0 to i64
56  %sext1 = sext i32 %arg1 to i64
57  %mul = mul i64 %sext0, %sext1
58  %mad = add i64 %arg2, %mul
59  ret i64 %mad
60}
61
62define i64 @mad_u64_u32_zextops(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
63; CI-LABEL: mad_u64_u32_zextops:
64; CI:       ; %bb.0:
65; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
66; CI-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[2:3]
67; CI-NEXT:    s_setpc_b64 s[30:31]
68;
69; SI-LABEL: mad_u64_u32_zextops:
70; SI:       ; %bb.0:
71; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
72; SI-NEXT:    v_mul_lo_u32 v4, v0, v1
73; SI-NEXT:    v_mul_hi_u32 v1, v0, v1
74; SI-NEXT:    v_add_i32_e32 v0, vcc, v4, v2
75; SI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
76; SI-NEXT:    s_setpc_b64 s[30:31]
77;
78; GFX9-LABEL: mad_u64_u32_zextops:
79; GFX9:       ; %bb.0:
80; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
81; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[2:3]
82; GFX9-NEXT:    s_setpc_b64 s[30:31]
83  %sext0 = zext i32 %arg0 to i64
84  %sext1 = zext i32 %arg1 to i64
85  %mul = mul i64 %sext0, %sext1
86  %mad = add i64 %mul, %arg2
87  ret i64 %mad
88}
89
90define i64 @mad_u64_u32_zextops_commute(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
91; CI-LABEL: mad_u64_u32_zextops_commute:
92; CI:       ; %bb.0:
93; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
94; CI-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[2:3]
95; CI-NEXT:    s_setpc_b64 s[30:31]
96;
97; SI-LABEL: mad_u64_u32_zextops_commute:
98; SI:       ; %bb.0:
99; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
100; SI-NEXT:    v_mul_lo_u32 v4, v0, v1
101; SI-NEXT:    v_mul_hi_u32 v1, v0, v1
102; SI-NEXT:    v_add_i32_e32 v0, vcc, v2, v4
103; SI-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
104; SI-NEXT:    s_setpc_b64 s[30:31]
105;
106; GFX9-LABEL: mad_u64_u32_zextops_commute:
107; GFX9:       ; %bb.0:
108; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
109; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[2:3]
110; GFX9-NEXT:    s_setpc_b64 s[30:31]
111  %sext0 = zext i32 %arg0 to i64
112  %sext1 = zext i32 %arg1 to i64
113  %mul = mul i64 %sext0, %sext1
114  %mad = add i64 %arg2, %mul
115  ret i64 %mad
116}
117
118define i128 @mad_i64_i32_sextops_i32_i128(i32 %arg0, i32 %arg1, i128 %arg2) #0 {
119; CI-LABEL: mad_i64_i32_sextops_i32_i128:
120; CI:       ; %bb.0:
121; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
122; CI-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v0, v1, 0
123; CI-NEXT:    v_ashrrev_i32_e32 v13, 31, v0
124; CI-NEXT:    v_mov_b32_e32 v8, 0
125; CI-NEXT:    v_mad_u64_u32 v[9:10], s[4:5], v13, v1, v[7:8]
126; CI-NEXT:    v_ashrrev_i32_e32 v14, 31, v1
127; CI-NEXT:    v_mad_i64_i32 v[11:12], s[4:5], v1, v13, 0
128; CI-NEXT:    v_mov_b32_e32 v7, v10
129; CI-NEXT:    v_mov_b32_e32 v10, v8
130; CI-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v0, v14, v[9:10]
131; CI-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v14, v0, v[11:12]
132; CI-NEXT:    v_add_i32_e32 v9, vcc, v7, v9
133; CI-NEXT:    v_addc_u32_e64 v10, s[4:5], 0, 0, vcc
134; CI-NEXT:    v_mad_u64_u32 v[9:10], s[4:5], v13, v14, v[9:10]
135; CI-NEXT:    v_add_i32_e32 v7, vcc, v9, v0
136; CI-NEXT:    v_addc_u32_e32 v9, vcc, v10, v1, vcc
137; CI-NEXT:    v_mov_b32_e32 v1, v8
138; CI-NEXT:    v_add_i32_e32 v0, vcc, v6, v2
139; CI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
140; CI-NEXT:    v_addc_u32_e32 v2, vcc, v7, v4, vcc
141; CI-NEXT:    v_addc_u32_e32 v3, vcc, v9, v5, vcc
142; CI-NEXT:    s_setpc_b64 s[30:31]
143;
144; SI-LABEL: mad_i64_i32_sextops_i32_i128:
145; SI:       ; %bb.0:
146; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
147; SI-NEXT:    v_ashrrev_i32_e32 v6, 31, v0
148; SI-NEXT:    v_mul_lo_u32 v11, v6, v1
149; SI-NEXT:    v_mul_hi_u32 v12, v0, v1
150; SI-NEXT:    v_ashrrev_i32_e32 v7, 31, v1
151; SI-NEXT:    v_mul_hi_u32 v14, v6, v1
152; SI-NEXT:    v_mul_lo_u32 v13, v0, v7
153; SI-NEXT:    v_mul_hi_u32 v10, v0, v7
154; SI-NEXT:    v_add_i32_e32 v12, vcc, v11, v12
155; SI-NEXT:    v_addc_u32_e32 v14, vcc, 0, v14, vcc
156; SI-NEXT:    v_mul_hi_u32 v8, v6, v7
157; SI-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
158; SI-NEXT:    v_addc_u32_e32 v10, vcc, 0, v10, vcc
159; SI-NEXT:    v_mul_i32_i24_e32 v9, v6, v7
160; SI-NEXT:    v_add_i32_e32 v10, vcc, v14, v10
161; SI-NEXT:    v_mul_hi_i32 v6, v1, v6
162; SI-NEXT:    v_mul_hi_i32 v7, v7, v0
163; SI-NEXT:    v_addc_u32_e64 v14, s[4:5], 0, 0, vcc
164; SI-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
165; SI-NEXT:    v_addc_u32_e32 v8, vcc, v8, v14, vcc
166; SI-NEXT:    v_add_i32_e32 v10, vcc, v13, v11
167; SI-NEXT:    v_mul_lo_u32 v0, v0, v1
168; SI-NEXT:    v_addc_u32_e32 v6, vcc, v7, v6, vcc
169; SI-NEXT:    v_add_i32_e32 v7, vcc, v9, v10
170; SI-NEXT:    v_addc_u32_e32 v6, vcc, v8, v6, vcc
171; SI-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
172; SI-NEXT:    v_addc_u32_e32 v1, vcc, v12, v3, vcc
173; SI-NEXT:    v_addc_u32_e32 v2, vcc, v7, v4, vcc
174; SI-NEXT:    v_addc_u32_e32 v3, vcc, v6, v5, vcc
175; SI-NEXT:    s_setpc_b64 s[30:31]
176;
177; GFX9-LABEL: mad_i64_i32_sextops_i32_i128:
178; GFX9:       ; %bb.0:
179; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
180; GFX9-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v0, v1, 0
181; GFX9-NEXT:    v_ashrrev_i32_e32 v13, 31, v0
182; GFX9-NEXT:    v_mov_b32_e32 v9, 0
183; GFX9-NEXT:    v_mov_b32_e32 v8, v7
184; GFX9-NEXT:    v_mad_u64_u32 v[10:11], s[4:5], v13, v1, v[8:9]
185; GFX9-NEXT:    v_ashrrev_i32_e32 v14, 31, v1
186; GFX9-NEXT:    v_mov_b32_e32 v8, v11
187; GFX9-NEXT:    v_mov_b32_e32 v11, v9
188; GFX9-NEXT:    v_mad_u64_u32 v[10:11], s[4:5], v0, v14, v[10:11]
189; GFX9-NEXT:    v_mov_b32_e32 v12, v11
190; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, v8, v12
191; GFX9-NEXT:    v_addc_co_u32_e64 v9, s[4:5], 0, 0, vcc
192; GFX9-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v13, v14, v[8:9]
193; GFX9-NEXT:    v_mad_i64_i32 v[12:13], s[4:5], v1, v13, 0
194; GFX9-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v14, v0, v[12:13]
195; GFX9-NEXT:    v_add_co_u32_e32 v7, vcc, v8, v0
196; GFX9-NEXT:    v_addc_co_u32_e32 v8, vcc, v9, v1, vcc
197; GFX9-NEXT:    v_mov_b32_e32 v1, v10
198; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v6, v2
199; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
200; GFX9-NEXT:    v_addc_co_u32_e32 v2, vcc, v7, v4, vcc
201; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v8, v5, vcc
202; GFX9-NEXT:    s_setpc_b64 s[30:31]
203  %sext0 = sext i32 %arg0 to i128
204  %sext1 = sext i32 %arg1 to i128
205  %mul = mul i128 %sext0, %sext1
206  %mad = add i128 %mul, %arg2
207  ret i128 %mad
208}
209
210define i63 @mad_i64_i32_sextops_i32_i63(i32 %arg0, i32 %arg1, i63 %arg2) #0 {
211; CI-LABEL: mad_i64_i32_sextops_i32_i63:
212; CI:       ; %bb.0:
213; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
214; CI-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3]
215; CI-NEXT:    s_setpc_b64 s[30:31]
216;
217; SI-LABEL: mad_i64_i32_sextops_i32_i63:
218; SI:       ; %bb.0:
219; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
220; SI-NEXT:    v_mul_lo_u32 v4, v0, v1
221; SI-NEXT:    v_mul_hi_i32 v1, v0, v1
222; SI-NEXT:    v_add_i32_e32 v0, vcc, v4, v2
223; SI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
224; SI-NEXT:    s_setpc_b64 s[30:31]
225;
226; GFX9-LABEL: mad_i64_i32_sextops_i32_i63:
227; GFX9:       ; %bb.0:
228; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
229; GFX9-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3]
230; GFX9-NEXT:    s_setpc_b64 s[30:31]
231  %sext0 = sext i32 %arg0 to i63
232  %sext1 = sext i32 %arg1 to i63
233  %mul = mul i63 %sext0, %sext1
234  %mad = add i63 %mul, %arg2
235  ret i63 %mad
236}
237
238define i63 @mad_i64_i32_sextops_i31_i63(i31 %arg0, i31 %arg1, i63 %arg2) #0 {
239; CI-LABEL: mad_i64_i32_sextops_i31_i63:
240; CI:       ; %bb.0:
241; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
242; CI-NEXT:    v_bfe_i32 v1, v1, 0, 31
243; CI-NEXT:    v_bfe_i32 v0, v0, 0, 31
244; CI-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3]
245; CI-NEXT:    s_setpc_b64 s[30:31]
246;
247; SI-LABEL: mad_i64_i32_sextops_i31_i63:
248; SI:       ; %bb.0:
249; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
250; SI-NEXT:    v_lshlrev_b32_e32 v4, 1, v0
251; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v1
252; SI-NEXT:    v_ashr_i64 v[4:5], v[3:4], 33
253; SI-NEXT:    v_ashr_i64 v[0:1], v[0:1], 33
254; SI-NEXT:    v_mul_lo_u32 v1, v4, v0
255; SI-NEXT:    v_mul_hi_i32 v4, v4, v0
256; SI-NEXT:    v_add_i32_e32 v0, vcc, v1, v2
257; SI-NEXT:    v_addc_u32_e32 v1, vcc, v4, v3, vcc
258; SI-NEXT:    s_setpc_b64 s[30:31]
259;
260; GFX9-LABEL: mad_i64_i32_sextops_i31_i63:
261; GFX9:       ; %bb.0:
262; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
263; GFX9-NEXT:    v_bfe_i32 v1, v1, 0, 31
264; GFX9-NEXT:    v_bfe_i32 v0, v0, 0, 31
265; GFX9-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3]
266; GFX9-NEXT:    s_setpc_b64 s[30:31]
267  %sext0 = sext i31 %arg0 to i63
268  %sext1 = sext i31 %arg1 to i63
269  %mul = mul i63 %sext0, %sext1
270  %mad = add i63 %mul, %arg2
271  ret i63 %mad
272}
273
274define i64 @mad_i64_i32_extops_i32_i64(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
275; CI-LABEL: mad_i64_i32_extops_i32_i64:
276; CI:       ; %bb.0:
277; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
278; CI-NEXT:    v_ashrrev_i32_e32 v4, 31, v0
279; CI-NEXT:    v_mul_lo_u32 v4, v4, v1
280; CI-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[2:3]
281; CI-NEXT:    v_add_i32_e32 v1, vcc, v4, v1
282; CI-NEXT:    s_setpc_b64 s[30:31]
283;
284; SI-LABEL: mad_i64_i32_extops_i32_i64:
285; SI:       ; %bb.0:
286; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
287; SI-NEXT:    v_ashrrev_i32_e32 v4, 31, v0
288; SI-NEXT:    v_mul_hi_u32 v5, v0, v1
289; SI-NEXT:    v_mul_lo_u32 v4, v4, v1
290; SI-NEXT:    v_mul_lo_u32 v0, v0, v1
291; SI-NEXT:    v_add_i32_e32 v1, vcc, v5, v4
292; SI-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
293; SI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
294; SI-NEXT:    s_setpc_b64 s[30:31]
295;
296; GFX9-LABEL: mad_i64_i32_extops_i32_i64:
297; GFX9:       ; %bb.0:
298; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
299; GFX9-NEXT:    v_ashrrev_i32_e32 v4, 31, v0
300; GFX9-NEXT:    v_mul_lo_u32 v4, v4, v1
301; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[2:3]
302; GFX9-NEXT:    v_add_u32_e32 v1, v4, v1
303; GFX9-NEXT:    s_setpc_b64 s[30:31]
304  %ext0 = sext i32 %arg0 to i64
305  %ext1 = zext i32 %arg1 to i64
306  %mul = mul i64 %ext0, %ext1
307  %mad = add i64 %mul, %arg2
308  ret i64 %mad
309}
310
311define i64 @mad_u64_u32_bitops(i64 %arg0, i64 %arg1, i64 %arg2) #0 {
312; CI-LABEL: mad_u64_u32_bitops:
313; CI:       ; %bb.0:
314; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
315; CI-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v2, v[4:5]
316; CI-NEXT:    s_setpc_b64 s[30:31]
317;
318; SI-LABEL: mad_u64_u32_bitops:
319; SI:       ; %bb.0:
320; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
321; SI-NEXT:    v_mul_lo_u32 v1, v0, v2
322; SI-NEXT:    v_mul_hi_u32 v2, v0, v2
323; SI-NEXT:    v_add_i32_e32 v0, vcc, v1, v4
324; SI-NEXT:    v_addc_u32_e32 v1, vcc, v2, v5, vcc
325; SI-NEXT:    s_setpc_b64 s[30:31]
326;
327; GFX9-LABEL: mad_u64_u32_bitops:
328; GFX9:       ; %bb.0:
329; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
330; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v2, v[4:5]
331; GFX9-NEXT:    s_setpc_b64 s[30:31]
332  %trunc.lhs = and i64 %arg0, 4294967295
333  %trunc.rhs = and i64 %arg1, 4294967295
334  %mul = mul i64 %trunc.lhs, %trunc.rhs
335  %add = add i64 %mul, %arg2
336  ret i64 %add
337}
338
339define i64 @mad_u64_u32_bitops_lhs_mask_small(i64 %arg0, i64 %arg1, i64 %arg2) #0 {
340; CI-LABEL: mad_u64_u32_bitops_lhs_mask_small:
341; CI:       ; %bb.0:
342; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
343; CI-NEXT:    v_and_b32_e32 v3, 1, v1
344; CI-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v2, v[4:5]
345; CI-NEXT:    v_mul_lo_u32 v2, v3, v2
346; CI-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
347; CI-NEXT:    s_setpc_b64 s[30:31]
348;
349; SI-LABEL: mad_u64_u32_bitops_lhs_mask_small:
350; SI:       ; %bb.0:
351; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
352; SI-NEXT:    v_and_b32_e32 v1, 1, v1
353; SI-NEXT:    v_mul_hi_u32 v3, v0, v2
354; SI-NEXT:    v_mul_lo_u32 v1, v1, v2
355; SI-NEXT:    v_mul_lo_u32 v0, v0, v2
356; SI-NEXT:    v_add_i32_e32 v1, vcc, v3, v1
357; SI-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
358; SI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v5, vcc
359; SI-NEXT:    s_setpc_b64 s[30:31]
360;
361; GFX9-LABEL: mad_u64_u32_bitops_lhs_mask_small:
362; GFX9:       ; %bb.0:
363; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
364; GFX9-NEXT:    v_and_b32_e32 v3, 1, v1
365; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v2, v[4:5]
366; GFX9-NEXT:    v_mul_lo_u32 v2, v3, v2
367; GFX9-NEXT:    v_add_u32_e32 v1, v2, v1
368; GFX9-NEXT:    s_setpc_b64 s[30:31]
369  %trunc.lhs = and i64 %arg0, 8589934591
370  %trunc.rhs = and i64 %arg1, 4294967295
371  %mul = mul i64 %trunc.lhs, %trunc.rhs
372  %add = add i64 %mul, %arg2
373  ret i64 %add
374}
375
376define i64 @mad_u64_u32_bitops_rhs_mask_small(i64 %arg0, i64 %arg1, i64 %arg2) #0 {
377; CI-LABEL: mad_u64_u32_bitops_rhs_mask_small:
378; CI:       ; %bb.0:
379; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
380; CI-NEXT:    v_mov_b32_e32 v6, v0
381; CI-NEXT:    v_and_b32_e32 v3, 1, v3
382; CI-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v6, v2, v[4:5]
383; CI-NEXT:    v_mul_lo_u32 v2, v6, v3
384; CI-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
385; CI-NEXT:    s_setpc_b64 s[30:31]
386;
387; SI-LABEL: mad_u64_u32_bitops_rhs_mask_small:
388; SI:       ; %bb.0:
389; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
390; SI-NEXT:    v_and_b32_e32 v1, 1, v3
391; SI-NEXT:    v_mul_hi_u32 v3, v0, v2
392; SI-NEXT:    v_mul_lo_u32 v1, v0, v1
393; SI-NEXT:    v_mul_lo_u32 v0, v0, v2
394; SI-NEXT:    v_add_i32_e32 v1, vcc, v3, v1
395; SI-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
396; SI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v5, vcc
397; SI-NEXT:    s_setpc_b64 s[30:31]
398;
399; GFX9-LABEL: mad_u64_u32_bitops_rhs_mask_small:
400; GFX9:       ; %bb.0:
401; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
402; GFX9-NEXT:    v_mov_b32_e32 v6, v0
403; GFX9-NEXT:    v_and_b32_e32 v3, 1, v3
404; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v6, v2, v[4:5]
405; GFX9-NEXT:    v_mul_lo_u32 v2, v6, v3
406; GFX9-NEXT:    v_add_u32_e32 v1, v2, v1
407; GFX9-NEXT:    s_setpc_b64 s[30:31]
408  %trunc.lhs = and i64 %arg0, 4294967295
409  %trunc.rhs = and i64 %arg1, 8589934591
410  %mul = mul i64 %trunc.lhs, %trunc.rhs
411  %add = add i64 %mul, %arg2
412  ret i64 %add
413}
414
415define i64 @mad_i64_i32_bitops(i64 %arg0, i64 %arg1, i64 %arg2) #0 {
416; CI-LABEL: mad_i64_i32_bitops:
417; CI:       ; %bb.0:
418; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
419; CI-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v2, v[4:5]
420; CI-NEXT:    s_setpc_b64 s[30:31]
421;
422; SI-LABEL: mad_i64_i32_bitops:
423; SI:       ; %bb.0:
424; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
425; SI-NEXT:    v_mul_lo_u32 v1, v0, v2
426; SI-NEXT:    v_mul_hi_i32 v2, v0, v2
427; SI-NEXT:    v_add_i32_e32 v0, vcc, v1, v4
428; SI-NEXT:    v_addc_u32_e32 v1, vcc, v2, v5, vcc
429; SI-NEXT:    s_setpc_b64 s[30:31]
430;
431; GFX9-LABEL: mad_i64_i32_bitops:
432; GFX9:       ; %bb.0:
433; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
434; GFX9-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v2, v[4:5]
435; GFX9-NEXT:    s_setpc_b64 s[30:31]
436  %shl.lhs = shl i64 %arg0, 32
437  %trunc.lhs = ashr i64 %shl.lhs, 32
438  %shl.rhs = shl i64 %arg1, 32
439  %trunc.rhs = ashr i64 %shl.rhs, 32
440  %mul = mul i64 %trunc.lhs, %trunc.rhs
441  %add = add i64 %mul, %arg2
442  ret i64 %add
443}
444
445; Example from bug report
446define i64 @mad_i64_i32_unpack_i64ops(i64 %arg0) #0 {
447; CI-LABEL: mad_i64_i32_unpack_i64ops:
448; CI:       ; %bb.0:
449; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
450; CI-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v1, v0, v[0:1]
451; CI-NEXT:    s_setpc_b64 s[30:31]
452;
453; SI-LABEL: mad_i64_i32_unpack_i64ops:
454; SI:       ; %bb.0:
455; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
456; SI-NEXT:    v_mul_lo_u32 v2, v1, v0
457; SI-NEXT:    v_mul_hi_u32 v3, v1, v0
458; SI-NEXT:    v_add_i32_e32 v0, vcc, v2, v0
459; SI-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
460; SI-NEXT:    s_setpc_b64 s[30:31]
461;
462; GFX9-LABEL: mad_i64_i32_unpack_i64ops:
463; GFX9:       ; %bb.0:
464; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
465; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v1, v0, v[0:1]
466; GFX9-NEXT:    s_setpc_b64 s[30:31]
467  %tmp4 = lshr i64 %arg0, 32
468  %tmp5 = and i64 %arg0, 4294967295
469  %mul = mul nuw i64 %tmp4, %tmp5
470  %mad = add i64 %mul, %arg0
471  ret i64 %mad
472}
473
474define amdgpu_kernel void @mad_i64_i32_uniform(i64 addrspace(1)* %out, i32 %arg0, i32 %arg1, i64 %arg2) #0 {
475; CI-LABEL: mad_i64_i32_uniform:
476; CI:       ; %bb.0:
477; CI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xb
478; CI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
479; CI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
480; CI-NEXT:    s_waitcnt lgkmcnt(0)
481; CI-NEXT:    v_mov_b32_e32 v2, s3
482; CI-NEXT:    v_mov_b32_e32 v0, s4
483; CI-NEXT:    v_mov_b32_e32 v1, s5
484; CI-NEXT:    v_mad_u64_u32 v[0:1], s[2:3], s2, v2, v[0:1]
485; CI-NEXT:    s_mov_b32 s3, 0xf000
486; CI-NEXT:    s_mov_b32 s2, -1
487; CI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
488; CI-NEXT:    s_endpgm
489;
490; SI-LABEL: mad_i64_i32_uniform:
491; SI:       ; %bb.0:
492; SI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xb
493; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
494; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
495; SI-NEXT:    s_mov_b32 s7, 0xf000
496; SI-NEXT:    s_mov_b32 s6, -1
497; SI-NEXT:    s_waitcnt lgkmcnt(0)
498; SI-NEXT:    v_mov_b32_e32 v0, s3
499; SI-NEXT:    v_mul_hi_u32 v1, s2, v0
500; SI-NEXT:    s_mul_i32 s2, s2, s3
501; SI-NEXT:    v_mov_b32_e32 v0, s2
502; SI-NEXT:    v_mov_b32_e32 v2, s1
503; SI-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
504; SI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v2, vcc
505; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
506; SI-NEXT:    s_endpgm
507;
508; GFX9-LABEL: mad_i64_i32_uniform:
509; GFX9:       ; %bb.0:
510; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
511; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
512; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0x24
513; GFX9-NEXT:    v_mov_b32_e32 v2, 0
514; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
515; GFX9-NEXT:    s_mul_i32 s0, s2, s3
516; GFX9-NEXT:    s_mul_hi_u32 s1, s2, s3
517; GFX9-NEXT:    s_add_u32 s0, s0, s4
518; GFX9-NEXT:    s_addc_u32 s1, s1, s5
519; GFX9-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
520; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
521; GFX9-NEXT:    s_endpgm
522  %ext0 = zext i32 %arg0 to i64
523  %ext1 = zext i32 %arg1 to i64
524  %mul = mul i64 %ext0, %ext1
525  %mad = add i64 %mul, %arg2
526  store i64 %mad, i64 addrspace(1)* %out
527  ret void
528}
529
530define i64 @mad_i64_i32_twice(i32 %arg0, i32 %arg1, i64 %arg2, i64 %arg3) #0 {
531; CI-LABEL: mad_i64_i32_twice:
532; CI:       ; %bb.0:
533; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
534; CI-NEXT:    v_mad_i64_i32 v[2:3], s[4:5], v0, v1, v[2:3]
535; CI-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[4:5]
536; CI-NEXT:    v_xor_b32_e32 v1, v3, v1
537; CI-NEXT:    v_xor_b32_e32 v0, v2, v0
538; CI-NEXT:    s_setpc_b64 s[30:31]
539;
540; SI-LABEL: mad_i64_i32_twice:
541; SI:       ; %bb.0:
542; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
543; SI-NEXT:    v_mul_lo_u32 v6, v0, v1
544; SI-NEXT:    v_mul_hi_i32 v0, v0, v1
545; SI-NEXT:    v_add_i32_e32 v2, vcc, v6, v2
546; SI-NEXT:    v_addc_u32_e32 v1, vcc, v0, v3, vcc
547; SI-NEXT:    v_add_i32_e32 v3, vcc, v6, v4
548; SI-NEXT:    v_addc_u32_e32 v0, vcc, v0, v5, vcc
549; SI-NEXT:    v_xor_b32_e32 v1, v1, v0
550; SI-NEXT:    v_xor_b32_e32 v0, v2, v3
551; SI-NEXT:    s_setpc_b64 s[30:31]
552;
553; GFX9-LABEL: mad_i64_i32_twice:
554; GFX9:       ; %bb.0:
555; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
556; GFX9-NEXT:    v_mad_i64_i32 v[2:3], s[4:5], v0, v1, v[2:3]
557; GFX9-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[4:5]
558; GFX9-NEXT:    v_xor_b32_e32 v1, v3, v1
559; GFX9-NEXT:    v_xor_b32_e32 v0, v2, v0
560; GFX9-NEXT:    s_setpc_b64 s[30:31]
561  %sext0 = sext i32 %arg0 to i64
562  %sext1 = sext i32 %arg1 to i64
563  %mul = mul i64 %sext0, %sext1
564  %mad1 = add i64 %mul, %arg2
565  %mad2 = add i64 %mul, %arg3
566  %out = xor i64 %mad1, %mad2
567  ret i64 %out
568}
569
570define i64 @mad_i64_i32_thrice(i32 %arg0, i32 %arg1, i64 %arg2, i64 %arg3, i64 %arg4) #0 {
571; CI-LABEL: mad_i64_i32_thrice:
572; CI:       ; %bb.0:
573; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
574; CI-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, 0
575; CI-NEXT:    v_add_i32_e32 v2, vcc, v0, v2
576; CI-NEXT:    v_addc_u32_e32 v3, vcc, v1, v3, vcc
577; CI-NEXT:    v_add_i32_e32 v4, vcc, v0, v4
578; CI-NEXT:    v_addc_u32_e32 v5, vcc, v1, v5, vcc
579; CI-NEXT:    v_add_i32_e32 v0, vcc, v0, v6
580; CI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v7, vcc
581; CI-NEXT:    v_xor_b32_e32 v3, v3, v5
582; CI-NEXT:    v_xor_b32_e32 v2, v2, v4
583; CI-NEXT:    v_xor_b32_e32 v1, v3, v1
584; CI-NEXT:    v_xor_b32_e32 v0, v2, v0
585; CI-NEXT:    s_setpc_b64 s[30:31]
586;
587; SI-LABEL: mad_i64_i32_thrice:
588; SI:       ; %bb.0:
589; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
590; SI-NEXT:    v_mul_lo_u32 v8, v0, v1
591; SI-NEXT:    v_mul_hi_i32 v0, v0, v1
592; SI-NEXT:    v_add_i32_e32 v1, vcc, v8, v2
593; SI-NEXT:    v_addc_u32_e32 v2, vcc, v0, v3, vcc
594; SI-NEXT:    v_add_i32_e32 v3, vcc, v8, v4
595; SI-NEXT:    v_addc_u32_e32 v4, vcc, v0, v5, vcc
596; SI-NEXT:    v_add_i32_e32 v5, vcc, v8, v6
597; SI-NEXT:    v_addc_u32_e32 v0, vcc, v0, v7, vcc
598; SI-NEXT:    v_xor_b32_e32 v2, v2, v4
599; SI-NEXT:    v_xor_b32_e32 v3, v1, v3
600; SI-NEXT:    v_xor_b32_e32 v1, v2, v0
601; SI-NEXT:    v_xor_b32_e32 v0, v3, v5
602; SI-NEXT:    s_setpc_b64 s[30:31]
603;
604; GFX9-LABEL: mad_i64_i32_thrice:
605; GFX9:       ; %bb.0:
606; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
607; GFX9-NEXT:    v_mad_i64_i32 v[2:3], s[4:5], v0, v1, v[2:3]
608; GFX9-NEXT:    v_mad_i64_i32 v[4:5], s[4:5], v0, v1, v[4:5]
609; GFX9-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[6:7]
610; GFX9-NEXT:    v_xor_b32_e32 v3, v3, v5
611; GFX9-NEXT:    v_xor_b32_e32 v2, v2, v4
612; GFX9-NEXT:    v_xor_b32_e32 v1, v3, v1
613; GFX9-NEXT:    v_xor_b32_e32 v0, v2, v0
614; GFX9-NEXT:    s_setpc_b64 s[30:31]
615  %sext0 = sext i32 %arg0 to i64
616  %sext1 = sext i32 %arg1 to i64
617  %mul = mul i64 %sext0, %sext1
618  %mad1 = add i64 %mul, %arg2
619  %mad2 = add i64 %mul, %arg3
620  %mad3 = add i64 %mul, %arg4
621  %out.p = xor i64 %mad1, %mad2
622  %out = xor i64 %out.p, %mad3
623  ret i64 %out
624}
625
626define i64 @mad_i64_i32_secondary_use(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
627; CI-LABEL: mad_i64_i32_secondary_use:
628; CI:       ; %bb.0:
629; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
630; CI-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, 0
631; CI-NEXT:    v_add_i32_e32 v2, vcc, v0, v2
632; CI-NEXT:    v_addc_u32_e32 v3, vcc, v1, v3, vcc
633; CI-NEXT:    v_xor_b32_e32 v1, v3, v1
634; CI-NEXT:    v_xor_b32_e32 v0, v2, v0
635; CI-NEXT:    s_setpc_b64 s[30:31]
636;
637; SI-LABEL: mad_i64_i32_secondary_use:
638; SI:       ; %bb.0:
639; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
640; SI-NEXT:    v_mul_lo_u32 v4, v0, v1
641; SI-NEXT:    v_mul_hi_i32 v0, v0, v1
642; SI-NEXT:    v_add_i32_e32 v2, vcc, v4, v2
643; SI-NEXT:    v_addc_u32_e32 v1, vcc, v0, v3, vcc
644; SI-NEXT:    v_xor_b32_e32 v1, v1, v0
645; SI-NEXT:    v_xor_b32_e32 v0, v2, v4
646; SI-NEXT:    s_setpc_b64 s[30:31]
647;
648; GFX9-LABEL: mad_i64_i32_secondary_use:
649; GFX9:       ; %bb.0:
650; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
651; GFX9-NEXT:    v_mad_i64_i32 v[4:5], s[4:5], v0, v1, 0
652; GFX9-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3]
653; GFX9-NEXT:    v_xor_b32_e32 v1, v1, v5
654; GFX9-NEXT:    v_xor_b32_e32 v0, v0, v4
655; GFX9-NEXT:    s_setpc_b64 s[30:31]
656  %sext0 = sext i32 %arg0 to i64
657  %sext1 = sext i32 %arg1 to i64
658  %mul = mul i64 %sext0, %sext1
659  %mad = add i64 %mul, %arg2
660  %out = xor i64 %mad, %mul
661  ret i64 %out
662}
663
664define i48 @mad_i48_i48(i48 %arg0, i48 %arg1, i48 %arg2) #0 {
665; CI-LABEL: mad_i48_i48:
666; CI:       ; %bb.0:
667; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
668; CI-NEXT:    v_mov_b32_e32 v6, v1
669; CI-NEXT:    v_mov_b32_e32 v7, v0
670; CI-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v7, v2, v[4:5]
671; CI-NEXT:    v_mul_lo_u32 v2, v6, v2
672; CI-NEXT:    v_mul_lo_u32 v3, v7, v3
673; CI-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
674; CI-NEXT:    v_add_i32_e32 v1, vcc, v3, v1
675; CI-NEXT:    s_setpc_b64 s[30:31]
676;
677; SI-LABEL: mad_i48_i48:
678; SI:       ; %bb.0:
679; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
680; SI-NEXT:    v_mul_lo_u32 v3, v0, v3
681; SI-NEXT:    v_mul_hi_u32 v6, v0, v2
682; SI-NEXT:    v_mul_lo_u32 v1, v1, v2
683; SI-NEXT:    v_mul_lo_u32 v0, v0, v2
684; SI-NEXT:    v_add_i32_e32 v3, vcc, v6, v3
685; SI-NEXT:    v_add_i32_e32 v1, vcc, v3, v1
686; SI-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
687; SI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v5, vcc
688; SI-NEXT:    s_setpc_b64 s[30:31]
689;
690; GFX9-LABEL: mad_i48_i48:
691; GFX9:       ; %bb.0:
692; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
693; GFX9-NEXT:    v_mov_b32_e32 v6, v1
694; GFX9-NEXT:    v_mov_b32_e32 v7, v0
695; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v7, v2, v[4:5]
696; GFX9-NEXT:    v_mul_lo_u32 v3, v7, v3
697; GFX9-NEXT:    v_mul_lo_u32 v2, v6, v2
698; GFX9-NEXT:    v_add3_u32 v1, v2, v1, v3
699; GFX9-NEXT:    s_setpc_b64 s[30:31]
700  %m = mul i48 %arg0, %arg1
701  %a = add i48 %m, %arg2
702  ret i48 %a
703}
704
705attributes #0 = { nounwind }
706attributes #1 = { nounwind readnone speculatable }
707