1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -march=amdgcn -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -check-prefixes=CI %s
3; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefixes=SI %s
4; RUN: llc -march=amdgcn -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9 %s
5
6define i64 @mad_i64_i32_sextops(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
7; CI-LABEL: mad_i64_i32_sextops:
8; CI:       ; %bb.0:
9; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
10; CI-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3]
11; CI-NEXT:    s_setpc_b64 s[30:31]
12;
13; SI-LABEL: mad_i64_i32_sextops:
14; SI:       ; %bb.0:
15; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
16; SI-NEXT:    v_mul_lo_u32 v4, v0, v1
17; SI-NEXT:    v_mul_hi_i32 v1, v0, v1
18; SI-NEXT:    v_add_i32_e32 v0, vcc, v4, v2
19; SI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
20; SI-NEXT:    s_setpc_b64 s[30:31]
21;
22; GFX9-LABEL: mad_i64_i32_sextops:
23; GFX9:       ; %bb.0:
24; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
25; GFX9-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3]
26; GFX9-NEXT:    s_setpc_b64 s[30:31]
27  %sext0 = sext i32 %arg0 to i64
28  %sext1 = sext i32 %arg1 to i64
29  %mul = mul i64 %sext0, %sext1
30  %mad = add i64 %mul, %arg2
31  ret i64 %mad
32}
33
34define i64 @mad_i64_i32_sextops_commute(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
35; CI-LABEL: mad_i64_i32_sextops_commute:
36; CI:       ; %bb.0:
37; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
38; CI-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3]
39; CI-NEXT:    s_setpc_b64 s[30:31]
40;
41; SI-LABEL: mad_i64_i32_sextops_commute:
42; SI:       ; %bb.0:
43; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
44; SI-NEXT:    v_mul_lo_u32 v4, v0, v1
45; SI-NEXT:    v_mul_hi_i32 v1, v0, v1
46; SI-NEXT:    v_add_i32_e32 v0, vcc, v2, v4
47; SI-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
48; SI-NEXT:    s_setpc_b64 s[30:31]
49;
50; GFX9-LABEL: mad_i64_i32_sextops_commute:
51; GFX9:       ; %bb.0:
52; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
53; GFX9-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3]
54; GFX9-NEXT:    s_setpc_b64 s[30:31]
55  %sext0 = sext i32 %arg0 to i64
56  %sext1 = sext i32 %arg1 to i64
57  %mul = mul i64 %sext0, %sext1
58  %mad = add i64 %arg2, %mul
59  ret i64 %mad
60}
61
62define i64 @mad_u64_u32_zextops(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
63; CI-LABEL: mad_u64_u32_zextops:
64; CI:       ; %bb.0:
65; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
66; CI-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[2:3]
67; CI-NEXT:    s_setpc_b64 s[30:31]
68;
69; SI-LABEL: mad_u64_u32_zextops:
70; SI:       ; %bb.0:
71; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
72; SI-NEXT:    v_mul_lo_u32 v4, v0, v1
73; SI-NEXT:    v_mul_hi_u32 v1, v0, v1
74; SI-NEXT:    v_add_i32_e32 v0, vcc, v4, v2
75; SI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
76; SI-NEXT:    s_setpc_b64 s[30:31]
77;
78; GFX9-LABEL: mad_u64_u32_zextops:
79; GFX9:       ; %bb.0:
80; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
81; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[2:3]
82; GFX9-NEXT:    s_setpc_b64 s[30:31]
83  %sext0 = zext i32 %arg0 to i64
84  %sext1 = zext i32 %arg1 to i64
85  %mul = mul i64 %sext0, %sext1
86  %mad = add i64 %mul, %arg2
87  ret i64 %mad
88}
89
90define i64 @mad_u64_u32_zextops_commute(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
91; CI-LABEL: mad_u64_u32_zextops_commute:
92; CI:       ; %bb.0:
93; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
94; CI-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[2:3]
95; CI-NEXT:    s_setpc_b64 s[30:31]
96;
97; SI-LABEL: mad_u64_u32_zextops_commute:
98; SI:       ; %bb.0:
99; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
100; SI-NEXT:    v_mul_lo_u32 v4, v0, v1
101; SI-NEXT:    v_mul_hi_u32 v1, v0, v1
102; SI-NEXT:    v_add_i32_e32 v0, vcc, v2, v4
103; SI-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
104; SI-NEXT:    s_setpc_b64 s[30:31]
105;
106; GFX9-LABEL: mad_u64_u32_zextops_commute:
107; GFX9:       ; %bb.0:
108; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
109; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[2:3]
110; GFX9-NEXT:    s_setpc_b64 s[30:31]
111  %sext0 = zext i32 %arg0 to i64
112  %sext1 = zext i32 %arg1 to i64
113  %mul = mul i64 %sext0, %sext1
114  %mad = add i64 %arg2, %mul
115  ret i64 %mad
116}
117
118define i128 @mad_i64_i32_sextops_i32_i128(i32 %arg0, i32 %arg1, i128 %arg2) #0 {
119; CI-LABEL: mad_i64_i32_sextops_i32_i128:
120; CI:       ; %bb.0:
121; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
122; CI-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v0, v1, 0
123; CI-NEXT:    v_ashrrev_i32_e32 v13, 31, v0
124; CI-NEXT:    v_mov_b32_e32 v8, 0
125; CI-NEXT:    v_mad_u64_u32 v[9:10], s[4:5], v13, v1, v[7:8]
126; CI-NEXT:    v_ashrrev_i32_e32 v14, 31, v1
127; CI-NEXT:    v_mad_i64_i32 v[11:12], s[4:5], v1, v13, 0
128; CI-NEXT:    v_mov_b32_e32 v7, v10
129; CI-NEXT:    v_mov_b32_e32 v10, v8
130; CI-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v0, v14, v[9:10]
131; CI-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v14, v0, v[11:12]
132; CI-NEXT:    v_add_i32_e32 v9, vcc, v7, v9
133; CI-NEXT:    v_addc_u32_e64 v10, s[4:5], 0, 0, vcc
134; CI-NEXT:    v_mad_u64_u32 v[9:10], s[4:5], v13, v14, v[9:10]
135; CI-NEXT:    v_add_i32_e32 v7, vcc, v9, v0
136; CI-NEXT:    v_addc_u32_e32 v9, vcc, v10, v1, vcc
137; CI-NEXT:    v_mov_b32_e32 v1, v8
138; CI-NEXT:    v_add_i32_e32 v0, vcc, v6, v2
139; CI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
140; CI-NEXT:    v_addc_u32_e32 v2, vcc, v7, v4, vcc
141; CI-NEXT:    v_addc_u32_e32 v3, vcc, v9, v5, vcc
142; CI-NEXT:    s_setpc_b64 s[30:31]
143;
144; SI-LABEL: mad_i64_i32_sextops_i32_i128:
145; SI:       ; %bb.0:
146; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
147; SI-NEXT:    v_ashrrev_i32_e32 v6, 31, v0
148; SI-NEXT:    v_mul_lo_u32 v11, v6, v1
149; SI-NEXT:    v_mul_hi_u32 v12, v0, v1
150; SI-NEXT:    v_ashrrev_i32_e32 v7, 31, v1
151; SI-NEXT:    v_mul_hi_u32 v14, v6, v1
152; SI-NEXT:    v_mul_lo_u32 v13, v0, v7
153; SI-NEXT:    v_mul_hi_u32 v10, v0, v7
154; SI-NEXT:    v_add_i32_e32 v12, vcc, v11, v12
155; SI-NEXT:    v_addc_u32_e32 v14, vcc, 0, v14, vcc
156; SI-NEXT:    v_mul_hi_u32 v8, v6, v7
157; SI-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
158; SI-NEXT:    v_addc_u32_e32 v10, vcc, 0, v10, vcc
159; SI-NEXT:    v_mul_i32_i24_e32 v9, v6, v7
160; SI-NEXT:    v_add_i32_e32 v10, vcc, v14, v10
161; SI-NEXT:    v_mul_hi_i32 v6, v1, v6
162; SI-NEXT:    v_mul_hi_i32 v7, v7, v0
163; SI-NEXT:    v_addc_u32_e64 v14, s[4:5], 0, 0, vcc
164; SI-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
165; SI-NEXT:    v_addc_u32_e32 v8, vcc, v8, v14, vcc
166; SI-NEXT:    v_add_i32_e32 v10, vcc, v13, v11
167; SI-NEXT:    v_mul_lo_u32 v0, v0, v1
168; SI-NEXT:    v_addc_u32_e32 v6, vcc, v7, v6, vcc
169; SI-NEXT:    v_add_i32_e32 v7, vcc, v9, v10
170; SI-NEXT:    v_addc_u32_e32 v6, vcc, v8, v6, vcc
171; SI-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
172; SI-NEXT:    v_addc_u32_e32 v1, vcc, v12, v3, vcc
173; SI-NEXT:    v_addc_u32_e32 v2, vcc, v7, v4, vcc
174; SI-NEXT:    v_addc_u32_e32 v3, vcc, v6, v5, vcc
175; SI-NEXT:    s_setpc_b64 s[30:31]
176;
177; GFX9-LABEL: mad_i64_i32_sextops_i32_i128:
178; GFX9:       ; %bb.0:
179; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
180; GFX9-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v0, v1, 0
181; GFX9-NEXT:    v_ashrrev_i32_e32 v13, 31, v0
182; GFX9-NEXT:    v_mov_b32_e32 v9, 0
183; GFX9-NEXT:    v_mov_b32_e32 v8, v7
184; GFX9-NEXT:    v_mad_u64_u32 v[10:11], s[4:5], v13, v1, v[8:9]
185; GFX9-NEXT:    v_ashrrev_i32_e32 v14, 31, v1
186; GFX9-NEXT:    v_mov_b32_e32 v8, v11
187; GFX9-NEXT:    v_mov_b32_e32 v11, v9
188; GFX9-NEXT:    v_mad_u64_u32 v[10:11], s[4:5], v0, v14, v[10:11]
189; GFX9-NEXT:    v_mov_b32_e32 v12, v11
190; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, v8, v12
191; GFX9-NEXT:    v_addc_co_u32_e64 v9, s[4:5], 0, 0, vcc
192; GFX9-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v13, v14, v[8:9]
193; GFX9-NEXT:    v_mad_i64_i32 v[12:13], s[4:5], v1, v13, 0
194; GFX9-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v14, v0, v[12:13]
195; GFX9-NEXT:    v_add_co_u32_e32 v7, vcc, v8, v0
196; GFX9-NEXT:    v_addc_co_u32_e32 v8, vcc, v9, v1, vcc
197; GFX9-NEXT:    v_mov_b32_e32 v1, v10
198; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v6, v2
199; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
200; GFX9-NEXT:    v_addc_co_u32_e32 v2, vcc, v7, v4, vcc
201; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v8, v5, vcc
202; GFX9-NEXT:    s_setpc_b64 s[30:31]
203  %sext0 = sext i32 %arg0 to i128
204  %sext1 = sext i32 %arg1 to i128
205  %mul = mul i128 %sext0, %sext1
206  %mad = add i128 %mul, %arg2
207  ret i128 %mad
208}
209
210define i63 @mad_i64_i32_sextops_i32_i63(i32 %arg0, i32 %arg1, i63 %arg2) #0 {
211; CI-LABEL: mad_i64_i32_sextops_i32_i63:
212; CI:       ; %bb.0:
213; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
214; CI-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3]
215; CI-NEXT:    s_setpc_b64 s[30:31]
216;
217; SI-LABEL: mad_i64_i32_sextops_i32_i63:
218; SI:       ; %bb.0:
219; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
220; SI-NEXT:    v_mul_lo_u32 v4, v0, v1
221; SI-NEXT:    v_mul_hi_i32 v1, v0, v1
222; SI-NEXT:    v_add_i32_e32 v0, vcc, v4, v2
223; SI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
224; SI-NEXT:    s_setpc_b64 s[30:31]
225;
226; GFX9-LABEL: mad_i64_i32_sextops_i32_i63:
227; GFX9:       ; %bb.0:
228; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
229; GFX9-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3]
230; GFX9-NEXT:    s_setpc_b64 s[30:31]
231  %sext0 = sext i32 %arg0 to i63
232  %sext1 = sext i32 %arg1 to i63
233  %mul = mul i63 %sext0, %sext1
234  %mad = add i63 %mul, %arg2
235  ret i63 %mad
236}
237
238define i63 @mad_i64_i32_sextops_i31_i63(i31 %arg0, i31 %arg1, i63 %arg2) #0 {
239; CI-LABEL: mad_i64_i32_sextops_i31_i63:
240; CI:       ; %bb.0:
241; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
242; CI-NEXT:    v_bfe_i32 v1, v1, 0, 31
243; CI-NEXT:    v_bfe_i32 v0, v0, 0, 31
244; CI-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3]
245; CI-NEXT:    s_setpc_b64 s[30:31]
246;
247; SI-LABEL: mad_i64_i32_sextops_i31_i63:
248; SI:       ; %bb.0:
249; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
250; SI-NEXT:    v_lshlrev_b32_e32 v4, 1, v0
251; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v1
252; SI-NEXT:    v_ashr_i64 v[4:5], v[3:4], 33
253; SI-NEXT:    v_ashr_i64 v[0:1], v[0:1], 33
254; SI-NEXT:    v_mul_lo_u32 v1, v4, v0
255; SI-NEXT:    v_mul_hi_i32 v4, v4, v0
256; SI-NEXT:    v_add_i32_e32 v0, vcc, v1, v2
257; SI-NEXT:    v_addc_u32_e32 v1, vcc, v4, v3, vcc
258; SI-NEXT:    s_setpc_b64 s[30:31]
259;
260; GFX9-LABEL: mad_i64_i32_sextops_i31_i63:
261; GFX9:       ; %bb.0:
262; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
263; GFX9-NEXT:    v_bfe_i32 v1, v1, 0, 31
264; GFX9-NEXT:    v_bfe_i32 v0, v0, 0, 31
265; GFX9-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3]
266; GFX9-NEXT:    s_setpc_b64 s[30:31]
267  %sext0 = sext i31 %arg0 to i63
268  %sext1 = sext i31 %arg1 to i63
269  %mul = mul i63 %sext0, %sext1
270  %mad = add i63 %mul, %arg2
271  ret i63 %mad
272}
273
274define i64 @mad_i64_i32_extops_i32_i64(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
275; CI-LABEL: mad_i64_i32_extops_i32_i64:
276; CI:       ; %bb.0:
277; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
278; CI-NEXT:    v_ashrrev_i32_e32 v4, 31, v0
279; CI-NEXT:    v_mul_lo_u32 v4, v4, v1
280; CI-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[2:3]
281; CI-NEXT:    v_add_i32_e32 v1, vcc, v4, v1
282; CI-NEXT:    s_setpc_b64 s[30:31]
283;
284; SI-LABEL: mad_i64_i32_extops_i32_i64:
285; SI:       ; %bb.0:
286; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
287; SI-NEXT:    v_ashrrev_i32_e32 v4, 31, v0
288; SI-NEXT:    v_mul_hi_u32 v5, v0, v1
289; SI-NEXT:    v_mul_lo_u32 v4, v4, v1
290; SI-NEXT:    v_mul_lo_u32 v0, v0, v1
291; SI-NEXT:    v_add_i32_e32 v1, vcc, v5, v4
292; SI-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
293; SI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
294; SI-NEXT:    s_setpc_b64 s[30:31]
295;
296; GFX9-LABEL: mad_i64_i32_extops_i32_i64:
297; GFX9:       ; %bb.0:
298; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
299; GFX9-NEXT:    v_mov_b32_e32 v4, v1
300; GFX9-NEXT:    v_ashrrev_i32_e32 v5, 31, v0
301; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v4, v[2:3]
302; GFX9-NEXT:    v_mov_b32_e32 v2, v1
303; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v5, v4, v[2:3]
304; GFX9-NEXT:    v_mov_b32_e32 v1, v2
305; GFX9-NEXT:    s_setpc_b64 s[30:31]
306  %ext0 = sext i32 %arg0 to i64
307  %ext1 = zext i32 %arg1 to i64
308  %mul = mul i64 %ext0, %ext1
309  %mad = add i64 %mul, %arg2
310  ret i64 %mad
311}
312
313define i64 @mad_u64_u32_bitops(i64 %arg0, i64 %arg1, i64 %arg2) #0 {
314; CI-LABEL: mad_u64_u32_bitops:
315; CI:       ; %bb.0:
316; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
317; CI-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v2, v[4:5]
318; CI-NEXT:    s_setpc_b64 s[30:31]
319;
320; SI-LABEL: mad_u64_u32_bitops:
321; SI:       ; %bb.0:
322; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
323; SI-NEXT:    v_mul_lo_u32 v1, v0, v2
324; SI-NEXT:    v_mul_hi_u32 v2, v0, v2
325; SI-NEXT:    v_add_i32_e32 v0, vcc, v1, v4
326; SI-NEXT:    v_addc_u32_e32 v1, vcc, v2, v5, vcc
327; SI-NEXT:    s_setpc_b64 s[30:31]
328;
329; GFX9-LABEL: mad_u64_u32_bitops:
330; GFX9:       ; %bb.0:
331; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
332; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v2, v[4:5]
333; GFX9-NEXT:    s_setpc_b64 s[30:31]
334  %trunc.lhs = and i64 %arg0, 4294967295
335  %trunc.rhs = and i64 %arg1, 4294967295
336  %mul = mul i64 %trunc.lhs, %trunc.rhs
337  %add = add i64 %mul, %arg2
338  ret i64 %add
339}
340
341define i64 @mad_u64_u32_bitops_lhs_mask_small(i64 %arg0, i64 %arg1, i64 %arg2) #0 {
342; CI-LABEL: mad_u64_u32_bitops_lhs_mask_small:
343; CI:       ; %bb.0:
344; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
345; CI-NEXT:    v_and_b32_e32 v3, 1, v1
346; CI-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v2, v[4:5]
347; CI-NEXT:    v_mul_lo_u32 v2, v3, v2
348; CI-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
349; CI-NEXT:    s_setpc_b64 s[30:31]
350;
351; SI-LABEL: mad_u64_u32_bitops_lhs_mask_small:
352; SI:       ; %bb.0:
353; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
354; SI-NEXT:    v_and_b32_e32 v1, 1, v1
355; SI-NEXT:    v_mul_hi_u32 v3, v0, v2
356; SI-NEXT:    v_mul_lo_u32 v1, v1, v2
357; SI-NEXT:    v_mul_lo_u32 v0, v0, v2
358; SI-NEXT:    v_add_i32_e32 v1, vcc, v3, v1
359; SI-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
360; SI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v5, vcc
361; SI-NEXT:    s_setpc_b64 s[30:31]
362;
363; GFX9-LABEL: mad_u64_u32_bitops_lhs_mask_small:
364; GFX9:       ; %bb.0:
365; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
366; GFX9-NEXT:    v_and_b32_e32 v3, 1, v1
367; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v0, v2, v[4:5]
368; GFX9-NEXT:    v_mov_b32_e32 v4, v1
369; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v3, v2, v[4:5]
370; GFX9-NEXT:    v_mov_b32_e32 v1, v2
371; GFX9-NEXT:    s_setpc_b64 s[30:31]
372  %trunc.lhs = and i64 %arg0, 8589934591
373  %trunc.rhs = and i64 %arg1, 4294967295
374  %mul = mul i64 %trunc.lhs, %trunc.rhs
375  %add = add i64 %mul, %arg2
376  ret i64 %add
377}
378
379define i64 @mad_u64_u32_bitops_rhs_mask_small(i64 %arg0, i64 %arg1, i64 %arg2) #0 {
380; CI-LABEL: mad_u64_u32_bitops_rhs_mask_small:
381; CI:       ; %bb.0:
382; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
383; CI-NEXT:    v_mov_b32_e32 v6, v0
384; CI-NEXT:    v_and_b32_e32 v3, 1, v3
385; CI-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v6, v2, v[4:5]
386; CI-NEXT:    v_mul_lo_u32 v2, v6, v3
387; CI-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
388; CI-NEXT:    s_setpc_b64 s[30:31]
389;
390; SI-LABEL: mad_u64_u32_bitops_rhs_mask_small:
391; SI:       ; %bb.0:
392; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
393; SI-NEXT:    v_and_b32_e32 v1, 1, v3
394; SI-NEXT:    v_mul_hi_u32 v3, v0, v2
395; SI-NEXT:    v_mul_lo_u32 v1, v0, v1
396; SI-NEXT:    v_mul_lo_u32 v0, v0, v2
397; SI-NEXT:    v_add_i32_e32 v1, vcc, v3, v1
398; SI-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
399; SI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v5, vcc
400; SI-NEXT:    s_setpc_b64 s[30:31]
401;
402; GFX9-LABEL: mad_u64_u32_bitops_rhs_mask_small:
403; GFX9:       ; %bb.0:
404; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
405; GFX9-NEXT:    v_mov_b32_e32 v6, v0
406; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v6, v2, v[4:5]
407; GFX9-NEXT:    v_and_b32_e32 v3, 1, v3
408; GFX9-NEXT:    v_mov_b32_e32 v2, v1
409; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v6, v3, v[2:3]
410; GFX9-NEXT:    v_mov_b32_e32 v1, v2
411; GFX9-NEXT:    s_setpc_b64 s[30:31]
412  %trunc.lhs = and i64 %arg0, 4294967295
413  %trunc.rhs = and i64 %arg1, 8589934591
414  %mul = mul i64 %trunc.lhs, %trunc.rhs
415  %add = add i64 %mul, %arg2
416  ret i64 %add
417}
418
419define i64 @mad_i64_i32_bitops(i64 %arg0, i64 %arg1, i64 %arg2) #0 {
420; CI-LABEL: mad_i64_i32_bitops:
421; CI:       ; %bb.0:
422; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
423; CI-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v2, v[4:5]
424; CI-NEXT:    s_setpc_b64 s[30:31]
425;
426; SI-LABEL: mad_i64_i32_bitops:
427; SI:       ; %bb.0:
428; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
429; SI-NEXT:    v_mul_lo_u32 v1, v0, v2
430; SI-NEXT:    v_mul_hi_i32 v2, v0, v2
431; SI-NEXT:    v_add_i32_e32 v0, vcc, v1, v4
432; SI-NEXT:    v_addc_u32_e32 v1, vcc, v2, v5, vcc
433; SI-NEXT:    s_setpc_b64 s[30:31]
434;
435; GFX9-LABEL: mad_i64_i32_bitops:
436; GFX9:       ; %bb.0:
437; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
438; GFX9-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v2, v[4:5]
439; GFX9-NEXT:    s_setpc_b64 s[30:31]
440  %shl.lhs = shl i64 %arg0, 32
441  %trunc.lhs = ashr i64 %shl.lhs, 32
442  %shl.rhs = shl i64 %arg1, 32
443  %trunc.rhs = ashr i64 %shl.rhs, 32
444  %mul = mul i64 %trunc.lhs, %trunc.rhs
445  %add = add i64 %mul, %arg2
446  ret i64 %add
447}
448
449; Example from bug report
450define i64 @mad_i64_i32_unpack_i64ops(i64 %arg0) #0 {
451; CI-LABEL: mad_i64_i32_unpack_i64ops:
452; CI:       ; %bb.0:
453; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
454; CI-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v1, v0, v[0:1]
455; CI-NEXT:    s_setpc_b64 s[30:31]
456;
457; SI-LABEL: mad_i64_i32_unpack_i64ops:
458; SI:       ; %bb.0:
459; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
460; SI-NEXT:    v_mul_lo_u32 v2, v1, v0
461; SI-NEXT:    v_mul_hi_u32 v3, v1, v0
462; SI-NEXT:    v_add_i32_e32 v0, vcc, v2, v0
463; SI-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
464; SI-NEXT:    s_setpc_b64 s[30:31]
465;
466; GFX9-LABEL: mad_i64_i32_unpack_i64ops:
467; GFX9:       ; %bb.0:
468; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
469; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v1, v0, v[0:1]
470; GFX9-NEXT:    s_setpc_b64 s[30:31]
471  %tmp4 = lshr i64 %arg0, 32
472  %tmp5 = and i64 %arg0, 4294967295
473  %mul = mul nuw i64 %tmp4, %tmp5
474  %mad = add i64 %mul, %arg0
475  ret i64 %mad
476}
477
478define amdgpu_kernel void @mad_i64_i32_uniform(i64 addrspace(1)* %out, i32 %arg0, i32 %arg1, i64 %arg2) #0 {
479; CI-LABEL: mad_i64_i32_uniform:
480; CI:       ; %bb.0:
481; CI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xb
482; CI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
483; CI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
484; CI-NEXT:    s_waitcnt lgkmcnt(0)
485; CI-NEXT:    v_mov_b32_e32 v2, s3
486; CI-NEXT:    v_mov_b32_e32 v0, s4
487; CI-NEXT:    v_mov_b32_e32 v1, s5
488; CI-NEXT:    v_mad_u64_u32 v[0:1], s[2:3], s2, v2, v[0:1]
489; CI-NEXT:    s_mov_b32 s3, 0xf000
490; CI-NEXT:    s_mov_b32 s2, -1
491; CI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
492; CI-NEXT:    s_endpgm
493;
494; SI-LABEL: mad_i64_i32_uniform:
495; SI:       ; %bb.0:
496; SI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xb
497; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
498; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
499; SI-NEXT:    s_mov_b32 s7, 0xf000
500; SI-NEXT:    s_mov_b32 s6, -1
501; SI-NEXT:    s_waitcnt lgkmcnt(0)
502; SI-NEXT:    v_mov_b32_e32 v0, s3
503; SI-NEXT:    v_mul_hi_u32 v1, s2, v0
504; SI-NEXT:    s_mul_i32 s2, s2, s3
505; SI-NEXT:    v_mov_b32_e32 v0, s2
506; SI-NEXT:    v_mov_b32_e32 v2, s1
507; SI-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
508; SI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v2, vcc
509; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
510; SI-NEXT:    s_endpgm
511;
512; GFX9-LABEL: mad_i64_i32_uniform:
513; GFX9:       ; %bb.0:
514; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
515; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
516; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0x24
517; GFX9-NEXT:    v_mov_b32_e32 v2, 0
518; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
519; GFX9-NEXT:    s_mul_i32 s0, s2, s3
520; GFX9-NEXT:    s_mul_hi_u32 s1, s2, s3
521; GFX9-NEXT:    s_add_u32 s0, s0, s4
522; GFX9-NEXT:    s_addc_u32 s1, s1, s5
523; GFX9-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
524; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
525; GFX9-NEXT:    s_endpgm
526  %ext0 = zext i32 %arg0 to i64
527  %ext1 = zext i32 %arg1 to i64
528  %mul = mul i64 %ext0, %ext1
529  %mad = add i64 %mul, %arg2
530  store i64 %mad, i64 addrspace(1)* %out
531  ret void
532}
533
534define i64 @mad_i64_i32_twice(i32 %arg0, i32 %arg1, i64 %arg2, i64 %arg3) #0 {
535; CI-LABEL: mad_i64_i32_twice:
536; CI:       ; %bb.0:
537; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
538; CI-NEXT:    v_mad_i64_i32 v[2:3], s[4:5], v0, v1, v[2:3]
539; CI-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[4:5]
540; CI-NEXT:    v_xor_b32_e32 v1, v3, v1
541; CI-NEXT:    v_xor_b32_e32 v0, v2, v0
542; CI-NEXT:    s_setpc_b64 s[30:31]
543;
544; SI-LABEL: mad_i64_i32_twice:
545; SI:       ; %bb.0:
546; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
547; SI-NEXT:    v_mul_lo_u32 v6, v0, v1
548; SI-NEXT:    v_mul_hi_i32 v0, v0, v1
549; SI-NEXT:    v_add_i32_e32 v2, vcc, v6, v2
550; SI-NEXT:    v_addc_u32_e32 v1, vcc, v0, v3, vcc
551; SI-NEXT:    v_add_i32_e32 v3, vcc, v6, v4
552; SI-NEXT:    v_addc_u32_e32 v0, vcc, v0, v5, vcc
553; SI-NEXT:    v_xor_b32_e32 v1, v1, v0
554; SI-NEXT:    v_xor_b32_e32 v0, v2, v3
555; SI-NEXT:    s_setpc_b64 s[30:31]
556;
557; GFX9-LABEL: mad_i64_i32_twice:
558; GFX9:       ; %bb.0:
559; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
560; GFX9-NEXT:    v_mad_i64_i32 v[2:3], s[4:5], v0, v1, v[2:3]
561; GFX9-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[4:5]
562; GFX9-NEXT:    v_xor_b32_e32 v1, v3, v1
563; GFX9-NEXT:    v_xor_b32_e32 v0, v2, v0
564; GFX9-NEXT:    s_setpc_b64 s[30:31]
565  %sext0 = sext i32 %arg0 to i64
566  %sext1 = sext i32 %arg1 to i64
567  %mul = mul i64 %sext0, %sext1
568  %mad1 = add i64 %mul, %arg2
569  %mad2 = add i64 %mul, %arg3
570  %out = xor i64 %mad1, %mad2
571  ret i64 %out
572}
573
574define i64 @mad_i64_i32_thrice(i32 %arg0, i32 %arg1, i64 %arg2, i64 %arg3, i64 %arg4) #0 {
575; CI-LABEL: mad_i64_i32_thrice:
576; CI:       ; %bb.0:
577; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
578; CI-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, 0
579; CI-NEXT:    v_add_i32_e32 v2, vcc, v0, v2
580; CI-NEXT:    v_addc_u32_e32 v3, vcc, v1, v3, vcc
581; CI-NEXT:    v_add_i32_e32 v4, vcc, v0, v4
582; CI-NEXT:    v_addc_u32_e32 v5, vcc, v1, v5, vcc
583; CI-NEXT:    v_add_i32_e32 v0, vcc, v0, v6
584; CI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v7, vcc
585; CI-NEXT:    v_xor_b32_e32 v3, v3, v5
586; CI-NEXT:    v_xor_b32_e32 v2, v2, v4
587; CI-NEXT:    v_xor_b32_e32 v1, v3, v1
588; CI-NEXT:    v_xor_b32_e32 v0, v2, v0
589; CI-NEXT:    s_setpc_b64 s[30:31]
590;
591; SI-LABEL: mad_i64_i32_thrice:
592; SI:       ; %bb.0:
593; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
594; SI-NEXT:    v_mul_lo_u32 v8, v0, v1
595; SI-NEXT:    v_mul_hi_i32 v0, v0, v1
596; SI-NEXT:    v_add_i32_e32 v1, vcc, v8, v2
597; SI-NEXT:    v_addc_u32_e32 v2, vcc, v0, v3, vcc
598; SI-NEXT:    v_add_i32_e32 v3, vcc, v8, v4
599; SI-NEXT:    v_addc_u32_e32 v4, vcc, v0, v5, vcc
600; SI-NEXT:    v_add_i32_e32 v5, vcc, v8, v6
601; SI-NEXT:    v_addc_u32_e32 v0, vcc, v0, v7, vcc
602; SI-NEXT:    v_xor_b32_e32 v2, v2, v4
603; SI-NEXT:    v_xor_b32_e32 v3, v1, v3
604; SI-NEXT:    v_xor_b32_e32 v1, v2, v0
605; SI-NEXT:    v_xor_b32_e32 v0, v3, v5
606; SI-NEXT:    s_setpc_b64 s[30:31]
607;
608; GFX9-LABEL: mad_i64_i32_thrice:
609; GFX9:       ; %bb.0:
610; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
611; GFX9-NEXT:    v_mad_i64_i32 v[2:3], s[4:5], v0, v1, v[2:3]
612; GFX9-NEXT:    v_mad_i64_i32 v[4:5], s[4:5], v0, v1, v[4:5]
613; GFX9-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[6:7]
614; GFX9-NEXT:    v_xor_b32_e32 v3, v3, v5
615; GFX9-NEXT:    v_xor_b32_e32 v2, v2, v4
616; GFX9-NEXT:    v_xor_b32_e32 v1, v3, v1
617; GFX9-NEXT:    v_xor_b32_e32 v0, v2, v0
618; GFX9-NEXT:    s_setpc_b64 s[30:31]
619  %sext0 = sext i32 %arg0 to i64
620  %sext1 = sext i32 %arg1 to i64
621  %mul = mul i64 %sext0, %sext1
622  %mad1 = add i64 %mul, %arg2
623  %mad2 = add i64 %mul, %arg3
624  %mad3 = add i64 %mul, %arg4
625  %out.p = xor i64 %mad1, %mad2
626  %out = xor i64 %out.p, %mad3
627  ret i64 %out
628}
629
630define i64 @mad_i64_i32_secondary_use(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
631; CI-LABEL: mad_i64_i32_secondary_use:
632; CI:       ; %bb.0:
633; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
634; CI-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, 0
635; CI-NEXT:    v_add_i32_e32 v2, vcc, v0, v2
636; CI-NEXT:    v_addc_u32_e32 v3, vcc, v1, v3, vcc
637; CI-NEXT:    v_xor_b32_e32 v1, v3, v1
638; CI-NEXT:    v_xor_b32_e32 v0, v2, v0
639; CI-NEXT:    s_setpc_b64 s[30:31]
640;
641; SI-LABEL: mad_i64_i32_secondary_use:
642; SI:       ; %bb.0:
643; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
644; SI-NEXT:    v_mul_lo_u32 v4, v0, v1
645; SI-NEXT:    v_mul_hi_i32 v0, v0, v1
646; SI-NEXT:    v_add_i32_e32 v2, vcc, v4, v2
647; SI-NEXT:    v_addc_u32_e32 v1, vcc, v0, v3, vcc
648; SI-NEXT:    v_xor_b32_e32 v1, v1, v0
649; SI-NEXT:    v_xor_b32_e32 v0, v2, v4
650; SI-NEXT:    s_setpc_b64 s[30:31]
651;
652; GFX9-LABEL: mad_i64_i32_secondary_use:
653; GFX9:       ; %bb.0:
654; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
655; GFX9-NEXT:    v_mad_i64_i32 v[4:5], s[4:5], v0, v1, 0
656; GFX9-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v0, v1, v[2:3]
657; GFX9-NEXT:    v_xor_b32_e32 v1, v1, v5
658; GFX9-NEXT:    v_xor_b32_e32 v0, v0, v4
659; GFX9-NEXT:    s_setpc_b64 s[30:31]
660  %sext0 = sext i32 %arg0 to i64
661  %sext1 = sext i32 %arg1 to i64
662  %mul = mul i64 %sext0, %sext1
663  %mad = add i64 %mul, %arg2
664  %out = xor i64 %mad, %mul
665  ret i64 %out
666}
667
668define i48 @mad_i48_i48(i48 %arg0, i48 %arg1, i48 %arg2) #0 {
669; CI-LABEL: mad_i48_i48:
670; CI:       ; %bb.0:
671; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
672; CI-NEXT:    v_mov_b32_e32 v6, v1
673; CI-NEXT:    v_mov_b32_e32 v7, v0
674; CI-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v7, v2, v[4:5]
675; CI-NEXT:    v_mul_lo_u32 v2, v6, v2
676; CI-NEXT:    v_mul_lo_u32 v3, v7, v3
677; CI-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
678; CI-NEXT:    v_add_i32_e32 v1, vcc, v3, v1
679; CI-NEXT:    s_setpc_b64 s[30:31]
680;
681; SI-LABEL: mad_i48_i48:
682; SI:       ; %bb.0:
683; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
684; SI-NEXT:    v_mul_lo_u32 v3, v0, v3
685; SI-NEXT:    v_mul_hi_u32 v6, v0, v2
686; SI-NEXT:    v_mul_lo_u32 v1, v1, v2
687; SI-NEXT:    v_mul_lo_u32 v0, v0, v2
688; SI-NEXT:    v_add_i32_e32 v3, vcc, v6, v3
689; SI-NEXT:    v_add_i32_e32 v1, vcc, v3, v1
690; SI-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
691; SI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v5, vcc
692; SI-NEXT:    s_setpc_b64 s[30:31]
693;
694; GFX9-LABEL: mad_i48_i48:
695; GFX9:       ; %bb.0:
696; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
697; GFX9-NEXT:    v_mov_b32_e32 v6, v1
698; GFX9-NEXT:    v_mov_b32_e32 v7, v0
699; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v7, v2, v[4:5]
700; GFX9-NEXT:    v_mul_lo_u32 v3, v7, v3
701; GFX9-NEXT:    v_mul_lo_u32 v2, v6, v2
702; GFX9-NEXT:    v_add3_u32 v1, v2, v1, v3
703; GFX9-NEXT:    s_setpc_b64 s[30:31]
704  %m = mul i48 %arg0, %arg1
705  %a = add i48 %m, %arg2
706  ret i48 %a
707}
708
709attributes #0 = { nounwind }
710attributes #1 = { nounwind readnone speculatable }
711